From b80df0c22b593a04e02e3f1d43f2e2b66157b92f Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 09:07:07 -0700 Subject: [PATCH 001/127] test(git): establish indexed read scaling baseline --- Cargo.lock | 2 + .../docs/architecture/indexed-remote-reads.md | 197 +++++++ crab/scripts/e2e/run_concurrent_push_smoke.py | 9 +- .../e2e/run_large_file_edits_rustfs.py | 11 +- .../e2e/run_remote_read_scaling_rustfs.py | 187 ++++++ crab/scripts/e2e/test_remote_read_scaling.py | 87 +++ .../e2e/test_run_concurrent_push_smoke.py | 3 + crab/scripts/e2e/verify_remote_browse.py | 161 ++++- crates/crab-metadata/src/path_state/codec.rs | 2 + .../crab-metadata/src/split_commit_graph.rs | 2 + crates/crab-remote-git/Cargo.toml | 2 + crates/crab-remote-git/REFERENCE.md | 34 +- .../examples/qualify_browse.rs | 116 +++- .../examples/qualify_remote.rs | 553 ++++++++---------- .../crab-remote-git/examples/support/mod.rs | 282 +++++++++ 15 files changed, 1257 insertions(+), 391 deletions(-) create mode 100644 crab/docs/architecture/indexed-remote-reads.md create mode 100644 crab/scripts/e2e/run_remote_read_scaling_rustfs.py create mode 100644 crab/scripts/e2e/test_remote_read_scaling.py create mode 100644 crates/crab-remote-git/examples/support/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 881f38c15..ff8852f4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2555,6 +2555,8 @@ dependencies = [ "bytes", "crab-git", "crab-metadata", + "crab-read", + "crab-remote", "crab-storage", "crab-xet", "flate2", diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md new file mode 100644 index 000000000..e1bde7f1b --- /dev/null +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -0,0 +1,197 @@ +# Indexed remote reads: implementation and qualification + +Status: implementation in progress. This document tracks the entire initiative; +passing one delivery gate does not establish completion of the later gates. +Baseline: `bb590b78f58` on `crab-v2`, including the large-file push audit. + +## Required behavior + +A fixed request must load metadata proportional to its requested objects, paths, +or relevant file changes, with bounded tree-depth and compaction-level overhead. +Increasing unrelated history must not introduce proportional decoding, scanning, +or memory consumption. Requests needing complete output, including enumeration, +archives, full integrity checks and graph traversal, retain output-sized costs. + +Repository/ref authority is captured before queries. Index presence never grants +visibility. Derived records bind the root and visible per-ref positions, including +multi-ref activation. Reader snapshots remain immutable across publication. + +Keep current first-parent file/tree attribution and ordinary UTF-8 blame semantics +through the performance milestone. Full merge-parent attribution, rename following, +source line positions and supported blame options have a separate correctness gate. +Xet version attribution requires no payload hydration. Logical line blame for large +Xet files and unified local staging are subsequent workstreams. + +## Delivery gates and evidence + +| Order | Deliverable | Completion evidence | Current state | +| --- | --- | --- | --- | +| 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | +| 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Pending | +| 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Pending | +| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Pending | +| 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | +| 6 | Indexed historical Xet catalog | Indexed checkpoints and visible deltas; bounded compaction; 10-GiB edits and byte-identical cold historical hydration; protected receive and sibling consumers | Pending | +| 7 | Lifecycle and migration | Interrupted publication/builds, concurrent owners, GC retained roots/readers, stale caches and released-format migration qualification | Pending; also required in every earlier implementation | +| 8 | Fuller Git blame semantics | Line-porcelain parity across merges/renames/repeated lines/recreation and each supported whitespace/ignored-revision option | Pending | + +Each implementation PR owns one delivery gate or a complete vertical slice within +it. Tests alone are insufficient: user operations against a real S3-protocol store +must produce independently verified results. Representative cloud latency remains +a separate cell from local RustFS correctness. + +## Measurement contract + +`qualify_remote` opens capsule-v2 state, attaches only state-matching published +browse indexes, and executes one named query twice. Each workload has a fresh +runtime; its second query is warm. Opening is reported independently before and +after the query. The cold query includes required snapshot/path setup so those +reads cannot disappear into unreported preparation. It is cold relative to the +captured handle: opening may already load inline pack/index data, which belongs +to the opening measurements. + +The backend observer includes reads through `Store::inner()`, including locator +SST/manifest access. It reports logical backend invocations and consumed bytes; +provider HTTP retries and listing pages require the separate HTTP meter. Decoder +observations count verified graph/path-state blocks and their encoded input bytes. +Current blocks are layers, not individual pages. Do not relabel those bytes as +network transfer or resident memory. New paged formats must add their decode events +at the actual decoding boundary. + +The HTTP meter records requests and request/response bytes by bounded object +class. Capsule/run/checkpoint objects can contain both control and Git payload +ranges: report that class as mixed rather than pretending its bytes are entirely +metadata or payload. Standalone index, pack and xorb classes can be reported +separately. Raw request paths are disabled in the scaling harness. Timing and +counts contain no raw paths, OIDs, endpoints, content or credentials; separate +oracle evidence contains the paths/OIDs needed to prove correctness. + +`/usr/bin/time` supplies process CPU and peak RSS. Those cover opening and both +queries together; they are not per-query RSS. Index construction uses an explicit +`build-indexes` operation, outside measured reads, with its own requests, bytes, +CPU and peak RSS. Read-only qualifications do not trigger maintenance. + +The verifier checks complete cold/warm measurements, unfinished backend work, +Git object content hashes, page order/continuation, first-parent history and every +blame line. Tree attribution is compared with native Git path history. Verification +remains enabled under `python -O`. + +## Reproducible qualification + +Use the existing per-checkout target on the mounted workspace volume; set it on +all compiling invocations. No local `target/` or external-source mutation. + +```sh +CARGO_TARGET_DIR="$HOME/Workspace/crabbuild-target/crab-ios-abf8fc00-push-audit" \ + cargo build -p crab-remote-git --release --examples --locked + +python3 crab/scripts/e2e/run_remote_read_scaling_rustfs.py \ + --root "$HOME/Workspace/CrabData/indexed-read-abf8fc00" \ + --endpoint-url http://127.0.0.1:19090 \ + --bucket --run-id \ + --crab-bin "$HOME/Workspace/crabbuild-target/crab-ios-abf8fc00-push-audit/release/crab" \ + --qualifier "$HOME/Workspace/crabbuild-target/crab-ios-abf8fc00-push-audit/release/examples/qualify_remote" \ + --history 100 1000 10000 --repeats 3 +``` + +The harness uses local development credentials and an existing RustFS service; +cloud qualification uses the same read binaries with authorized credentials and +a representative dedicated dataset. Run the existing large-file harness with +10,240 MiB per file separately. Preserve receipts outside the source checkout. + +The scaling fixture fixes the selected file at 50 lines and 20 content edits, +and the directory page at 100 entries. It changes a separate path in every +commit. Commit timestamps and data generation are deterministic. Fixtures are +published separately; native Git independently verifies every query. Initial +history cells exercise unrelated commit growth; distinct Xet chunk history, +ref-count growth and long uncompacted frontiers remain mandatory later cells. + +Latency targets must be established from pre-optimization receipts before +changing readers. Deterministic tests enforce request/page/work bounds; latency +qualification records cold/warm distributions, CPU, memory and maintenance cost. + +## Pre-optimization measurements and acceptance targets + +Local RustFS baseline `baseline-r1`: 100/1,000/10,000 commits, three isolated +process runs per workload, ten workloads, 90 native Git query comparisons, +450 checked blame lines, and 207 harness checks. All passed. Complete browse/archive oracles also passed at 100 and 10,000 +commits, checking 102 entries/blobs and 50 blame lines in each; 1,100 commit +records were checked across those runs. Receipts and exact +binary/fixture hashes remain under the external qualification root; they are not +source files. Timings below are query medians, excluding opening. + +| Commits | Cold attribution | Attribution decoded bytes | Cold blame | Attribution process peak RSS (median) | +| --- | --- | --- | --- | --- | +| 100 | 8.001 ms | 137,704 | 5.707 ms | 22,118,400 bytes | +| 1,000 | 29.865 ms | 1,355,056 | 27.127 ms | 34,488,320 bytes | +| 10,000 | 142.636 ms | 13,052,443 | 954.441 ms | 171,622,400 bytes | + +Opening also scales: the attribution runs consumed 150,598 / 1,118,721 / +6,695,676 backend bytes. Before a checkpoint the control-opening path retains +complete frontier capsules; lazy graph loading alone will not remove that cost. +Opening qualification must cover both pre-checkpoint frontiers and compacted +sources instead of warming payloads before the measured query. + +Acceptance targets, fixed before optimization, for this same fixture/provider: + +- At 10,000 commits, the fixed directory attribution query decodes at most + 512 KiB and downloads at most 1 MiB; it never decodes historical layers. +- First-parent blame resolves change points instead of every unrelated commit. + Its work-count regression must remain bounded by the 20 edits and path depth. +- For attribution and blame, the largest-history cold query p95 is no greater + than three times the smallest-history p95 plus 20 ms. Use at least 20 samples + in final performance qualification; three samples establish this baseline, + not a reliable tail-latency distribution. +- Opening and query decoded metadata/peak memory must not grow proportionally + with unrelated history. Exact request bounds follow the implemented page + height, requested paths and compaction-level limit and are tested directly. + +Every target also requires identical Git results. Cloud targets are established +from representative S3 measurements before the cloud optimization comparison; +local numbers do not establish cloud latency. + +## Release audit and migration boundary + +Audit performed against upstream peeled release commits, not local tag names: + +| Release | Upstream commit | +| --- | --- | +| v1.0.1 | `0cd19b61721ea07cdb5d997b07a8d55aace88732` | +| v1.2.0 | `773256d06a70dac187e16c3676f54cc629382b90` | +| v1.2.1 | `0f9fec98d77a9f30f3bc5a38af1b355871f2b3ca` | +| v1.2.2 | `30fae43517412f7e4b17bf7f1bab82232d8d986e` | +| v1.2.3 | `49d3791b484450dac4bc8350cdf15d7caf8f3e15` | +| v1.2.4 | `b4d5fd361f9a1a43fdd468dc604901d2c61006f0` | + +All listed commits contain `split_commit_graph.rs`; none contain the capsule +protocol module or `path_state.rs`. The local v1.2.4 tag differs from upstream; +upstream was fetched into a separate audit ref without overwriting the local tag. +Paged path-state/capsule formats can replace their unreleased predecessors. +Split graph migration must additionally inspect tagged producers, descriptor +bindings and repair paths before changing its published format. File absence +alone is not proof that every related serialized shape is unshipped. + +## Ownership and consumer coverage + +- `crab-metadata`: canonical codecs, index formats, validation and visibility. +- `crab-storage`: range I/O, retries and backend observation. +- `crab-read`: captured capsule views, Xet dependency resolution and hydration. +- `crab-write`: page durability, incremental builders, compaction and checkpoints. +- `crab-remote-git`: bounded queries, operation admission and explicit cleanup. +- `crab-remote`: maintenance coordination and state-fenced derived publication. +- `crab-http-server`: canonical reads, projection ingestion and readiness behavior. +- CLI push/protected receive/SDK/VFS/fsck/GC: relevant sibling contract consumers. + +Every format/read change records caller, callee, sibling, error and lifecycle +proof before publication. Retain maintenance streaming scans for integrity and +compaction; remove eager assembly from normal point queries. Do not introduce a +second authority database or an indefinite old-format runtime fallback. + +## Current validation limitation + +Strict example Clippy reaches pre-existing unfulfilled `double_must_use` lint +expectations in six coordinator trait annotations on the current `crab-v2` base. +Those files are unchanged by this work. A diagnostic run allowing only +`unfulfilled-lint-expectations` passes; this is not a strict Clippy pass and no +source suppression was added. Record the baseline issue separately from new +format/read correctness gates. diff --git a/crab/scripts/e2e/run_concurrent_push_smoke.py b/crab/scripts/e2e/run_concurrent_push_smoke.py index 4ba09715c..5bb583154 100755 --- a/crab/scripts/e2e/run_concurrent_push_smoke.py +++ b/crab/scripts/e2e/run_concurrent_push_smoke.py @@ -88,6 +88,8 @@ def __init__(self, upstream_url: str, key_prefix: str) -> None: self.operations: dict[str, int] = {} self.categories: dict[str, int] = {} self.classes: dict[str, int] = {} + self.class_request_bytes: dict[str, int] = {} + self.class_response_bytes: dict[str, int] = {} self.statuses: dict[str, int] = {} self.proxy_errors: dict[str, int] = {} self.trace_paths = os.environ.get("CRAB_E2E_TRACE_REQUEST_PATHS") == "1" @@ -530,6 +532,8 @@ def record( self.operations[operation] = self.operations.get(operation, 0) + 1 self.categories[category] = self.categories.get(category, 0) + 1 self.classes[request_class] = self.classes.get(request_class, 0) + 1 + self.class_request_bytes[request_class] = self.class_request_bytes.get(request_class, 0) + request_bytes + self.class_response_bytes[request_class] = self.class_response_bytes.get(request_class, 0) + response_bytes self.statuses[status_class] = self.statuses.get(status_class, 0) + 1 if proxy_error is not None: self.proxy_errors[proxy_error] = self.proxy_errors.get(proxy_error, 0) + 1 @@ -596,6 +600,8 @@ def snapshot(self, *, include_paths: bool = True) -> dict[str, Any]: "operations": dict(sorted(self.operations.items())), "categories": dict(sorted(self.categories.items())), "classes": dict(sorted(self.classes.items())), + "class_request_body_bytes": dict(sorted(self.class_request_bytes.items())), + "class_response_body_bytes": dict(sorted(self.class_response_bytes.items())), "statuses": dict(sorted(self.statuses.items())), "proxy_errors": dict(sorted(self.proxy_errors.items())), } @@ -614,7 +620,7 @@ def delta(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]: result: dict[str, Any] = {} for key in {"requests", "request_body_bytes", "response_body_bytes"}: result[key] = int(after.get(key, 0)) - int(before.get(key, 0)) - for key in {"methods", "operations", "categories", "classes", "statuses", "proxy_errors"}: + for key in {"methods", "operations", "categories", "classes", "statuses", "proxy_errors", "class_request_body_bytes", "class_response_body_bytes"}: earlier = before.get(key, {}) later = after.get(key, {}) result[key] = { @@ -702,6 +708,7 @@ def store_category(relative_key: str) -> str: return "/".join(parts[:3]) if len(parts) > 1 and parts[0] in { "git_object_catalog_db", + "v2", "git-visibility", "locks", "manifests", diff --git a/crab/scripts/e2e/run_large_file_edits_rustfs.py b/crab/scripts/e2e/run_large_file_edits_rustfs.py index 9b28d497a..71323b5bd 100644 --- a/crab/scripts/e2e/run_large_file_edits_rustfs.py +++ b/crab/scripts/e2e/run_large_file_edits_rustfs.py @@ -38,7 +38,16 @@ def run_cmd(self, name, args, cwd, **kwargs): match = re.search(pattern, stderr) if match: self.report.commands[-1]["max_rss_bytes"] = int(match[1]) * (1 if sys.platform == "darwin" else 1024) - self.write_report() + if sys.platform == "darwin": + cpu = re.search(r"([\d.]+) user\s+([\d.]+) sys", stderr) + if cpu: + self.report.commands[-1]["cpu_seconds"] = float(cpu[1]) + float(cpu[2]) + else: + user = re.search(r"User time \(seconds\):\s*([\d.]+)", stderr) + system = re.search(r"System time \(seconds\):\s*([\d.]+)", stderr) + if user and system: + self.report.commands[-1]["cpu_seconds"] = float(user[1]) + float(system[1]) + self.write_report() return result diff --git a/crab/scripts/e2e/run_remote_read_scaling_rustfs.py b/crab/scripts/e2e/run_remote_read_scaling_rustfs.py new file mode 100644 index 000000000..d7efcd793 --- /dev/null +++ b/crab/scripts/e2e/run_remote_read_scaling_rustfs.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Qualify fixed read workloads while increasing unrelated Git history on S3. + +Uses the existing smoke runner, HTTP meter and native Git oracle. Source fixtures +and receipts belong on the mounted workspace volume. Every workload runs in a +fresh process; its cold/warm query and opening measurements are reported apart. +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +from run_add_commit_push_rustfs_smoke import AddCommitPushSmoke, REPO_ROOT, sha256_file +from run_concurrent_push_smoke import RequestCountingProxy +from verify_remote_browse import verify + +WORKLOADS = ("resolve", "commit", "tree", "blob", "object-point", "object-batch", + "history", "path-history", "attribution", "blame") + + +def fixture_import(commits: int, changes: int = 20) -> bytes: + if commits <= changes: + raise ValueError("history must exceed the fixed number of file changes") + stream = bytearray() + positions = {(number * (commits - 1)) // changes: number for number in range(1, changes + 1)} + lines = [f"original line {number}\n" for number in range(50)] + + def file(path: str, body: bytes) -> None: + stream.extend(f"M 100644 inline {path}\ndata {len(body)}\n".encode()) + stream.extend(body + b"\n") + + for revision in range(commits): + stream.extend(f"commit refs/heads/main\ncommitter Qualification {1700000000 + revision} +0000\ndata 0\n\n".encode()) + if revision == 0: + for number in range(100): + file(f"file-{number:03}.txt", f"fixed file {number}\n".encode()) + change = positions.get(revision) + if revision == 0 or change is not None: + if change is not None: + lines[change - 1] = f"edited line {change}\n" + file("target.txt", "".join(lines).encode()) + file("unrelated.txt", f"unrelated revision {revision}\n".encode()) + stream.extend(b"\n") + stream.extend(b"done\n") + return bytes(stream) + + +class ReadScalingRunner(AddCommitPushSmoke): + def measured(self, name: str, args: list[str], cwd: Path): + timed = ["/usr/bin/time", "-l" if sys.platform == "darwin" else "-v", *args] + result = self.run_cmd(name, timed, cwd, timeout=self.args.timeout) + stderr = Path(result.stderr_log).read_text() + if sys.platform == "darwin": + rss = re.search(r"(\d+)\s+maximum resident set size", stderr) + cpu = re.search(r"([\d.]+) user\s+([\d.]+) sys", stderr) + else: + rss = re.search(r"Maximum resident set size \(kbytes\):\s*(\d+)", stderr) + user = re.search(r"User time \(seconds\):\s*([\d.]+)", stderr) + system = re.search(r"System time \(seconds\):\s*([\d.]+)", stderr) + cpu = None + resources = {"peak_rss_bytes": int(rss[1]) * (1 if sys.platform == "darwin" else 1024) if rss else None} + if sys.platform == "darwin": + resources["cpu_seconds"] = float(cpu[1]) + float(cpu[2]) if cpu else None + else: + resources["cpu_seconds"] = float(user[1]) + float(system[1]) if user and system else None + self.check(name + "-resource-accounting", all(value is not None for value in resources.values())) + return result, resources + + def qualify(self) -> None: + self.preflight() + self.report.artifacts["harness_sha256"] = sha256_file(Path(__file__)) + self.report.artifacts["qualifier_sha256"] = sha256_file(self.args.qualifier) + source_root = self.artifacts / "qualified-source" + source_root.mkdir() + inputs = ( + "Cargo.lock", "crates/crab-remote-git/Cargo.toml", + "crates/crab-remote-git/examples/qualify_remote.rs", + "crates/crab-remote-git/examples/support/mod.rs", + "crates/crab-metadata/src/path_state/codec.rs", + "crates/crab-metadata/src/split_commit_graph.rs", + "crab/scripts/e2e/run_remote_read_scaling_rustfs.py", + "crab/scripts/e2e/run_concurrent_push_smoke.py", + "crab/scripts/e2e/verify_remote_browse.py", + ) + hashes = {} + for relative in inputs: + target = source_root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((REPO_ROOT / relative).read_bytes()) + hashes[relative] = sha256_file(target) + manifest = source_root / "sha256.json" + manifest.write_text(json.dumps(hashes, indent=2) + "\n") + self.report.artifacts["qualified_source"] = str(manifest) + self.report.artifacts["source_patch_sha256"] = sha256_file(Path(self.run_git(REPO_ROOT, ["diff", "HEAD", "--binary"]).stdout_log)) + results = [] + for commits in self.args.history: + repo, remote, prefix = self.prepare_git_repo(f"history-{commits}") + source = fixture_import(commits) + import_file = self.artifacts / f"history-{commits}.fi" + import_file.write_bytes(source) + imported = subprocess.run(["git", "fast-import", "--quiet"], input=source, cwd=repo, + env=self.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=self.args.timeout) + record = self.record_command("fixture import", ["git", "fast-import", "--quiet"], repo, + imported.returncode, 0, imported.stdout.decode(), imported.stderr.decode()) + self.check(f"history-{commits}-import", record.exit_code == 0) + self.check(f"history-{commits}-fixed-file-changes", int(self.read_stdout( + self.run_git(repo, ["rev-list", "--count", "HEAD", "--", "target.txt"]))) == 21) + self.run_git(repo, ["fsck", "--full", "--strict"]) + self.run_crab(repo, ["push", "origin", "HEAD:refs/heads/main"], name=f"history-{commits} publish") + proxy = RequestCountingProxy(self.args.endpoint_url, self.args.bucket + "/" + prefix) + proxy.start() + old_endpoint = {key:self.env.get(key) for key in ("AWS_ENDPOINT_URL", "AWS_ENDPOINT_URL_S3")} + self.env.update(AWS_ENDPOINT_URL=proxy.url, AWS_ENDPOINT_URL_S3=proxy.url) + try: + before = proxy.snapshot() + maintenance, resources = self.measured(f"history-{commits} build-indexes", + [str(self.args.qualifier), self.args.bucket, prefix, "target.txt", "build-indexes"], repo) + results.append({"commits":commits, "operation":"build-indexes", "resources":resources, + "transport":RequestCountingProxy.delta(before, proxy.snapshot()), "evidence":maintenance.stdout_log}) + for name in WORKLOADS: + for repeat in range(self.args.repeats): + before = proxy.snapshot() + record, resources = self.measured(f"history-{commits} {name} repeat-{repeat}", + [str(self.args.qualifier), self.args.bucket, prefix, "target.txt", name], repo) + proof = verify(repo, Path(record.stdout_log), "HEAD") + self.check(f"history-{commits}-{name}-{repeat}-git-oracle", proof["status"] == "passed") + results.append({"commits":commits, "fixed_file_changes":20, "fixed_directory_entries":100, + "operation":name, "repeat":repeat, "resources":resources, "proof":proof, + "transport":RequestCountingProxy.delta(before, proxy.snapshot()), "evidence":record.stdout_log}) + self.write_results(results) + self.check(f"history-{commits}-transport-clean", not proxy.snapshot()["proxy_errors"]) + finally: + proxy.close() + for key, value in old_endpoint.items(): + if value is None: + self.env.pop(key, None) + else: + self.env[key] = value + self.check("qualifier-unchanged", sha256_file(self.args.qualifier) == self.report.artifacts["qualifier_sha256"]) + self.check_credential_disclosure() + self.report.status = "passed" + self.write_results(results) + + def write_results(self, results: list[dict]) -> None: + output = self.artifacts / "read-scaling.json" + output.write_text(json.dumps({"schema":1, "results":results}, indent=2) + "\n") + self.report.artifacts["read_scaling"] = str(output) + self.write_report() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--bucket", required=True) + parser.add_argument("--endpoint-url", required=True) + parser.add_argument("--crab-bin", required=True) + parser.add_argument("--qualifier", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--history", type=int, nargs="+", default=[100, 1000, 10000]) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--timeout", type=int, default=3600) + args = parser.parse_args() + if any(value <= 20 for value in args.history) or args.repeats < 1 or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", args.run_id): + parser.error("require histories >20, positive repeats, and a safe run id") + if not args.root.resolve().is_relative_to((Path.home() / "Workspace").resolve()): + parser.error("qualification root must be on the mounted Workspace volume") + args.root.mkdir(parents=True, exist_ok=True) + (args.root / args.run_id).mkdir() + args.access_key, args.secret_key, args.session_token = "crab", "crab", "" + args.region, args.push_timeout, args.source = "us-east-1", args.timeout, None + runner = ReadScalingRunner(args) + try: + runner.qualify() + except Exception as error: + runner.report.status = "failed" + runner.report.artifacts["failure"] = str(error) + raise + finally: + runner.write_report() + + +if __name__ == "__main__": + main() diff --git a/crab/scripts/e2e/test_remote_read_scaling.py b/crab/scripts/e2e/test_remote_read_scaling.py new file mode 100644 index 000000000..4595d00c0 --- /dev/null +++ b/crab/scripts/e2e/test_remote_read_scaling.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Regression checks for fixed-history inputs and the independent attribution oracle.""" +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from run_remote_read_scaling_rustfs import fixture_import +from verify_remote_browse import verify + + +class ReadScalingTests(unittest.TestCase): + def setUp(self): + root = Path.home() / "Workspace" / "CrabData" / "indexed-read-oracle-tests" + root.mkdir(parents=True, exist_ok=True) + self.workspace = tempfile.TemporaryDirectory(dir=root) + self.root = Path(self.workspace.name) + self.addCleanup(self.workspace.cleanup) + + def git(self, repo, *args, body=None): + return subprocess.check_output(["git", "-C", str(repo), *args], input=body, + stderr=subprocess.PIPE) + + def source(self, name, count): + repo = self.root / name + repo.mkdir() + self.git(repo, "init", "-b", "main") + self.git(repo, "fast-import", "--quiet", body=fixture_import(count)) + return repo + + def test_unrelated_history_keeps_target_bytes_and_change_count_fixed(self): + small, large = self.source("small", 25), self.source("large", 100) + self.assertEqual(self.git(small, "show", "HEAD:target.txt"), self.git(large, "show", "HEAD:target.txt")) + for repo, count in ((small, 25), (large, 100)): + self.assertEqual(int(self.git(repo, "rev-list", "--count", "HEAD")), count) + self.assertEqual(int(self.git(repo, "rev-list", "--count", "HEAD", "--", "target.txt")), 21) + + def evidence(self, repo): + head = self.git(repo, "rev-parse", "HEAD").decode().strip() + expected = self.git(repo, "blame", "--first-parent", "--root", "--line-porcelain", head, "--", "target.txt") + origins = [line.split()[0].decode() for line in expected.splitlines() + if len(line.split()) >= 3 and len(line.split()[0]) == 40 + and line.split()[1].isdigit() and not line.startswith(b"\t")] + entries = [{"start":line, "lines":1, "commit":origin, "source_path":list(b"target.txt")} + for line, origin in enumerate(origins, 1)] + records = [{"kind":"measurement", "operation":operation, "cache":cache, + "outcome":"ok", "active_backend_operations":0} + for operation, cache in (("open", "cold"), ("blame", "cold"), ("blame", "warm"), ("open", "warm"))] + records += [{"kind":"result", "operation":"blame", "path":list(b"target.txt"), "value":{"ranges":entries}}, + {"kind":"complete", "schema":1, "selection":"blame"}] + return records + + def write_evidence(self, records): + output = self.root / "evidence.jsonl" + output.write_text("".join(json.dumps(record) + "\n" for record in records)) + return output + + def test_every_blame_line_is_checked(self): + repo = self.source("source", 25) + records = self.evidence(repo) + proof = verify(repo, self.write_evidence(records), "HEAD") + self.assertEqual(proof["blame_lines_checked"], 50) + records[-2]["value"]["ranges"][-1]["commit"] = "0" * 40 + with self.assertRaisesRegex(ValueError, "blame line origins differ"): + verify(repo, self.write_evidence(records), "HEAD") + + def test_incomplete_measurements_do_not_pass_the_oracle(self): + repo = self.source("source", 25) + records = self.evidence(repo) + records.pop(2) + with self.assertRaisesRegex(ValueError, "missing cold/warm"): + verify(repo, self.write_evidence(records), "HEAD") + + def test_optimized_python_still_rejects_incorrect_origins(self): + repo = self.source("source", 25) + records = self.evidence(repo) + records[-2]["value"]["ranges"][-1]["commit"] = "0" * 40 + result = subprocess.run([os.sys.executable, "-O", str(Path(__file__).with_name("verify_remote_browse.py")), + str(repo), str(self.write_evidence(records)), "--output", str(self.root / "proof.json")], + capture_output=True) + self.assertNotEqual(result.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/crab/scripts/e2e/test_run_concurrent_push_smoke.py b/crab/scripts/e2e/test_run_concurrent_push_smoke.py index 17b28294a..308972908 100644 --- a/crab/scripts/e2e/test_run_concurrent_push_smoke.py +++ b/crab/scripts/e2e/test_run_concurrent_push_smoke.py @@ -293,6 +293,9 @@ def test_forwards_body_and_records_bounded_request_class(self) -> None: body = response.read() self.assertEqual(body, b"payload") + snapshot = self.proxy.snapshot() + self.assertEqual(snapshot["class_request_body_bytes"], {"git_object_catalog_db/manifest:put": 7}) + self.assertEqual(snapshot["class_response_body_bytes"], {"git_object_catalog_db/manifest:put": 7}) self.assertEqual( self.proxy.snapshot()["classes"], {"git_object_catalog_db/manifest:put": 1}, diff --git a/crab/scripts/e2e/verify_remote_browse.py b/crab/scripts/e2e/verify_remote_browse.py index 8c1ceb779..9c7979c34 100644 --- a/crab/scripts/e2e/verify_remote_browse.py +++ b/crab/scripts/e2e/verify_remote_browse.py @@ -1,12 +1,109 @@ #!/usr/bin/env python3 """Compare qualify_browse JSONL with a read-only source Git repository.""" import argparse +import ast import hashlib import json import subprocess from pathlib import Path +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def check_line_origins(git, head: str, path: bytes, ranges: list[dict]) -> int: + native = [] + source_path = None + for line in git("blame", "--first-parent", "--root", "--line-porcelain", head, "--", path).splitlines(): + fields = line.split() + if not line.startswith(b"\t") and len(fields) >= 3 and len(fields[0]) == 40 and fields[1].isdigit() and fields[2].isdigit(): + origin = fields[0].decode() + final_line = int(fields[2]) + elif line.startswith(b"filename "): + source_path = line[9:] + if source_path.startswith(b'"'): + source_path = ast.literal_eval("b" + source_path.decode("ascii")) + elif line.startswith(b"\t"): + native.append((final_line, origin, source_path)) + actual = [(line, item["commit"], bytes(item["source_path"])) for item in ranges + for line in range(item["start"], item["start"] + item["lines"])] + require(actual == native, "blame line origins differ") + return len(native) + + +def verify_queries(source: Path, records: list[dict], revision: str) -> dict: + """Check isolated qualification queries against native Git, including every blame line.""" + def git(*args) -> bytes: + return subprocess.check_output(["git", "-C", str(source), *args]) + + head = git("rev-parse", revision).decode().strip() + results = [record for record in records if record["kind"] == "result"] + names = [record["operation"] for record in results] + expected_names = {"resolve", "commit", "tree", "blob", "object-point", "object-batch", + "history", "path-history", "attribution", "blame"} + selection = records[-1]["selection"] + require(len(names) == len(set(names)), "duplicate query evidence") + require(set(names) == (expected_names if selection == "all" else {selection}), "missing query evidence") + measurements = [record for record in records if record["kind"] == "measurement"] + for name in names: + pair = [record for record in measurements if record["operation"] == name] + require([record["cache"] for record in pair] == ["cold", "warm"], "missing cold/warm measurement") + require(sum(record["operation"] == "open" for record in measurements) == 2 * len(names), "missing open measurement") + require(all(record["outcome"] == "ok" and record["active_backend_operations"] == 0 + for record in measurements), "failed or unfinished query") + root = [] + for item in git("ls-tree", "-z", head).split(b"\0"): + if item: + metadata, path = item.split(b"\t", 1) + mode, kind, oid = metadata.split() + root.append({"path":list(path), "oid":oid.decode(), "mode":int(mode, 8)}) + + def check_object(value: dict) -> None: + body = git("cat-file", value["kind"], value["oid"]) + header = f"{value['kind']} {len(body)}\0".encode() + require(value["computed_oid"] == value["oid"] == hashlib.sha1(header + body).hexdigest(), "object body hash differs") + require(value["bytes"] == len(body), "object size differs") + + lines_checked = 0 + for record in results: + name, value, path = record["operation"], record["value"], bytes(record["path"]) + if name == "resolve": + require(value["commit"] == head, "resolved commit differs") + elif name == "commit": + require(value["oid"] == head, "commit differs") + require(value["tree"] == git("rev-parse", f"{head}^{{tree}}").decode().strip(), "commit tree differs") + require(value["parents"] == git("rev-list", "--parents", "-n", "1", head).decode().split()[1:], "commit parents differ") + elif name == "tree": + require(value["entries"] == root[:100], "directory page differs") + require(value["has_more"] == (len(root) > 100), "directory continuation differs") + elif name in {"blob", "object-point"}: + require(value["oid"] == git("rev-parse", head.encode() + b":" + path).decode().strip(), "file object differs") + check_object(value) + elif name == "object-batch": + require([item["oid"] for item in value["objects"]] == [item["oid"] for item in root[:100]], "object batch order differs") + for item in value["objects"]: + check_object(item) + elif name in {"history", "path-history"}: + args = ["log", "--first-parent", "--format=%H", "--max-count=1001", head] + if name == "path-history": + args += ["--", path] + expected = git(*args).decode().splitlines() + require(value["commits"] == expected[:1000], "history differs") + require(value["has_more"] == (len(expected) > 1000), "history continuation differs") + elif name == "attribution": + require([(item["path"], item["oid"]) for item in value["entries"]] == + [(item["path"], item["oid"]) for item in root[:100]], "attribution page differs") + for item in value["entries"]: + expected = git("log", "--first-parent", "-1", "--format=%H", head, "--", bytes(item["path"])).decode().strip() + require(item["commit"] == expected, "last-change commit differs") + elif name == "blame": + lines_checked += check_line_origins(git, head, path, value["ranges"]) + return {"status":"passed", "head":head, "queries_checked":len(results), + "blame_lines_checked":lines_checked, "measurements":measurements} + + def verify(source: Path, evidence: Path, revision: str) -> dict: def git(*args: str) -> bytes: return subprocess.check_output(["git", "-C", str(source), *args]) @@ -14,12 +111,14 @@ def git(*args: str) -> bytes: records = [json.loads(line) for line in evidence.read_text().splitlines()] if not records or records[-1]["kind"] != "complete": raise ValueError("reader did not complete") + if records[-1].get("schema") == 1 and "selection" in records[-1]: + return verify_queries(source, records, revision) head = git("rev-parse", revision).decode().strip() snapshot, = [r for r in records if r["kind"] == "snapshot"] - assert snapshot["commit"] == head, "snapshot revision differs" - assert snapshot["tree"] == git("rev-parse", f"{head}^{{tree}}").decode().strip() + require(snapshot["commit"] == head, "snapshot revision differs") + require(snapshot["tree"] == git("rev-parse", f"{head}^{{tree}}").decode().strip(), 'browse evidence differs from Git') refs = [r for r in records if r["kind"] == "ref"] - assert refs == [{"kind": "ref", "name": "refs/heads/main", "oid": head}] + require(refs == [{"kind": "ref", "name": "refs/heads/main", "oid": head}], 'browse evidence differs from Git') expected = {} for line in git("ls-tree", "-r", "-t", "-z", head).split(b"\0"): @@ -30,53 +129,69 @@ def git(*args: str) -> bytes: expected[path] = (int(mode, 8), kind.decode(), oid.decode()) entries = [r for r in records if r["kind"] == "entry"] actual = {bytes(r["path"]): (r["mode"], r["oid"]) for r in entries} - assert len(actual) == len(entries), "duplicate directory entry" - assert actual == {p: (v[0], v[2]) for p, v in expected.items()}, "tree differs" + require(len(actual) == len(entries), "duplicate directory entry") + require(actual == {p: (v[0], v[2]) for p, v in expected.items()}, "tree differs") + + attribution = [r for r in records if r["kind"] == "attribution"] + require(len(attribution) == len(entries), "missing tree attribution") + require({bytes(r["path"]):r["oid"] for r in attribution} == {p:v[2] for p,v in expected.items()}, "attribution entries differ") + for item in attribution: + expected_commit = git("log", "--first-parent", "-1", "--format=%H", head, "--", bytes(item["path"])).decode().strip() + require(item["commit"] == expected_commit, "tree attribution differs") commits = [r for r in records if r["kind"] == "commit"] history = git("rev-list", "--first-parent", "--max-count=1000", head).decode().splitlines() - assert [r["oid"] for r in commits] == history, "paginated history differs" + require([r["oid"] for r in commits] == history, "paginated history differs") for commit in commits: raw = git("cat-file", "commit", commit["oid"]) header, message = raw.split(b"\n\n", 1) fields = [line.split(b" ", 1) for line in header.splitlines() if not line.startswith(b" ")] parents = [v.decode() for k, v in fields if k == b"parent"] - assert parents == commit["parents"], "commit parents differ" + require(parents == commit["parents"], "commit parents differ") values = dict(fields) - assert values[b"tree"].decode() == commit["tree"], "commit tree differs" - assert list(message) == commit["message"], "commit message differs" + require(values[b"tree"].decode() == commit["tree"], "commit tree differs") + require(list(message) == commit["message"], "commit message differs") identity, timestamp, _ = values[b"author"].rsplit(b" ", 2) name, email = identity.rsplit(b" <", 1) - assert list(name) == commit["author_name"], "author differs" - assert list(email[:-1]) == commit["author_email"], "author email differs" - assert int(timestamp) == commit["author_seconds"], "author time differs" - assert int(values[b"committer"].rsplit(b" ", 2)[1]) == commit["committer_seconds"] + require(list(name) == commit["author_name"], "author differs") + require(list(email[:-1]) == commit["author_email"], "author email differs") + require(int(timestamp) == commit["author_seconds"], "author time differs") + require(int(values[b"committer"].rsplit(b" ", 2)[1]) == commit["committer_seconds"], 'browse evidence differs from Git') + + selected_history, = [record for record in records if record["kind"] == "path-history"] + expected_history = git("log", "--first-parent", "--format=%H", "--max-count=1001", head, + "--", bytes(selected_history["path"])).decode().splitlines() + require(selected_history["commits"] == expected_history[:1000], "selected path history differs") + require(selected_history["has_more"] == (len(expected_history) > 1000), "selected history continuation differs") + blame, = [record for record in records if record["kind"] == "blame"] + require(blame["path"] == selected_history["path"], "blame and history paths differ") + blame_lines = check_line_origins(git, head, bytes(blame["path"]), blame["ranges"]) samples = [r for r in records if r["kind"] == "blob"] - assert len(samples) == min(128, sum(v[1] == "blob" for v in expected.values())) + require(len(samples) == min(128, sum(v[1] == "blob" for v in expected.values())), 'browse evidence differs from Git') for blob in samples: oid = expected[bytes(blob["path"])][2] - assert blob["oid"] == blob["computed_oid"] == oid, "sample content differs" - assert len(git("cat-file", "blob", oid)) == blob["bytes"], "sample size differs" + require(blob["oid"] == blob["computed_oid"] == oid, "sample content differs") + require(len(git("cat-file", "blob", oid)) == blob["bytes"], "sample size differs") errors, = [r for r in records if r["kind"] == "errors"] - assert errors == {"kind": "errors", "directory_as_blob": "EntryNotBlob", "cancelled": "Cancelled"} + require(errors == {"kind": "errors", "directory_as_blob": "EntryNotBlob", "cancelled": "Cancelled"}, 'browse evidence differs from Git') archive = [r for r in records if r["kind"] == "archive"] - assert len(archive) == len(expected), "archive count differs" - assert {bytes(r["path"]) for r in archive} == expected.keys(), "archive paths differ" + require(len(archive) == len(expected), "archive count differs") + require({bytes(r["path"]) for r in archive} == expected.keys(), "archive paths differ") blob_count = total_bytes = 0 for entry in archive: mode, kind, oid = expected[bytes(entry["path"])] - assert entry["mode"] == mode, "archive mode differs" + require(entry["mode"] == mode, "archive mode differs") if kind == "blob": - assert entry["computed_oid"] == oid, "archive content differs" + require(entry["computed_oid"] == oid, "archive content differs") blob_count += 1 total_bytes += entry["bytes"] else: - assert entry["computed_oid"] is None and entry["bytes"] is None + require(entry["computed_oid"] is None and entry["bytes"] is None, 'browse evidence differs from Git') return {"status": "passed", "head": head, "commits_checked": len(commits), - "tree_entries_checked": len(entries), "blobs_checked": blob_count, + "tree_entries_checked": len(entries), "blame_lines_checked": blame_lines, "blobs_checked": blob_count, "blob_bytes_checked": total_bytes, "direct_blob_samples": len(samples), "reader": records[-1], "snapshot": snapshot, "error_checks": errors, "evidence_sha256": hashlib.sha256(evidence.read_bytes()).hexdigest()} diff --git a/crates/crab-metadata/src/path_state/codec.rs b/crates/crab-metadata/src/path_state/codec.rs index 1b5e072eb..8ade8e809 100644 --- a/crates/crab-metadata/src/path_state/codec.rs +++ b/crates/crab-metadata/src/path_state/codec.rs @@ -205,6 +205,8 @@ pub(super) fn decode_layer( if cursor != bytes.len() { return corrupt_at(path, "path-state layer has trailing bytes"); } + tracing::debug!(target: "crab_metadata::index_decode", index_format = "path_state", + decoded_bytes = bytes.len() as u64, "index block decoded"); Ok(PathStateLayer { base_ordinal, records, diff --git a/crates/crab-metadata/src/split_commit_graph.rs b/crates/crab-metadata/src/split_commit_graph.rs index 218388004..14ba6d999 100644 --- a/crates/crab-metadata/src/split_commit_graph.rs +++ b/crates/crab-metadata/src/split_commit_graph.rs @@ -759,6 +759,8 @@ pub fn decode_commit_graph_layer(bytes: &[u8], path: &str) -> Result + [operation|all|build-indexes] ``` Use a unique target directory for your checkout on the mounted workspace volume. - -The example reports elapsed time plus canonical `crab-storage` read attempts -and bytes for each operation. Those counters include manifest, inventory, -pack-index, and pack-body reads but exclude SlateDB locator-internal reads, so -they are useful for regression comparison rather than complete provider -billing. The example uses an explicit larger archive qualification budget; it -does not change library or service defaults. +`build-indexes` explicitly runs state-fenced background maintenance; normal +queries only consume published indexes and require attribution readiness. +The JSONL measurements include backend operation outcomes/consumed bytes, +locator-internal reads and graph/path-state decode counts. Backend invocations +are distinct from provider HTTP retries; the scaling harness meters HTTP and +process CPU/peak RSS separately. Oracle records verify Git content hashes and +complete supported blame origins. Archive qualification remains in +`qualify_browse`. See the [delivery and measurement contract](../../crab/docs/architecture/indexed-remote-reads.md). ## Browsing correctness against a real repository `qualify_browse` emits JSONL evidence using only the remote storage API. It -paginates 1,000 first-parent commits and the complete HEAD tree, checks 128 +paginates 1,000 first-parent commits and the complete HEAD tree, checks every directory entry attribution and selected file history/blame, reads 128 spread-out blob paths twice, and streams every HEAD blob through an independent Git SHA-1 calculation. Paths and commit messages are emitted as byte arrays. The run succeeds only when a final `complete` record is emitted. A fresh push can precede object-catalog publication. -Run `crab metadb owner --once` from the uploader repository to advance the -catalog; repeat owner passes until `action=none` to finish all derived maintenance. The direct reader -returns `RepositoryIndexing` while locator coverage is absent or stale and does -not perform this write-side work. See [metadata ownership](../../crab/docs/guides/metadb.md). +Run the explicit `qualify_remote ... build-indexes` operation after publishing +the capsule fixture to construct complete browse indexes. The direct reader +does not perform this write-side work; incomplete attribution remains indexing. See [metadata ownership](../../crab/docs/guides/metadb.md). Run the built example from an empty directory with the local RustFS environment configured as described in the [local development guide](../../crab/docs/guides/local-dev-rustfs.md): ```console -/path/to/qualify_browse > browse.jsonl +/path/to/qualify_browse > browse.jsonl python3 crab/scripts/e2e/verify_remote_browse.py \ /path/to/read-only/source-repository browse.jsonl --output report.json ``` diff --git a/crates/crab-remote-git/examples/qualify_browse.rs b/crates/crab-remote-git/examples/qualify_browse.rs index 7dfeae61c..7f447e217 100644 --- a/crates/crab-remote-git/examples/qualify_browse.rs +++ b/crates/crab-remote-git/examples/qualify_browse.rs @@ -1,38 +1,44 @@ //! Emit remote-only browsing evidence for comparison with an independent Git oracle. -use std::error::Error as StdError; +mod support; use std::io; use std::sync::Arc; use std::time::{Duration, Instant}; use crab_remote_git::{ EntryKind, Error, GitPath, HistoryTraversal, ObjectLimits, OperationKind, OperationLimits, - PageRequest, RemoteGitRepository, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, - Revision, + PageRequest, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, Revision, }; use crab_storage::{StorageProviderKind, StoreLayout, build_static_env_store}; use futures_util::TryStreamExt; use serde_json::json; -use sha1::{Digest, Sha1}; use tokio_util::sync::CancellationToken; -type Result = std::result::Result>; +use support::{Measurements, Result, open_repository}; #[tokio::main] async fn main() -> Result<()> { let args: Vec<_> = std::env::args().skip(1).collect(); - let [bucket, prefix] = args.as_slice() else { - return Err(io::Error::other("usage: qualify_browse ").into()); + let [bucket, prefix, text_path] = args.as_slice() else { + return Err(io::Error::other("usage: qualify_browse ").into()); }; let runtime = Arc::new(RemoteGitRuntime::default()); - let result = qualify(bucket, prefix, Arc::clone(&runtime)).await; + let result = qualify(bucket, prefix, text_path, Arc::clone(&runtime)).await; runtime.shutdown().await; result } -async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> Result<()> { +async fn qualify( + bucket: &str, + prefix: &str, + text_path: &str, + runtime: Arc, +) -> Result<()> { let started = Instant::now(); let cancellation = CancellationToken::new(); - let store = build_static_env_store(bucket, StorageProviderKind::S3)?; + let counters = Arc::new(Measurements::default()); + counters.install()?; + let store = build_static_env_store(bucket, StorageProviderKind::S3)? + .with_storage_observer(counters.clone()); // A complete large snapshot needs a larger aggregate archive budget than // interactive browsing. Each object remains subject to the default bound. let options = RepositoryOptions::new( @@ -46,15 +52,19 @@ async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> ..OperationLimits::default() }, )?; - let repository = RemoteGitRepository::open( - store.clone(), - StoreLayout::new(store, prefix.to_owned()), - RepositoryIdentity::new(format!("s3:{bucket}"), prefix, 1)?, - runtime, - options, - &cancellation, - ) - .await?; + let repository = counters + .measure( + "open", + "cold", + open_repository( + StoreLayout::new(store, prefix.to_owned()), + RepositoryIdentity::new(format!("s3:{bucket}"), prefix, 1)?, + runtime, + options, + &cancellation, + ), + ) + .await?; for reference in &repository.refs().entries { println!( "{}", @@ -80,6 +90,45 @@ async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> "packs":repository.pack_count()}) ); + let selected_path = GitPath::new(bytes::Bytes::copy_from_slice(text_path.as_bytes()))?; + let path_history = counters + .measure("path-history", "shared", async { + let operation = repository + .operation(OperationKind::PathHistory, &cancellation) + .await?; + let result = snapshot + .path_history( + &selected_path, + HistoryTraversal::FirstParent, + &PageRequest::new(1000, None)?, + &operation, + ) + .await; + Ok(operation.finish(result).await?) + }) + .await?; + println!( + "{}", + json!({"kind":"path-history", "path":selected_path.as_bytes(), + "commits":path_history.items.iter().map(|entry| entry.commit.oid.to_string()).collect::>(), + "has_more":path_history.next.is_some()}) + ); + let blame = counters + .measure("blame", "shared", async { + let operation = repository + .operation(OperationKind::Blame, &cancellation) + .await?; + let result = snapshot.blame(&selected_path, &operation).await; + Ok(operation.finish(result).await?) + }) + .await?; + println!( + "{}", + json!({"kind":"blame", "path":selected_path.as_bytes(), + "ranges":blame.ranges.iter().map(|range| json!({"start":range.start,"lines":range.lines, + "commit":range.commit.oid.to_string(),"source_path":range.source_path.as_bytes()})).collect::>()}) + ); + let mut cursor = None; for _ in 0..10 { let operation = repository @@ -118,7 +167,23 @@ async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> let request = PageRequest::new(17, cursor)?; let result = snapshot.list_directory(&path, &request, &operation).await; let page = operation.finish(result).await?; - for entry in page.items { + let commits = counters + .measure("attribution", "shared", async { + let operation = repository + .operation(OperationKind::DirectoryAttribution, &cancellation) + .await?; + let result = snapshot + .latest_directory_entry_commits(&path, &page.items, &operation) + .await; + Ok(operation.finish(result).await?) + }) + .await?; + for (entry, commit) in page.items.into_iter().zip(commits) { + println!( + "{}", + json!({"kind":"attribution", "path":entry.path.as_bytes(), + "oid":entry.oid.to_string(), "commit":commit.oid.to_string()}) + ); println!( "{}", json!({"kind":"entry", "path":entry.path.as_bytes(), @@ -162,7 +227,7 @@ async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> "{}", json!({"kind":"blob", "path":path.as_bytes(), "oid":blob.metadata.oid.to_string(), "bytes":blob.bytes.len(), - "computed_oid":blob_oid(&blob.bytes)}) + "computed_oid":support::object_oid("blob", &blob.bytes)}) ); samples += 1; } @@ -204,7 +269,7 @@ async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> "{}", json!({"kind":"archive", "path":entry.path.as_bytes(), "mode":entry.mode.raw(), "bytes":entry.bytes.as_ref().map(|bytes| bytes.len()), - "computed_oid":entry.bytes.as_ref().map(|bytes| blob_oid(bytes))}) + "computed_oid":entry.bytes.as_ref().map(|bytes| support::object_oid("blob", bytes))}) ); archive_entries += 1; if archive_entries.is_multiple_of(5000) { @@ -219,10 +284,3 @@ async fn qualify(bucket: &str, prefix: &str, runtime: Arc) -> ); Ok(()) } - -fn blob_oid(bytes: &[u8]) -> String { - let mut hash = Sha1::new(); - hash.update(format!("blob {}\0", bytes.len())); - hash.update(bytes); - format!("{:x}", hash.finalize()) -} diff --git a/crates/crab-remote-git/examples/qualify_remote.rs b/crates/crab-remote-git/examples/qualify_remote.rs index 09618aacb..5863e5df8 100644 --- a/crates/crab-remote-git/examples/qualify_remote.rs +++ b/crates/crab-remote-git/examples/qualify_remote.rs @@ -1,352 +1,265 @@ -use std::error::Error as StdError; -use std::future::Future; +//! Measure isolated cold/warm capsule reads and emit independent-oracle evidence. +mod support; + use std::io; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Duration; use bytes::Bytes; use crab_remote_git::{ GitPath, HistoryTraversal, ObjectLimits, OperationKind, OperationLimits, PageRequest, - RemoteGitRepository, RemoteGitRuntime, RepositoryOptions, Revision, + RemoteGitRepository, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, Revision, }; -use crab_storage::{StorageProviderKind, StorageReadKind, StoreLayout, build_static_env_store}; -use futures_util::TryStreamExt as _; +use crab_storage::{StorageProviderKind, StoreLayout, build_static_env_store}; +use serde_json::{Value, json}; +use support::{Measurements, Result, open_repository}; use tokio_util::sync::CancellationToken; -type QualificationResult = Result>; - -#[derive(Default)] -struct ReadCounters { - get: AtomicU64, - get_version: AtomicU64, - stream: AtomicU64, - head: AtomicU64, - range: AtomicU64, - bytes: AtomicU64, -} - -#[derive(Clone, Copy)] -struct ReadSnapshot { - get: u64, - get_version: u64, - stream: u64, - head: u64, - range: u64, - bytes: u64, -} - -impl ReadCounters { - fn record_request(&self, kind: StorageReadKind) { - let counter = match kind { - StorageReadKind::Get => &self.get, - StorageReadKind::GetVersion => &self.get_version, - StorageReadKind::Stream => &self.stream, - StorageReadKind::Head => &self.head, - StorageReadKind::Range => &self.range, - }; - counter.fetch_add(1, Ordering::Relaxed); - } - - fn record_bytes(&self, bytes: u64) { - self.bytes.fetch_add(bytes, Ordering::Relaxed); - } - - fn snapshot(&self) -> ReadSnapshot { - ReadSnapshot { - get: self.get.load(Ordering::Relaxed), - get_version: self.get_version.load(Ordering::Relaxed), - stream: self.stream.load(Ordering::Relaxed), - head: self.head.load(Ordering::Relaxed), - range: self.range.load(Ordering::Relaxed), - bytes: self.bytes.load(Ordering::Relaxed), - } - } -} - -impl ReadSnapshot { - fn since(self, earlier: Self) -> Self { - Self { - get: self.get.saturating_sub(earlier.get), - get_version: self.get_version.saturating_sub(earlier.get_version), - stream: self.stream.saturating_sub(earlier.stream), - head: self.head.saturating_sub(earlier.head), - range: self.range.saturating_sub(earlier.range), - bytes: self.bytes.saturating_sub(earlier.bytes), - } - } - - fn requests(self) -> u64 { - self.get - .saturating_add(self.get_version) - .saturating_add(self.stream) - .saturating_add(self.head) - .saturating_add(self.range) - } -} +const WORKLOADS: &[&str] = &[ + "resolve", + "commit", + "tree", + "blob", + "object-point", + "object-batch", + "history", + "path-history", + "attribution", + "blame", +]; #[tokio::main] -async fn main() -> QualificationResult<()> { - let mut args = std::env::args().skip(1); - let bucket = required_arg(&mut args, "bucket")?; - let repo_prefix = required_arg(&mut args, "repository prefix")?; - let selected_path = GitPath::new(Bytes::from( - required_arg(&mut args, "path changed by HEAD")?.into_bytes(), - ))?; - if args.next().is_some() { - return Err(io::Error::other("unexpected extra argument").into()); +async fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + let (bucket, prefix, path, selection) = + match args.as_slice() { + [bucket, prefix, path] => (bucket, prefix, path, "all"), + [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), + _ => return Err(io::Error::other( + "usage: qualify_remote [operation|all|build-indexes]", + ) + .into()), + }; + if selection != "all" && selection != "build-indexes" && !WORKLOADS.contains(&selection) { + return Err(io::Error::other("unknown qualification operation").into()); } - - let counters = Arc::new(ReadCounters::default()); - let request_counters = Arc::clone(&counters); - let byte_counters = Arc::clone(&counters); - let store = build_static_env_store(&bucket, StorageProviderKind::S3)? - .with_read_request_observer(Arc::new(move |kind| { - request_counters.record_request(kind); - })) - .with_read_byte_observer(Arc::new(move |bytes| { - byte_counters.record_bytes(bytes); - })); - let layout = StoreLayout::new(store.clone(), repo_prefix.clone()); - let identity = - crab_remote_git::RepositoryIdentity::new(format!("s3:{bucket}"), repo_prefix, 1)?; - let runtime = Arc::new(RemoteGitRuntime::default()); - let cancellation = CancellationToken::new(); - let repository_options = RepositoryOptions::new( + let path = GitPath::new(Bytes::copy_from_slice(path.as_bytes()))?; + let options = RepositoryOptions::new( ObjectLimits::default(), OperationLimits { - max_logical_objects: 50_000, - max_storage_requests: 100_000, + max_duration: Duration::from_secs(3600), + max_logical_objects: 200_000, + max_history_commits: 200_000, + max_storage_requests: 500_000, + max_fetched_bytes: 4 * 1024 * 1024 * 1024, + max_inflated_bytes: 4 * 1024 * 1024 * 1024, ..OperationLimits::default() }, )?; - - let repository = measured("open", &counters, async { - RemoteGitRepository::open( - store, - layout, - identity, - Arc::clone(&runtime), - repository_options, - &cancellation, - ) - .await - }) - .await?; - let head = repository - .refs() - .head - .as_ref() - .ok_or_else(|| io::Error::other("repository is empty"))?; - let revision = Revision::Reference(head.name.clone()); - - let snapshot = measured("snapshot", &counters, async { - let operation = repository - .operation(OperationKind::Snapshot, &cancellation) - .await?; - let result = repository.snapshot(&revision, &operation).await; - operation.finish(result).await - }) - .await?; - let commit = measured("commit", &counters, async { - let operation = repository - .operation(OperationKind::Commit, &cancellation) - .await?; - let result = snapshot.commit(&operation).await; - operation.finish(result).await - }) - .await?; - let root = measured("root-cold", &counters, async { - let operation = repository - .operation(OperationKind::Tree, &cancellation) - .await?; - let result = snapshot - .list_directory( - &GitPath::root(), - &PageRequest::new(1_000, None)?, - &operation, - ) - .await; - operation.finish(result).await - }) - .await?; - let warm_root = measured("root-warm", &counters, async { - let operation = repository - .operation(OperationKind::Tree, &cancellation) - .await?; - let result = snapshot - .list_directory( - &GitPath::root(), - &PageRequest::new(1_000, None)?, - &operation, - ) - .await; - operation.finish(result).await - }) - .await?; - let blob = measured("blob", &counters, async { - let operation = repository - .operation(OperationKind::Content, &cancellation) - .await?; - let result = snapshot.read_blob(&selected_path, &operation).await; - operation.finish(result).await - }) - .await?; - let warm_blob = measured("blob-warm", &counters, async { - let operation = repository - .operation(OperationKind::Content, &cancellation) - .await?; - let result = snapshot.read_blob(&selected_path, &operation).await; - operation.finish(result).await - }) - .await?; - let history = measured("history", &counters, async { - let operation = repository - .operation(OperationKind::History, &cancellation) - .await?; - let result = snapshot - .history( - HistoryTraversal::FirstParent, - &PageRequest::new(20, None)?, - &operation, - ) + let counters = Arc::new(Measurements::default()); + counters.install()?; + let store = build_static_env_store(bucket, StorageProviderKind::S3)? + .with_storage_observer(counters.clone()); + let layout = StoreLayout::new(store, prefix.to_owned()); + let identity = RepositoryIdentity::new(format!("s3:{bucket}"), prefix.as_str(), 1)?; + if selection == "build-indexes" { + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let result = counters + .measure("build-indexes", "maintenance", async { + let complete = crab_remote::browse_indexes::ensure( + &layout, + &identity, + runtime.clone(), + options, + 2 * 1024 * 1024 * 1024, + &cancel, + ) + .await? + .ok_or_else(|| { + io::Error::other("index build was superseded or repository was empty") + })?; + if complete.path_state_hash().is_none() { + return Err( + io::Error::other("path attribution was not completely published").into(), + ); + } + Ok(()) + }) .await; - operation.finish(result).await - }) - .await?; - let path_history = measured("path-history", &counters, async { - let operation = repository - .operation(OperationKind::PathHistory, &cancellation) - .await?; - let result = snapshot - .path_history( - &selected_path, - HistoryTraversal::FirstParent, - &PageRequest::new(1, None)?, - &operation, + runtime.shutdown().await; + result?; + } else { + for &name in WORKLOADS + .iter() + .filter(|&&name| selection == "all" || selection == name) + { + let runtime = Arc::new(RemoteGitRuntime::default()); + let result = qualify( + name, + &layout, + &identity, + runtime.clone(), + options, + &path, + &counters, ) .await; - operation.finish(result).await - }) - .await?; - - let parent = commit - .parents - .first() - .copied() - .ok_or_else(|| io::Error::other("HEAD has no parent for comparison"))?; - let base = measured("base-snapshot", &counters, async { - let operation = repository - .operation(OperationKind::Snapshot, &cancellation) - .await?; - let result = repository - .snapshot(&Revision::Commit(parent), &operation) - .await; - operation.finish(result).await - }) - .await?; - let comparison = measured("compare", &counters, async { - let operation = repository - .operation(OperationKind::Compare, &cancellation) - .await?; - let result = snapshot.compare(&base, &operation).await; - operation.finish(result).await - }) - .await?; - let diff = measured("diff", &counters, async { - let operation = repository - .operation(OperationKind::Diff, &cancellation) - .await?; - let result = snapshot.diff(&base, &selected_path, &operation).await; - operation.finish(result).await - }) - .await?; - let blame = measured("blame", &counters, async { - let operation = repository - .operation(OperationKind::Blame, &cancellation) - .await?; - let result = snapshot.blame(&selected_path, &operation).await; - operation.finish(result).await - }) - .await?; - let (archive_entries, archive_bytes) = measured("archive", &counters, async { - let operation = repository - .operation(OperationKind::Archive, &cancellation) - .await?; - let mut stream = snapshot.archive_stream(operation)?; - let mut entries = 0u64; - let mut bytes = 0u64; - while let Some(entry) = stream.try_next().await? { - entries = entries.saturating_add(1); - bytes = bytes.saturating_add(entry.bytes.map_or(0, |value| value.len() as u64)); + runtime.shutdown().await; + result?; } - Ok((entries, bytes)) - }) - .await?; - - let runtime_snapshot = runtime.snapshot().await; - println!("generation={}", repository.generation()); - println!("pack_count={}", repository.pack_count()); - println!("head_ref={}", head.name); - println!("head_oid={}", snapshot.commit_oid()); - println!("parent_oid={parent}"); - println!("root_entries={}", root.items.len()); - println!("warm_root_entries={}", warm_root.items.len()); - println!("blob_oid={}", blob.metadata.oid); - println!("blob_bytes={}", blob.bytes.len()); - println!("warm_blob_bytes={}", warm_blob.bytes.len()); - println!("history_entries={}", history.items.len()); - println!("path_history_entries={}", path_history.items.len()); - println!("comparison_changes={}", comparison.changes.len()); - println!("diff_classification={:?}", diff.classification); - println!("diff_hunks={}", diff.hunks.len()); - println!("blame_ranges={}", blame.ranges.len()); - println!("archive_entries={archive_entries}"); - println!("archive_blob_bytes={archive_bytes}"); - println!("runtime_object_entries={}", runtime_snapshot.object_entries); - println!("runtime_object_bytes={}", runtime_snapshot.object_bytes); + } println!( - "runtime_pack_index_bytes={}", - runtime_snapshot.pack_index_bytes + "{}", + json!({"kind":"complete", "schema":1, "selection":selection}) ); - println!("runtime_parsed_bytes={}", runtime_snapshot.parsed_bytes); - runtime.shutdown().await; Ok(()) } -async fn measured( +async fn qualify( name: &str, - counters: &ReadCounters, - operation: F, -) -> crab_remote_git::Result -where - F: Future>, -{ - let before = counters.snapshot(); - let started = Instant::now(); - let result = operation.await; - let elapsed = started.elapsed(); - let reads = counters.snapshot().since(before); + layout: &StoreLayout, + identity: &RepositoryIdentity, + runtime: Arc, + options: RepositoryOptions, + path: &GitPath, + counters: &Measurements, +) -> Result<()> { + let cancel = CancellationToken::new(); + let repository = counters + .measure( + "open", + "cold", + open_repository( + layout.clone(), + identity.clone(), + runtime.clone(), + options, + &cancel, + ), + ) + .await?; + let head = repository + .refs() + .head + .as_ref() + .ok_or_else(|| io::Error::other("repository has no HEAD"))?; + let revision = Revision::Reference(head.name.clone()); + let cold = counters + .measure( + name, + "cold", + query(name, &repository, &revision, path, &cancel), + ) + .await?; + let warm = counters + .measure( + name, + "warm", + query(name, &repository, &revision, path, &cancel), + ) + .await?; + if cold != warm { + return Err(io::Error::other("cold/warm query results differ").into()); + } println!( - "operation={name} outcome={} elapsed_ms={} store_requests={} get={} get_version={} stream={} head={} range={} store_bytes={}", - if result.is_ok() { "ok" } else { "error" }, - duration_millis(elapsed), - reads.requests(), - reads.get, - reads.get_version, - reads.stream, - reads.head, - reads.range, - reads.bytes, + "{}", + json!({"kind":"result", "operation":name, "path":path.as_bytes(), "value":cold}) ); - result -} - -fn duration_millis(duration: Duration) -> u128 { - duration.as_micros().div_ceil(1_000) + let reopened = counters + .measure( + "open", + "warm", + open_repository(layout.clone(), identity.clone(), runtime, options, &cancel), + ) + .await?; + if repository.refs() != reopened.refs() { + return Err(io::Error::other("refs changed during qualification").into()); + } + Ok(()) } -fn required_arg(args: &mut impl Iterator, name: &'static str) -> io::Result { - args.next() - .ok_or_else(|| io::Error::other(format!("missing {name} argument"))) +async fn query( + name: &str, + repository: &RemoteGitRepository, + revision: &Revision, + path: &GitPath, + cancel: &CancellationToken, +) -> Result { + let kind = match name { + "resolve" => OperationKind::Resolve, + "commit" => OperationKind::Commit, + "tree" => OperationKind::Tree, + "history" => OperationKind::History, + "path-history" => OperationKind::PathHistory, + "attribution" => OperationKind::DirectoryAttribution, + "blame" => OperationKind::Blame, + _ => OperationKind::Content, + }; + let operation = repository.operation(kind, cancel).await?; + // Keep every failure inside the semantic result until the locator closes. + let result = async { + let snapshot = repository.snapshot(revision, &operation).await?; + match name { + "resolve" => Ok(json!({"commit":snapshot.commit_oid().to_string()})), + "commit" => { + let commit = snapshot.commit(&operation).await?; + Ok(json!({"oid":commit.oid.to_string(), "tree":commit.tree.to_string(), + "parents":commit.parents.iter().map(ToString::to_string).collect::>()})) + } + "blob" => { + let blob = snapshot.read_blob(path, &operation).await?; + Ok(json!({"oid":blob.metadata.oid.to_string(), "kind":"blob", "bytes":blob.bytes.len(), + "computed_oid":support::object_oid("blob", &blob.bytes)})) + } + "object-point" => { + let entry = snapshot.entry(path, &operation).await?.ok_or(crab_remote_git::Error::PathNotFound)?; + let object = operation.read_object(entry.oid).await?; + Ok(json!({"oid":object.oid.to_string(), "kind":&object.kind.to_string(), + "bytes":object.data.len(), "computed_oid":support::object_oid(&object.kind.to_string(), &object.data)})) + } + "history" => { + let page = snapshot.history(HistoryTraversal::FirstParent, &PageRequest::new(1_000, None)?, &operation).await?; + Ok(json!({"commits":page.items.iter().map(|commit| commit.oid.to_string()).collect::>(), + "has_more":page.next.is_some()})) + } + "path-history" => { + let page = snapshot.path_history(path, HistoryTraversal::FirstParent, + &PageRequest::new(1_000, None)?, &operation).await?; + Ok(json!({"commits":page.items.iter().map(|entry| entry.commit.oid.to_string()).collect::>(), + "has_more":page.next.is_some()})) + } + "blame" => { + let blame = snapshot.blame(path, &operation).await?; + Ok(json!({"ranges":blame.ranges.iter().map(|range| json!({ + "start":range.start, "lines":range.lines, "commit":range.commit.oid.to_string(), + "source_path":range.source_path.as_bytes(), + })).collect::>()})) + } + _ => { + let page = snapshot.list_directory(&GitPath::root(), &PageRequest::new(100, None)?, &operation).await?; + match name { + "tree" => Ok(json!({"entries":page.items.iter().map(|entry| json!({ + "path":entry.path.as_bytes(), "oid":entry.oid.to_string(), "mode":entry.mode.raw(), + })).collect::>(), "has_more":page.next.is_some()})), + "object-batch" => { + let oids = page.items.iter().map(|entry| entry.oid).collect::>(); + let objects = operation.read_objects(&oids).await?; + Ok(json!({"objects":objects.iter().map(|object| json!({ + "oid":object.oid.to_string(), "kind":&object.kind.to_string(), "bytes":object.data.len(), + "computed_oid":support::object_oid(&object.kind.to_string(), &object.data), + })).collect::>()})) + } + "attribution" => { + let commits = snapshot.latest_directory_entry_commits(&GitPath::root(), &page.items, &operation).await?; + Ok(json!({"entries":page.items.iter().zip(commits).map(|(entry, commit)| json!({ + "path":entry.path.as_bytes(), "oid":entry.oid.to_string(), "commit":commit.oid.to_string(), + })).collect::>()})) + } + _ => Err(crab_remote_git::Error::InternalInvariant { invariant:"unknown qualification operation" }), + } + } + } + }.await; + Ok(operation.finish(result).await?) } diff --git a/crates/crab-remote-git/examples/support/mod.rs b/crates/crab-remote-git/examples/support/mod.rs new file mode 100644 index 000000000..adfd35103 --- /dev/null +++ b/crates/crab-remote-git/examples/support/mod.rs @@ -0,0 +1,282 @@ +//! Shared measurements for qualification binaries, including locator-internal I/O. +use std::error::Error; +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use crab_remote_git::{ + RemoteGitRepository, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, +}; +use crab_storage::{ + StorageObservation, StorageObserver, StorageOperation, StorageOutcome, Store, StoreLayout, +}; +use serde_json::json; +use sha1::{Digest, Sha1}; +use tokio_util::sync::CancellationToken; +use tracing_subscriber::prelude::*; + +pub type Result = std::result::Result>; + +#[derive(Default)] +pub struct Measurements { + requests: [AtomicU64; 11], + outcomes: [AtomicU64; 9], + read_bytes: AtomicU64, + written_bytes: AtomicU64, + active: AtomicU64, + decoded_blocks: [AtomicU64; 2], + decoded_bytes: [AtomicU64; 2], +} + +#[derive(Clone, Copy)] +struct Snapshot { + requests: [u64; 11], + outcomes: [u64; 9], + read_bytes: u64, + written_bytes: u64, + decoded_blocks: [u64; 2], + decoded_bytes: [u64; 2], +} + +impl StorageObserver for Measurements { + fn started(&self, operation: StorageOperation) { + self.requests[operation.index()].fetch_add(1, Ordering::Relaxed); + self.active.fetch_add(1, Ordering::Relaxed); + } + + fn finished(&self, observation: StorageObservation) { + self.outcomes[observation.outcome.index()].fetch_add(1, Ordering::Relaxed); + self.read_bytes + .fetch_add(observation.bytes_read, Ordering::Relaxed); + self.written_bytes + .fetch_add(observation.bytes_written, Ordering::Relaxed); + self.active.fetch_sub(1, Ordering::Relaxed); + } +} + +struct DecodeLayer(Arc); + +#[derive(Default)] +struct DecodeEvent { + format: Option, + bytes: u64, +} + +impl tracing::field::Visit for DecodeEvent { + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + if field.name() == "index_format" { + self.format = match value { + "commit_graph" => Some(0), + "path_state" => Some(1), + _ => None, + }; + } + } + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + if field.name() == "decoded_bytes" { + self.bytes = value; + } + } +} + +impl tracing_subscriber::Layer for DecodeLayer { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + if event.metadata().target() != "crab_metadata::index_decode" { + return; + } + let mut decoded = DecodeEvent::default(); + event.record(&mut decoded); + if let Some(format) = decoded.format { + self.0.decoded_blocks[format].fetch_add(1, Ordering::Relaxed); + self.0.decoded_bytes[format].fetch_add(decoded.bytes, Ordering::Relaxed); + } + } +} + +impl Measurements { + pub fn install(self: &Arc) -> Result<()> { + tracing_subscriber::registry() + .with(DecodeLayer(self.clone())) + .try_init()?; + Ok(()) + } + fn snapshot(&self) -> Snapshot { + Snapshot { + requests: self + .requests + .each_ref() + .map(|counter| counter.load(Ordering::Relaxed)), + outcomes: self + .outcomes + .each_ref() + .map(|counter| counter.load(Ordering::Relaxed)), + read_bytes: self.read_bytes.load(Ordering::Relaxed), + written_bytes: self.written_bytes.load(Ordering::Relaxed), + decoded_blocks: self + .decoded_blocks + .each_ref() + .map(|counter| counter.load(Ordering::Relaxed)), + decoded_bytes: self + .decoded_bytes + .each_ref() + .map(|counter| counter.load(Ordering::Relaxed)), + } + } + + pub async fn measure( + &self, + name: &str, + cache: &str, + future: impl Future>, + ) -> Result { + let before = self.snapshot(); + let started = Instant::now(); + let result = future.await; + let after = self.snapshot(); + let operations = StorageOperation::ALL + .into_iter() + .map(|operation| { + ( + operation.label().to_owned(), + json!( + after.requests[operation.index()] + .saturating_sub(before.requests[operation.index()]) + ), + ) + }) + .collect::>(); + let outcomes = StorageOutcome::ALL + .into_iter() + .map(|outcome| { + ( + outcome.label().to_owned(), + json!( + after.outcomes[outcome.index()] + .saturating_sub(before.outcomes[outcome.index()]) + ), + ) + }) + .collect::>(); + let decoded = ["commit_graph", "path_state"] + .into_iter() + .enumerate() + .map(|(index, name)| { + let blocks = + after.decoded_blocks[index].saturating_sub(before.decoded_blocks[index]); + let bytes = after.decoded_bytes[index].saturating_sub(before.decoded_bytes[index]); + (name.to_owned(), json!({"blocks":blocks, "bytes":bytes})) + }) + .collect::>(); + println!( + "{}", + json!({ + "kind": "measurement", "schema": 1, "operation": name, "cache": cache, + "outcome": if result.is_ok() { "ok" } else { "error" }, + "elapsed_us": started.elapsed().as_micros(), "backend_operations": operations, + "backend_outcomes": outcomes, + "backend_read_bytes": after.read_bytes.saturating_sub(before.read_bytes), + "backend_written_bytes": after.written_bytes.saturating_sub(before.written_bytes), + "active_backend_operations": self.active.load(Ordering::Relaxed), + "decoded_index_blocks": decoded, + }) + ); + result + } +} + +// Qualification uses the same capsule opening path as the repository service. +// A legacy manifest open would measure different visibility and index behavior. +pub async fn open_repository( + layout: StoreLayout, + identity: RepositoryIdentity, + runtime: Arc, + options: RepositoryOptions, + cancel: &CancellationToken, +) -> Result { + let root = crab_metadata::capsule_protocol::load_root(&layout).await?; + let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&layout).await?; + let view = crab_read::capsule_protocol::open_view_from_root_with_control( + &layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 2 * 1024 * 1024 * 1024, + max_frontier_bytes: 2 * 1024 * 1024 * 1024, + }, + ) + .await? + .with_browse_indexes(indexes); + Ok(view + .git_repository_from_store( + layout, + identity, + runtime, + options, + 2 * 1024 * 1024 * 1024, + cancel, + ) + .await?) +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use object_store::ObjectStoreExt; + use object_store::memory::InMemory; + use object_store::path::Path; + + #[test] + fn decoder_counts_ignore_unrelated_events() { + let counters = Arc::new(Measurements::default()); + let subscriber = tracing_subscriber::registry().with(DecodeLayer(counters.clone())); + tracing::subscriber::with_default(subscriber, || { + tracing::debug!(target:"crab_metadata::index_decode", index_format="path_state", decoded_bytes=123_u64); + tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); + }); + let snapshot = counters.snapshot(); + assert_eq!( + (snapshot.decoded_blocks, snapshot.decoded_bytes), + ([0, 1], [0, 123]) + ); + } + + #[tokio::test] + async fn observation_includes_reads_through_inner_store_and_missing_objects() { + let counters = Arc::new(Measurements::default()); + let store = Store::new(Arc::new(InMemory::new())).with_storage_observer(counters.clone()); + store + .put(&Path::from("body"), Bytes::from_static(b"payload")) + .await + .unwrap(); + let before = counters.snapshot(); + // SlateDB uses inner(), bypassing canonical Store read callbacks. + let body = store + .inner() + .get(&Path::from("body")) + .await + .unwrap() + .bytes() + .await + .unwrap(); + assert_eq!(body.len(), 7); + assert!(store.inner().get(&Path::from("missing")).await.is_err()); + let after = counters.snapshot(); + assert_eq!( + after.requests[StorageOperation::Get.index()] + - before.requests[StorageOperation::Get.index()], + 2 + ); + assert_eq!(after.read_bytes - before.read_bytes, 7); + assert_eq!(after.outcomes[StorageOutcome::NotFound.index()], 1); + assert_eq!(counters.active.load(Ordering::Relaxed), 0); + } +} + +pub fn object_oid(kind: &str, data: &[u8]) -> String { + let mut hash = Sha1::new(); + hash.update(format!("{kind} {}\0", data.len())); + hash.update(data); + format!("{:x}", hash.finalize()) +} From 46214b21c4654d2cee407dad9c71c57ecdb0e1be Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 09:23:28 -0700 Subject: [PATCH 002/127] fix(architecture): own read qualification in orchestration --- Cargo.lock | 4 ++-- crab/docs/architecture/indexed-remote-reads.md | 2 +- crab/scripts/e2e/run_remote_read_scaling_rustfs.py | 6 +++--- crates/crab-remote-git/Cargo.toml | 2 -- crates/crab-remote-git/REFERENCE.md | 2 +- crates/crab-remote/Cargo.toml | 10 ++++++++++ crates/crab-remote/README.md | 9 +++++++++ .../examples/qualify_browse.rs | 0 .../examples/qualify_remote.rs | 0 .../examples/support/mod.rs | 0 10 files changed, 26 insertions(+), 9 deletions(-) rename crates/{crab-remote-git => crab-remote}/examples/qualify_browse.rs (100%) rename crates/{crab-remote-git => crab-remote}/examples/qualify_remote.rs (100%) rename crates/{crab-remote-git => crab-remote}/examples/support/mod.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index ff8852f4a..ad69dadea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2538,12 +2538,14 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", + "sha1 0.10.6", "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", "toml", "tracing", + "tracing-subscriber", ] [[package]] @@ -2555,8 +2557,6 @@ dependencies = [ "bytes", "crab-git", "crab-metadata", - "crab-read", - "crab-remote", "crab-storage", "crab-xet", "flate2", diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e1bde7f1b..eeb95b5b0 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -83,7 +83,7 @@ all compiling invocations. No local `target/` or external-source mutation. ```sh CARGO_TARGET_DIR="$HOME/Workspace/crabbuild-target/crab-ios-abf8fc00-push-audit" \ - cargo build -p crab-remote-git --release --examples --locked + cargo build -p crab-remote --features publication --release --examples --locked python3 crab/scripts/e2e/run_remote_read_scaling_rustfs.py \ --root "$HOME/Workspace/CrabData/indexed-read-abf8fc00" \ diff --git a/crab/scripts/e2e/run_remote_read_scaling_rustfs.py b/crab/scripts/e2e/run_remote_read_scaling_rustfs.py index d7efcd793..5a18f23df 100644 --- a/crab/scripts/e2e/run_remote_read_scaling_rustfs.py +++ b/crab/scripts/e2e/run_remote_read_scaling_rustfs.py @@ -77,9 +77,9 @@ def qualify(self) -> None: source_root = self.artifacts / "qualified-source" source_root.mkdir() inputs = ( - "Cargo.lock", "crates/crab-remote-git/Cargo.toml", - "crates/crab-remote-git/examples/qualify_remote.rs", - "crates/crab-remote-git/examples/support/mod.rs", + "Cargo.lock", "crates/crab-remote/Cargo.toml", + "crates/crab-remote/examples/qualify_remote.rs", + "crates/crab-remote/examples/support/mod.rs", "crates/crab-metadata/src/path_state/codec.rs", "crates/crab-metadata/src/split_commit_graph.rs", "crab/scripts/e2e/run_remote_read_scaling_rustfs.py", diff --git a/crates/crab-remote-git/Cargo.toml b/crates/crab-remote-git/Cargo.toml index f1a2d6668..051f586c8 100644 --- a/crates/crab-remote-git/Cargo.toml +++ b/crates/crab-remote-git/Cargo.toml @@ -37,8 +37,6 @@ tempfile = { workspace = true } tracing = { workspace = true } [dev-dependencies] -crab-read = { workspace = true } -crab-remote = { workspace = true, features = ["publication"] } crab-metadata = { workspace = true, features = ["remote-index", "storage"] } object_store = { workspace = true } proptest = "1" diff --git a/crates/crab-remote-git/REFERENCE.md b/crates/crab-remote-git/REFERENCE.md index cc778f525..901ee2500 100644 --- a/crates/crab-remote-git/REFERENCE.md +++ b/crates/crab-remote-git/REFERENCE.md @@ -296,7 +296,7 @@ workload gets a fresh runtime and runs cold/warm with opening reported separatel ```console CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/crab-remote-qualification \ - cargo run -p crab-remote-git --release --example qualify_remote -- \ + cargo run -p crab-remote --features publication --release --example qualify_remote -- \ [operation|all|build-indexes] ``` diff --git a/crates/crab-remote/Cargo.toml b/crates/crab-remote/Cargo.toml index fa82339fc..67659fc1a 100644 --- a/crates/crab-remote/Cargo.toml +++ b/crates/crab-remote/Cargo.toml @@ -41,9 +41,19 @@ futures-util = { workspace = true, optional = true } object_store = { workspace = true, optional = true } [dev-dependencies] +sha1 = "0.10" +tracing-subscriber = "0.3" async-trait = { workspace = true } crab-cache = { workspace = true, features = ["local-cache"] } crab-cache-store = { workspace = true } object_store = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[[example]] +name = "qualify_remote" +required-features = ["publication"] + +[[example]] +name = "qualify_browse" +required-features = ["publication"] diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index f2ceb65b0..35c663c26 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -69,3 +69,12 @@ Every publication callback retains its terminal outcome while leases and heartbeats drain. Missing historical proof stays indeterminate; current ref equality never invents attribution. Protected receive remains server-authorized and cannot be invoked as a client-authorized commit. + +## Read qualification + +The `publication` feature exposes `qualify_remote` and `qualify_browse` examples. +They compose capsule capture, read APIs and explicit browse-index maintenance at +this orchestration boundary. The low-level remote Git reader retains its narrow +Git/storage/metadata dependency boundary. See the +[indexed-read qualification contract](../../crab/docs/architecture/indexed-remote-reads.md) +for commands, native Git oracles and cold/warm measurement semantics. diff --git a/crates/crab-remote-git/examples/qualify_browse.rs b/crates/crab-remote/examples/qualify_browse.rs similarity index 100% rename from crates/crab-remote-git/examples/qualify_browse.rs rename to crates/crab-remote/examples/qualify_browse.rs diff --git a/crates/crab-remote-git/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs similarity index 100% rename from crates/crab-remote-git/examples/qualify_remote.rs rename to crates/crab-remote/examples/qualify_remote.rs diff --git a/crates/crab-remote-git/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs similarity index 100% rename from crates/crab-remote-git/examples/support/mod.rs rename to crates/crab-remote/examples/support/mod.rs From 2c7d30fb56d32b41b6732eb07e852810dc408783 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 10:41:42 -0700 Subject: [PATCH 003/127] feat(metadata): page path attribution indexes Replace unreleased whole-layer path-state data with authenticated pages. Keep layer object ownership for checkpoints, publication and GC; rebuild retired derived formats under the existing maintenance owner. --- .../docs/architecture/indexed-remote-reads.md | 52 +- crab/src/cmd/gc/mod.rs | 26 + crates/crab-metadata/README.md | 9 + crates/crab-metadata/src/path_state.rs | 53 +- crates/crab-metadata/src/path_state/codec.rs | 584 +++++++++-------- crates/crab-metadata/src/path_state/paged.rs | 602 ++++++++++++++++++ crates/crab-metadata/src/path_state/reader.rs | 511 +++++++++++++++ .../src/path_state/reader_tests.rs | 239 +++++++ .../crab-metadata/src/path_state/storage.rs | 8 +- 9 files changed, 1767 insertions(+), 317 deletions(-) create mode 100644 crates/crab-metadata/src/path_state/paged.rs create mode 100644 crates/crab-metadata/src/path_state/reader.rs create mode 100644 crates/crab-metadata/src/path_state/reader_tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index eeb95b5b0..8a284a53a 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -27,7 +27,7 @@ Xet files and unified local staging are subsequent workstreams. | Order | Deliverable | Completion evidence | Current state | | --- | --- | --- | --- | | 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | -| 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Pending | +| 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Implemented; 23 focused tests, GC retention regression and 1,000/10,000-commit RustFS format qualification passed | | 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Pending | | 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Pending | | 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | @@ -54,9 +54,10 @@ The backend observer includes reads through `Store::inner()`, including locator SST/manifest access. It reports logical backend invocations and consumed bytes; provider HTTP retries and listing pages require the separate HTTP meter. Decoder observations count verified graph/path-state blocks and their encoded input bytes. -Current blocks are layers, not individual pages. Do not relabel those bytes as -network transfer or resident memory. New paged formats must add their decode events -at the actual decoding boundary. +The frozen baseline counted layers. The paged path-state format counts individual +verified pages; the graph still counts layers. Do not relabel decoded bytes as +network transfer or resident memory. Decode events belong at the actual decoding +boundary. The HTTP meter records requests and request/response bytes by bounded object class. Capsule/run/checkpoint objects can contain both control and Git payload @@ -195,3 +196,46 @@ Those files are unchanged by this work. A diagnostic run allowing only `unfulfilled-lint-expectations` passes; this is not a strict Clippy pass and no source suppression was added. Record the baseline issue separately from new format/read correctness gates. + +## Paged path-state format + +Path-state descriptor version 5 replaces the unreleased whole-layer predecessor. +Each layer remains one immutable content-addressed object; its descriptor commits +to record, node and component-name page roots. Pages use a canonical binary +format with typed keys, authenticated byte ranges, exact key bounds and subtree +counts. Internal directories are paged as well. Typical pages target 32 KiB; +a single admitted long component or summary may use a page up to the hard 2 MiB +bound. Directory keys remain fixed-width regardless of component length. + +Commit records and nodes use dense ordinal keys. Wide child collections use +sorted component ordinals; a paged hash-to-component directory preserves exact +raw names. Repeated names are encoded once per layer and shared by decoded nodes. +Writer page deduplication preserves repeated immutable child collections. Unchanged +trie nodes still refer to retained layers. Existing incremental construction and +compaction retain their graph/ordinal binding and verified checkpoint behavior. + +Full maintenance decoding verifies the complete layer content hash, every page, +component use and contiguous physical byte coverage. It rejects overlapping, +conflicting or unreferenced page regions. Lazy reads verify their authenticated +page chain and selected records against the captured graph. They do not claim +full validation of unrelated payload entries; maintenance/fsck retain that role. +A missing required page or malformed routing is an error, never a deduplication +or attribution miss. Root pages are verified before accepting out-of-range keys. + +`PathStateReader` loads the bounded descriptor, groups node/record/name/child +queries, and reads only dependency pages through the caller's admitted clone of +the same origin store. It caches verified parsed pages with an 8 MiB accounted-data +bound and at most 256 cached pages; cache identity includes layer, offset, kind and complete routing metadata. +Each query admits unique encoded page bytes before consulting cache or launching +bounded concurrent I/O. Cache locks do not cover network waits. The caller keeps +its captured graph and authorization boundary; index hits add no ref visibility. +Graph layer identities also remain pinned: a rebuilt graph with matching scalar +generation/count/digest fields cannot substitute different positional records. + +Pages are ranges within the layer object, so existing GC marking of every retained +layer object also retains all referenced pages, including resumable construction +checkpoints. Publication verifies the complete immutable object before publishing +its descriptor/readiness pointer. No separate page-key scan, new authority object, +or old-format reader is introduced. Before GC or readiness accepts an index, its +version and descriptor roots must validate; retired derived indexes are rebuilt +under existing ownership and CAS rules. diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index 421c84f77..eea3c325d 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -5787,6 +5787,15 @@ mod tests { let current_graph_layer = layout.repo_path(¤t_graph.descriptor.layers[0].path); let current_path_state_descriptor = layout.bulk_manifest_path("path-state", ¤t_path_state_hash); + let current_path_state = crab_metadata::path_state::load_path_state_descriptor( + store.as_storage(), + &layout, + ¤t_path_state_hash, + crab_metadata::path_state::DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let current_path_state_layer = layout.repo_path(¤t_path_state.layers[0].path); let current_checkpoint = layout.repo_path( &crab_metadata::path_state::path_state_checkpoint_path(&manifest.git_validation_digest), ); @@ -5810,6 +5819,7 @@ mod tests { current_graph_descriptor.clone(), current_graph_layer.clone(), current_path_state_descriptor.clone(), + current_path_state_layer.clone(), current_checkpoint.clone(), ]; let stale_indexes = [ @@ -5864,6 +5874,22 @@ mod tests { for path in current_indexes { assert!(store.head(&path).await.is_ok()); } + // Authenticated pages live inside the layer marked by both readiness + // and construction roots; sweeping other generations must retain them. + let reader = crab_metadata::path_state::PathStateReader::load( + store.as_storage(), + &layout, + ¤t_path_state_hash, + ¤t_graph, + crab_metadata::path_state::DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let attribution = reader + .latest(store.as_storage(), ¤t_graph, &[3; 20], &[b"file"]) + .await + .unwrap(); + assert_eq!(attribution[0].oid, [3; 20]); for path in stale_indexes { let head = store.head(&path).await; assert!( diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index b84314a59..4e8df01b5 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -327,3 +327,12 @@ manifest or opens SlateDB. The root cardinality is checked before segment fetches, and the normal verified segment reader is reused. Clones share lookup state until explicit close; scope and content retention remain the caller's responsibility. + +### Paged path attribution + +Unreleased path-state descriptor version 5 authenticates bounded record, node, +component and child pages inside each immutable layer object. Canonical full +loads validate the complete physical closure; `PathStateReader` resolves selected +paths through verified ranges and a bounded parsed-page cache. Graph and ordinal +bindings are retained. GC retains the layer object and therefore every embedded +page. See the [format and qualification contract](../../crab/docs/architecture/indexed-remote-reads.md#paged-path-state-format). diff --git a/crates/crab-metadata/src/path_state.rs b/crates/crab-metadata/src/path_state.rs index 5f29517a4..fb0f10a47 100644 --- a/crates/crab-metadata/src/path_state.rs +++ b/crates/crab-metadata/src/path_state.rs @@ -11,6 +11,12 @@ use crate::{ }; mod codec; +mod paged; +#[cfg(feature = "storage")] +mod reader; +pub use paged::{PathStateLayerPages, PathStatePageRef}; +#[cfg(feature = "storage")] +pub use reader::PathStateReader; #[cfg(feature = "storage")] mod storage; @@ -23,12 +29,10 @@ pub use storage::{ load_path_state_descriptor, publish_path_state_checkpoint, upload_path_state, }; -const LAYER_MAGIC: &[u8; 8] = b"CRABPS03"; -const LAYER_VERSION: u32 = 3; -// Earlier descriptors can authenticate false mutations from Git-order merges. -// Retire those semantics; maintenance rebuilds them without changing the codec. -const DESCRIPTOR_VERSION: u32 = 4; -const LAYER_HEADER_BYTES: usize = 28; +// Whole-layer predecessors are unreleased derived data. Rebuild them before +// publishing paged readiness, retaining the exact tree-comparison semantics. +const DESCRIPTOR_VERSION: u32 = 5; +const MAX_DESCRIPTOR_BYTES: u64 = 128 * 1024; const MAX_AUTHOR_BYTES: usize = 16 * 1024; const MAX_MESSAGE_BYTES: usize = 1024 * 1024; const MAX_PATH_BYTES: usize = 1024 * 1024; @@ -113,6 +117,7 @@ pub struct PathStateLayerRef { pub commit_count: u32, pub node_count: u32, pub bytes: u64, + pub pages: PathStateLayerPages, } /// Immutable generation-bound path-state descriptor. @@ -415,7 +420,8 @@ pub fn append_path_state( records, nodes, }; - let bytes = encode_layer(&layer)?; + let encoded = encode_layer(&layer)?; + let bytes = encoded.bytes; let hash = blake3::hash(&bytes).to_hex().to_string(); let reference = PathStateLayerRef { path: path_state_layer_path(&hash), @@ -424,6 +430,7 @@ pub fn append_path_state( commit_count: layer.records.len() as u32, node_count: layer.nodes.len() as u32, bytes: bytes.len() as u64, + pages: encoded.pages, }; references.push(reference.clone()); objects.push(PathStateLayerObject { reference, bytes }); @@ -431,7 +438,8 @@ pub fn append_path_state( } if layers.len() > MAX_LAYERS_BEFORE_COMPACTION { let layer = compact_layers(&layers)?; - let bytes = encode_layer(&layer)?; + let encoded = encode_layer(&layer)?; + let bytes = encoded.bytes; let hash = blake3::hash(&bytes).to_hex().to_string(); let reference = PathStateLayerRef { path: path_state_layer_path(&hash), @@ -440,6 +448,7 @@ pub fn append_path_state( commit_count: layer.records.len() as u32, node_count: layer.nodes.len() as u32, bytes: bytes.len() as u64, + pages: encoded.pages, }; references = vec![reference.clone()]; objects = vec![PathStateLayerObject { reference, bytes }]; @@ -465,6 +474,9 @@ pub fn append_path_state( let descriptor_bytes = serde_json::to_vec(&descriptor).map_err(|source| { MetadataError::Internal(format!("path-state descriptor encode: {source}")) })?; + if descriptor_bytes.len() as u64 > MAX_DESCRIPTOR_BYTES { + return corrupt("path-state descriptor exceeds its byte bound"); + } Ok(PathStateWrite { descriptor_hash: blake3::hash(&descriptor_bytes).to_hex().to_string(), descriptor_bytes, @@ -791,7 +803,9 @@ fn validate_node_ref( } fn validate_descriptor_shape(descriptor: &PathStateDescriptor) -> Result<()> { - if descriptor.version != DESCRIPTOR_VERSION { + if descriptor.version != DESCRIPTOR_VERSION + || descriptor.layers.len() > MAX_LAYERS_BEFORE_COMPACTION + { return corrupt("unsupported path-state descriptor version"); } validate_content_hash( @@ -820,10 +834,27 @@ fn validate_descriptor_shape(descriptor: &PathStateDescriptor) -> Result<()> { || reference.base_ordinal != next || reference.commit_count == 0 || reference.node_count == 0 - || reference.bytes < LAYER_HEADER_BYTES as u64 + || reference.pages.records.count != reference.commit_count + || reference.pages.nodes.count != reference.node_count { return corrupt("invalid path-state layer reference"); } + reference + .pages + .records + .validate(paged::Kind::Records, reference.bytes)?; + reference + .pages + .nodes + .validate(paged::Kind::Nodes, reference.bytes)?; + if reference.pages.records.first != 0u32.to_be_bytes() + || reference.pages.nodes.first != 0u32.to_be_bytes() + { + return corrupt("positional layer roots do not start at zero"); + } + if let Some(names) = &reference.pages.names { + names.validate(paged::Kind::Names, reference.bytes)?; + } next = next .checked_add(reference.commit_count) .ok_or_else(|| corruption("path-state commit count overflows"))?; @@ -1067,7 +1098,7 @@ mod tests { graph_with_commits(2) } - fn graph_with_commits(count: u8) -> SplitCommitGraph { + pub(super) fn graph_with_commits(count: u8) -> SplitCommitGraph { let layer = CommitGraphLayer { base_ordinal: 0, records: (1..=count) diff --git a/crates/crab-metadata/src/path_state/codec.rs b/crates/crab-metadata/src/path_state/codec.rs index 8ade8e809..39c837b6d 100644 --- a/crates/crab-metadata/src/path_state/codec.rs +++ b/crates/crab-metadata/src/path_state/codec.rs @@ -3,267 +3,303 @@ use std::collections::BTreeMap; use std::sync::Arc; #[cfg(any(feature = "storage", test))] +use super::paged::Input; +use super::paged::{self, Kind, PathStateLayerPages, Writer}; use super::{ - LAYER_HEADER_BYTES, MAX_AUTHOR_BYTES, MAX_CHILDREN_PER_NODE, MAX_MESSAGE_BYTES, MAX_PATH_BYTES, - PathStateLayerRef, PathStateNode, PathStateNodeRef, PathStateRecord, corrupt_at, corruption, + MAX_AUTHOR_BYTES, MAX_MESSAGE_BYTES, MAX_PATH_BYTES, PathStateLayer, Result, corruption, }; -use super::{LAYER_MAGIC, LAYER_VERSION, PathStateLayer, Result}; #[cfg(any(feature = "storage", test))] -use crate::error::MetadataError; +use super::{PathStateLayerRef, PathStateNode, PathStateNodeRef, PathStateRecord, corrupt_at}; -pub(super) fn encode_layer(layer: &PathStateLayer) -> Result> { +pub(super) struct EncodedLayer { + pub(super) bytes: Vec, + pub(super) pages: PathStateLayerPages, +} + +pub(super) fn encode_layer(layer: &PathStateLayer) -> Result { let mut names = BTreeMap::<&[u8], u32>::new(); for node in &layer.nodes { for name in node.children.keys() { names.entry(name.as_ref()).or_default(); } } - let name_count = u32::try_from(names.len()) - .map_err(|_| super::corruption("path-state component count overflows"))?; - for (ordinal, value) in names.values_mut().enumerate() { - *value = u32::try_from(ordinal) - .map_err(|_| super::corruption("path-state component ordinal overflows"))?; - } - let mut bytes = Vec::new(); - bytes.extend_from_slice(LAYER_MAGIC); - bytes.extend_from_slice(&LAYER_VERSION.to_le_bytes()); - bytes.extend_from_slice(&layer.base_ordinal.to_le_bytes()); - bytes.extend_from_slice(&(layer.records.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&(layer.nodes.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&name_count.to_le_bytes()); - for name in names.keys() { - bytes.extend_from_slice(&(name.len() as u32).to_le_bytes()); - bytes.extend_from_slice(name); - } - for record in &layer.records { - bytes.extend_from_slice(&record.oid); - bytes.extend_from_slice(&record.first_parent.unwrap_or(u32::MAX).to_le_bytes()); - bytes.extend_from_slice(&record.author_seconds.to_le_bytes()); - bytes.extend_from_slice(&(record.author.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&(record.message.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&record.root.layer.to_le_bytes()); - bytes.extend_from_slice(&record.root.index.to_le_bytes()); - bytes.extend_from_slice(&record.author); - bytes.extend_from_slice(&record.message); + let mut name_rows = BTreeMap::new(); + for (index, (name, ordinal)) in names.iter_mut().enumerate() { + validate_name(name)?; + *ordinal = u32::try_from(index).map_err(|_| corruption("component ordinal overflows"))?; + let mut value = ordinal.to_le_bytes().to_vec(); + value.extend_from_slice(name); + if name_rows + .insert(blake3::hash(name).as_bytes().to_vec(), value) + .is_some() + { + return Err(corruption("component identities collide")); + } } - for node in &layer.nodes { - bytes.extend_from_slice(&node.value.unwrap_or(u32::MAX).to_le_bytes()); - bytes.extend_from_slice(&(node.children.len() as u32).to_le_bytes()); - for (name, child) in &node.children { - let ordinal = names - .get(name.as_ref()) - .ok_or_else(|| super::corruption("path-state component is missing"))?; - bytes.extend_from_slice(&ordinal.to_le_bytes()); - bytes.extend_from_slice(&child.layer.to_le_bytes()); - bytes.extend_from_slice(&child.index.to_le_bytes()); + let mut writer = Writer::default(); + let names_root = if name_rows.is_empty() { + None + } else { + Some(writer.tree(Kind::Names, name_rows.into_iter().collect())?) + }; + let mut nodes = Vec::new(); + for (index, node) in layer.nodes.iter().enumerate() { + let rows = node + .children + .iter() + .map(|(name, child)| { + let ordinal = names + .get(name.as_ref()) + .ok_or_else(|| corruption("missing component ordinal"))?; + let mut value = child.layer.to_le_bytes().to_vec(); + value.extend_from_slice(&child.index.to_le_bytes()); + Ok((ordinal.to_be_bytes().to_vec(), value)) + }) + .collect::>>()?; + let mut value = node.value.unwrap_or(u32::MAX).to_le_bytes().to_vec(); + if rows.is_empty() { + value.push(0); + } else { + value.push(1); + let root = writer.tree(Kind::Children, rows)?; + paged::encode_ref(&mut value, &root); } + let key = u32::try_from(index) + .map_err(|_| corruption("node ordinal overflows"))? + .to_be_bytes() + .to_vec(); + nodes.push((key, value)); + } + let nodes_root = writer.tree(Kind::Nodes, nodes)?; + let records = layer + .records + .iter() + .enumerate() + .map(|(index, record)| { + if record.author.len() > MAX_AUTHOR_BYTES || record.message.len() > MAX_MESSAGE_BYTES { + return Err(corruption("commit summary exceeds its limits")); + } + let mut value = record.oid.to_vec(); + value.extend_from_slice(&record.first_parent.unwrap_or(u32::MAX).to_le_bytes()); + value.extend_from_slice(&record.author_seconds.to_le_bytes()); + paged::put_u32(&mut value, record.author.len())?; + paged::put_u32(&mut value, record.message.len())?; + value.extend_from_slice(&record.root.layer.to_le_bytes()); + value.extend_from_slice(&record.root.index.to_le_bytes()); + value.extend_from_slice(&record.author); + value.extend_from_slice(&record.message); + let key = u32::try_from(index) + .map_err(|_| corruption("record ordinal overflows"))? + .to_be_bytes() + .to_vec(); + Ok((key, value)) + }) + .collect::>>()?; + let records_root = writer.tree(Kind::Records, records)?; + Ok(EncodedLayer { + bytes: writer.bytes, + pages: PathStateLayerPages { + records: records_root, + nodes: nodes_root, + names: names_root, + }, + }) +} + +fn validate_name(name: &[u8]) -> Result<()> { + if name.is_empty() || name.len() > MAX_PATH_BYTES || name.contains(&0) || name.contains(&b'/') { + return Err(corruption("invalid path-state component")); } - Ok(bytes) + Ok(()) } -// Only storage loads encoded layers; payload builders keep their validated index. -// Unit tests still exercise the decoder without enabling storage dependencies. #[cfg(any(feature = "storage", test))] pub(super) fn decode_layer( bytes: &[u8], reference: &PathStateLayerRef, path: &str, ) -> Result { - const RECORD_FIXED_BYTES: usize = 48; - const NODE_FIXED_BYTES: usize = 8; - const CHILD_FIXED_BYTES: usize = 12; - - if bytes.len() < LAYER_HEADER_BYTES || &bytes[..8] != LAYER_MAGIC { - return corrupt_at(path, "invalid path-state layer header"); - } - let mut cursor = 8; - if read_u32(bytes, &mut cursor, path)? != LAYER_VERSION { - return corrupt_at(path, "unsupported path-state layer version"); - } - let base_ordinal = read_u32(bytes, &mut cursor, path)?; - let record_count = read_u32(bytes, &mut cursor, path)? as usize; - let node_count = read_u32(bytes, &mut cursor, path)? as usize; - if base_ordinal != reference.base_ordinal - || record_count != reference.commit_count as usize - || node_count != reference.node_count as usize + if bytes.len() as u64 != reference.bytes + || blake3::hash(bytes).to_hex().as_str() != reference.hash { - return corrupt_at( - path, - "path-state layer header does not match its descriptor", - ); - } - let name_count = read_u32(bytes, &mut cursor, path)? as usize; - if name_count > bytes.len().saturating_sub(cursor) / 5 { - return corrupt_at(path, "path-state component count exceeds the layer bytes"); + return corrupt_at(path, "layer content identity mismatch"); } - let mut names = Vec::>::new(); - names - .try_reserve_exact(name_count) - .map_err(|_| corruption("path-state component allocation exceeds capacity"))?; - for _ in 0..name_count { - let length = read_u32(bytes, &mut cursor, path)? as usize; - if length == 0 || length > MAX_PATH_BYTES { - return corrupt_at(path, "path-state component exceeds its limits"); + let mut names = Vec::new(); + let mut visited = BTreeMap::new(); + if let Some(root) = &reference.pages.names { + let rows = paged::scan(bytes, root, Kind::Names, path, &mut visited)?; + let mut ordered = BTreeMap::new(); + for (key, value) in rows { + let (ordinal, name) = decode_name(&key, &value, path)?; + if ordered.insert(ordinal, Arc::<[u8]>::from(name)).is_some() { + return corrupt_at(path, "duplicate component ordinal"); + } } - let name = read_vec(bytes, &mut cursor, length, path)?; - if name.contains(&0) - || name.contains(&b'/') - || names - .last() - .is_some_and(|previous| previous.as_ref() >= name.as_slice()) - { - return corrupt_at(path, "path-state component table is not canonical"); + for (index, (ordinal, name)) in ordered.into_iter().enumerate() { + if ordinal as usize != index + || names + .last() + .is_some_and(|previous: &Arc<[u8]>| previous.as_ref() >= name.as_ref()) + { + return corrupt_at(path, "component ordinals are incomplete or unordered"); + } + names.push(name); } - names.push(Arc::from(name)); - } - let mut used_names = vec![false; name_count]; - if record_count > bytes.len().saturating_sub(cursor) / RECORD_FIXED_BYTES { - return corrupt_at(path, "path-state record count exceeds the layer bytes"); } - let mut records = Vec::new(); - records - .try_reserve_exact(record_count) - .map_err(|_| corruption("path-state record allocation exceeds capacity"))?; - for _ in 0..record_count { - if bytes.len().saturating_sub(cursor) < RECORD_FIXED_BYTES { - return corrupt_at(path, "truncated path-state record"); - } - let oid = read_array(bytes, &mut cursor, path)?; - let first_parent = match read_u32(bytes, &mut cursor, path)? { - u32::MAX => None, - parent => Some(parent), - }; - let author_seconds = read_i64(bytes, &mut cursor, path)?; - let author_len = read_u32(bytes, &mut cursor, path)? as usize; - let message_len = read_u32(bytes, &mut cursor, path)? as usize; - let root = PathStateNodeRef { - layer: read_u32(bytes, &mut cursor, path)?, - index: read_u32(bytes, &mut cursor, path)?, - }; - if author_len > MAX_AUTHOR_BYTES || message_len > MAX_MESSAGE_BYTES { - return corrupt_at(path, "path-state record exceeds its limits"); + let records = paged::scan( + bytes, + &reference.pages.records, + Kind::Records, + path, + &mut visited, + )? + .into_iter() + .map(|(_, value)| decode_record(&value, path)) + .collect::>>()?; + let mut used = vec![false; names.len()]; + let mut nodes = Vec::new(); + for (_, value) in paged::scan( + bytes, + &reference.pages.nodes, + Kind::Nodes, + path, + &mut visited, + )? { + let (value, children) = decode_node(&value, reference.bytes, path)?; + let mut entries = BTreeMap::new(); + if let Some(root) = children { + for (key, value) in paged::scan(bytes, &root, Kind::Children, path, &mut visited)? { + let ordinal = u32::from_be_bytes( + key.as_slice() + .try_into() + .map_err(|_| corruption("invalid component key"))?, + ) as usize; + let name = names + .get(ordinal) + .ok_or_else(|| corruption("component ordinal is out of bounds"))?; + used[ordinal] = true; + let child = decode_child(&value, path)?; + entries.insert(Arc::clone(name), child); + } } - records.push(PathStateRecord { - oid, - first_parent, - author: read_vec(bytes, &mut cursor, author_len, path)?, - author_seconds, - message: read_vec(bytes, &mut cursor, message_len, path)?, - root, + nodes.push(PathStateNode { + value, + children: entries, }); } - if node_count > bytes.len().saturating_sub(cursor) / NODE_FIXED_BYTES { - return corrupt_at(path, "path-state node count exceeds the layer bytes"); + if used.contains(&false) { + return corrupt_at(path, "unused path-state component"); } - let mut nodes = Vec::new(); - nodes - .try_reserve_exact(node_count) - .map_err(|_| corruption("path-state node allocation exceeds capacity"))?; - for _ in 0..node_count { - if bytes.len().saturating_sub(cursor) < NODE_FIXED_BYTES { - return corrupt_at(path, "truncated path-state node"); - } - let value = match read_u32(bytes, &mut cursor, path)? { - u32::MAX => None, - value => Some(value), - }; - let child_count = read_u32(bytes, &mut cursor, path)? as usize; - if child_count > MAX_CHILDREN_PER_NODE - || child_count > bytes.len().saturating_sub(cursor) / CHILD_FIXED_BYTES - { - return corrupt_at(path, "path-state node has too many children"); - } - let mut children = BTreeMap::new(); - let mut previous_name = None; - for _ in 0..child_count { - if bytes.len().saturating_sub(cursor) < CHILD_FIXED_BYTES { - return corrupt_at(path, "truncated path-state child"); - } - let name_ordinal = read_u32(bytes, &mut cursor, path)? as usize; - if previous_name.is_some_and(|previous| previous >= name_ordinal) { - return corrupt_at(path, "path-state child names are not in canonical order"); - } - let name = names - .get(name_ordinal) - .ok_or_else(|| corruption("path-state child component is out of bounds"))?; - let used = used_names - .get_mut(name_ordinal) - .ok_or_else(|| corruption("path-state child component is out of bounds"))?; - *used = true; - previous_name = Some(name_ordinal); - let child = PathStateNodeRef { - layer: read_u32(bytes, &mut cursor, path)?, - index: read_u32(bytes, &mut cursor, path)?, - }; - children.insert(Arc::clone(name), child); - } - nodes.push(PathStateNode { value, children }); + if records.len() != reference.commit_count as usize + || nodes.len() != reference.node_count as usize + { + return corrupt_at(path, "layer count does not match its descriptor"); } - if used_names.contains(&false) { - return corrupt_at(path, "path-state component table has an unused name"); + let mut offset = 0u64; + for (start, (_, page)) in visited { + if start != offset { + return corrupt_at(path, "layer pages overlap or leave unused bytes"); + } + offset = offset + .checked_add(u64::from(page.bytes)) + .ok_or_else(|| corruption("layer coverage overflows"))?; } - if cursor != bytes.len() { - return corrupt_at(path, "path-state layer has trailing bytes"); + if offset != reference.bytes { + return corrupt_at(path, "layer contains trailing unreferenced bytes"); } - tracing::debug!(target: "crab_metadata::index_decode", index_format = "path_state", - decoded_bytes = bytes.len() as u64, "index block decoded"); Ok(PathStateLayer { - base_ordinal, + base_ordinal: reference.base_ordinal, records, nodes, }) } #[cfg(any(feature = "storage", test))] -fn read_vec(bytes: &[u8], cursor: &mut usize, length: usize, path: &str) -> Result> { - let end = cursor - .checked_add(length) - .ok_or_else(|| corruption("path-state offset overflows"))?; - let value = bytes - .get(*cursor..end) - .ok_or_else(|| MetadataError::CorruptObject { - path: path.to_owned(), - reason: "truncated path-state bytes".to_owned(), - })?; - *cursor = end; - Ok(value.to_vec()) +pub(super) fn decode_record(bytes: &[u8], path: &str) -> Result { + let mut input = Input::new(bytes, path); + let oid = input.array()?; + let first_parent = optional(input.u32()?); + let author_seconds = i64::from_le_bytes(input.array()?); + let author_len = input.u32()? as usize; + let message_len = input.u32()? as usize; + let root = PathStateNodeRef { + layer: input.u32()?, + index: input.u32()?, + }; + if author_len > MAX_AUTHOR_BYTES || message_len > MAX_MESSAGE_BYTES { + return corrupt_at(path, "summary exceeds its byte limit"); + } + let author = input.take(author_len)?.to_vec(); + let message = input.take(message_len)?.to_vec(); + input.finish()?; + Ok(PathStateRecord { + oid, + first_parent, + author_seconds, + author, + message, + root, + }) } #[cfg(any(feature = "storage", test))] -fn read_u32(bytes: &[u8], cursor: &mut usize, path: &str) -> Result { - Ok(u32::from_le_bytes(read_array(bytes, cursor, path)?)) +pub(super) fn decode_node( + bytes: &[u8], + source_bytes: u64, + path: &str, +) -> Result<(Option, Option)> { + let mut input = Input::new(bytes, path); + let value = optional(input.u32()?); + let children = match input.array::<1>()?[0] { + 0 => None, + 1 => { + let root = paged::decode_ref(&mut input, Kind::Children)?; + root.validate(Kind::Children, source_bytes)?; + if root.count as usize > super::MAX_CHILDREN_PER_NODE { + return corrupt_at(path, "node has too many children"); + } + Some(root) + } + _ => return corrupt_at(path, "invalid child-root tag"), + }; + input.finish()?; + Ok((value, children)) } #[cfg(any(feature = "storage", test))] -fn read_i64(bytes: &[u8], cursor: &mut usize, path: &str) -> Result { - Ok(i64::from_le_bytes(read_array(bytes, cursor, path)?)) +pub(super) fn decode_name<'a>(key: &[u8], bytes: &'a [u8], path: &str) -> Result<(u32, &'a [u8])> { + let mut input = Input::new(bytes, path); + let ordinal = input.u32()?; + let name = input.take(input.remaining())?; + validate_name(name)?; + if blake3::hash(name).as_bytes().as_slice() != key { + return corrupt_at(path, "component identity mismatch"); + } + Ok((ordinal, name)) } #[cfg(any(feature = "storage", test))] -fn read_array(bytes: &[u8], cursor: &mut usize, path: &str) -> Result<[u8; N]> { - let end = cursor - .checked_add(N) - .ok_or_else(|| corruption("path-state offset overflows"))?; - let value = bytes - .get(*cursor..end) - .ok_or_else(|| MetadataError::CorruptObject { - path: path.to_owned(), - reason: "truncated path-state bytes".to_owned(), - })?; - *cursor = end; - value.try_into().map_err(|_| MetadataError::CorruptObject { - path: path.to_owned(), - reason: "invalid path-state scalar".to_owned(), - }) +pub(super) fn decode_child(bytes: &[u8], path: &str) -> Result { + let mut input = Input::new(bytes, path); + let child = PathStateNodeRef { + layer: input.u32()?, + index: input.u32()?, + }; + input.finish()?; + Ok(child) +} + +#[cfg(any(feature = "storage", test))] +fn optional(value: u32) -> Option { + (value != u32::MAX).then_some(value) } #[cfg(test)] mod tests { use super::*; - #[test] - fn repeated_raw_components_are_encoded_once_per_layer() { - let name: Arc<[u8]> = Arc::from(b"repeated-path-component-\xff".as_slice()); + fn fixture() -> PathStateLayer { + let name: Arc<[u8]> = Arc::from(b"repeated-component-\xff".as_slice()); let mut nodes = vec![PathStateNode { value: Some(0), children: BTreeMap::new(), @@ -272,7 +308,7 @@ mod tests { value: None, children: BTreeMap::from([(name.clone(), PathStateNodeRef { layer: 0, index: 0 })]), })); - let layer = PathStateLayer { + PathStateLayer { base_ordinal: 0, records: vec![PathStateRecord { oid: [1; 20], @@ -286,115 +322,63 @@ mod tests { }, }], nodes, - }; - let encoded = encode_layer(&layer).unwrap(); - let reference = PathStateLayerRef { - hash: blake3::hash(&encoded).to_hex().to_string(), + } + } + + fn reference(encoded: &EncodedLayer, layer: &PathStateLayer) -> PathStateLayerRef { + PathStateLayerRef { + hash: blake3::hash(&encoded.bytes).to_hex().to_string(), path: "layer".to_owned(), base_ordinal: 0, - commit_count: 1, + commit_count: layer.records.len() as u32, node_count: layer.nodes.len() as u32, - bytes: encoded.len() as u64, - }; - let decoded = decode_layer(&encoded, &reference, "layer").unwrap(); - let shared = Arc::ptr_eq( - decoded.nodes[1].children.keys().next().unwrap(), - decoded.nodes[1024].children.keys().next().unwrap(), - ); + bytes: encoded.bytes.len() as u64, + pages: encoded.pages.clone(), + } + } + + #[test] + fn repeated_raw_components_remain_shared_and_encoded_once() { + let layer = fixture(); + let encoded = encode_layer(&layer).unwrap(); + let decoded = decode_layer(&encoded.bytes, &reference(&encoded, &layer), "layer").unwrap(); + let name = layer.nodes[1].children.keys().next().unwrap(); assert_eq!( ( decoded == layer, encoded + .bytes .windows(name.len()) - .filter(|window| *window == name.as_ref()) + .filter(|bytes| *bytes == name.as_ref()) .count(), - shared + Arc::ptr_eq( + decoded.nodes[1].children.keys().next().unwrap(), + decoded.nodes[1024].children.keys().next().unwrap() + ) ), (true, 1, true) ); } #[test] - fn component_tables_reject_invalid_names_ordinals_and_truncated_layers() { - let leaf = PathStateNode { - value: Some(0), - children: BTreeMap::new(), - }; - let layer = PathStateLayer { - base_ordinal: 0, - records: vec![PathStateRecord { - oid: [1; 20], - first_parent: None, - author_seconds: 1, - author: Vec::new(), - message: Vec::new(), - root: PathStateNodeRef { layer: 0, index: 2 }, - }], - nodes: vec![ - leaf.clone(), - leaf, - PathStateNode { - value: None, - children: BTreeMap::from([ - ( - Arc::from(b"a".as_slice()), - PathStateNodeRef { layer: 0, index: 0 }, - ), - ( - Arc::from(b"b".as_slice()), - PathStateNodeRef { layer: 0, index: 1 }, - ), - ]), - }, - ], - }; + fn full_validation_rejects_trailing_unreferenced_bytes() { + let layer = fixture(); + let mut encoded = encode_layer(&layer).unwrap(); + encoded.bytes.extend_from_slice(b"unreferenced"); + let reference = reference(&encoded, &layer); + assert!(decode_layer(&encoded.bytes, &reference, "layer").is_err()); + } + + #[test] + fn stored_layers_reject_corruption_and_truncation() { + let layer = fixture(); let encoded = encode_layer(&layer).unwrap(); - let reference = PathStateLayerRef { - hash: blake3::hash(&encoded).to_hex().to_string(), - path: "layer".to_owned(), - base_ordinal: 0, - commit_count: 1, - node_count: 3, - bytes: encoded.len() as u64, - }; - let edges = LAYER_HEADER_BYTES + 2 * 5 + 48 + 3 * 8; - for (name, offset, replacement) in [ - ("excessive table count", 24, u32::MAX.to_le_bytes().to_vec()), - ( - "empty component", - LAYER_HEADER_BYTES, - 0_u32.to_le_bytes().to_vec(), - ), - ("nul component", LAYER_HEADER_BYTES + 4, vec![0]), - ("slash component", LAYER_HEADER_BYTES + 4, vec![b'/']), - ("duplicate table name", LAYER_HEADER_BYTES + 9, vec![b'a']), - ( - "invalid child ordinal", - edges, - u32::MAX.to_le_bytes().to_vec(), - ), - ( - "duplicate child ordinal", - edges + 12, - 0_u32.to_le_bytes().to_vec(), - ), - ] { - let mut invalid = encoded.clone(); - invalid[offset..offset + replacement.len()].copy_from_slice(&replacement); - assert!( - decode_layer(&invalid, &reference, "layer").is_err(), - "{name}" - ); - } - for length in 0..encoded.len() { - assert!( - decode_layer(&encoded[..length], &reference, "layer").is_err(), - "truncated at {length}" - ); + let reference = reference(&encoded, &layer); + for cut in [0, 13, encoded.bytes.len() - 1] { + assert!(decode_layer(&encoded.bytes[..cut], &reference, "layer").is_err()); } - let mut unused = encoded.clone(); - unused[edges - 4..edges].copy_from_slice(&1_u32.to_le_bytes()); - unused.truncate(unused.len() - 12); - assert!(decode_layer(&unused, &reference, "layer").is_err()); + let mut damaged = encoded.bytes; + damaged[20] ^= 1; + assert!(decode_layer(&damaged, &reference, "layer").is_err()); } } diff --git a/crates/crab-metadata/src/path_state/paged.rs b/crates/crab-metadata/src/path_state/paged.rs new file mode 100644 index 000000000..1997c67cc --- /dev/null +++ b/crates/crab-metadata/src/path_state/paged.rs @@ -0,0 +1,602 @@ +//! Authenticated, bounded trees inside one immutable path-state layer. +use serde::{Deserialize, Serialize}; + +#[cfg(any(feature = "storage", test))] +use super::corrupt_at; +use super::{Result, corruption}; + +const MAGIC: &[u8; 8] = b"CRABPP01"; +const HEADER: usize = 14; +pub(super) const TARGET_BYTES: usize = 32 * 1024; +pub(super) const MAX_BYTES: usize = 2 * 1024 * 1024; +const MAX_HEIGHT: u8 = 16; + +/// Authenticated location and key coverage of one immutable page. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PathStatePageRef { + pub offset: u64, + pub bytes: u32, + pub hash: [u8; 32], + pub height: u8, + pub count: u32, + pub first: Vec, + pub last: Vec, +} + +/// Bounded page roots committed by an immutable layer descriptor. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PathStateLayerPages { + pub records: PathStatePageRef, + pub nodes: PathStatePageRef, + pub names: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub(super) enum Kind { + Records, + Nodes, + Children, + Names, +} + +impl Kind { + pub(super) fn key_bytes(self) -> usize { + if self == Self::Names { 32 } else { 4 } + } +} + +pub(super) type Rows = Vec<(Vec, Vec)>; +#[cfg(any(feature = "storage", test))] +#[derive(Debug)] +pub(super) enum Page { + Leaf(Rows), + Branch(Vec), +} + +impl PathStatePageRef { + pub(super) fn validate(&self, kind: Kind, source_bytes: u64) -> Result<()> { + if self.first.len() != kind.key_bytes() + || self.last.len() != kind.key_bytes() + || self.first > self.last + || self.count == 0 + || self.height > MAX_HEIGHT + || self.bytes as usize > MAX_BYTES + || (self.bytes as usize) < HEADER + || self + .offset + .checked_add(u64::from(self.bytes)) + .is_none_or(|end| end > source_bytes) + { + return Err(corruption("invalid path-state page reference")); + } + if matches!(kind, Kind::Records | Kind::Nodes) { + let first = integer(&self.first)?; + let last = integer(&self.last)?; + if last.checked_sub(first).and_then(|span| span.checked_add(1)) != Some(self.count) { + return Err(corruption("positional page has incomplete key coverage")); + } + } + Ok(()) + } +} + +#[derive(Default)] +pub(super) struct Writer { + pub(super) bytes: Vec, + pages: std::collections::BTreeMap<[u8; 32], PathStatePageRef>, +} + +impl Writer { + pub(super) fn tree(&mut self, kind: Kind, rows: Rows) -> Result { + validate_rows(kind, &rows)?; + let mut level = Vec::new(); + let mut pending = Vec::new(); + let mut size = HEADER; + for row in rows { + let row_bytes = kind + .key_bytes() + .checked_add(4) + .and_then(|n| n.checked_add(row.1.len())) + .ok_or_else(|| corruption("path-state row size overflows"))?; + if row_bytes > MAX_BYTES - HEADER { + return Err(corruption("path-state row exceeds the page bound")); + } + if !pending.is_empty() && size + row_bytes > TARGET_BYTES { + level.push(self.leaf(kind, std::mem::take(&mut pending))?); + size = HEADER; + } + size += row_bytes; + pending.push(row); + } + if !pending.is_empty() { + level.push(self.leaf(kind, pending)?); + } + let fanout = (TARGET_BYTES - HEADER) / ref_bytes(kind); + while level.len() > 1 { + let mut parents = Vec::new(); + for children in level.chunks(fanout) { + parents.push(self.branch(kind, children)?); + } + level = parents; + } + level + .pop() + .ok_or_else(|| corruption("empty path-state page tree")) + } + + fn leaf(&mut self, kind: Kind, rows: Rows) -> Result { + let first = rows + .first() + .ok_or_else(|| corruption("empty page"))? + .0 + .clone(); + let last = rows + .last() + .ok_or_else(|| corruption("empty page"))? + .0 + .clone(); + let count = + u32::try_from(rows.len()).map_err(|_| corruption("page row count overflows"))?; + let mut bytes = header(kind, 0, count); + for (key, value) in rows { + bytes.extend_from_slice(&key); + put_u32(&mut bytes, value.len())?; + bytes.extend_from_slice(&value); + } + self.append(bytes, 0, count, first, last) + } + + fn branch(&mut self, kind: Kind, children: &[PathStatePageRef]) -> Result { + let first = children + .first() + .ok_or_else(|| corruption("empty directory"))?; + let last = children + .last() + .ok_or_else(|| corruption("empty directory"))?; + let height = first + .height + .checked_add(1) + .filter(|height| *height <= MAX_HEIGHT) + .ok_or_else(|| corruption("page tree is too deep"))?; + let count = children.iter().try_fold(0u32, |count, child| { + count + .checked_add(child.count) + .ok_or_else(|| corruption("directory count overflows")) + })?; + let mut bytes = header( + kind, + height, + u32::try_from(children.len()).map_err(|_| corruption("directory size overflows"))?, + ); + for child in children { + encode_ref(&mut bytes, child); + } + self.append(bytes, height, count, first.first.clone(), last.last.clone()) + } + + fn append( + &mut self, + bytes: Vec, + height: u8, + count: u32, + first: Vec, + last: Vec, + ) -> Result { + if bytes.len() > MAX_BYTES { + return Err(corruption("page exceeds its byte bound")); + } + let hash = *blake3::hash(&bytes).as_bytes(); + if let Some(reference) = self.pages.get(&hash) { + return Ok(reference.clone()); + } + let reference = PathStatePageRef { + offset: self.bytes.len() as u64, + bytes: bytes.len() as u32, + hash, + height, + count, + first, + last, + }; + self.bytes.extend_from_slice(&bytes); + self.pages.insert(hash, reference.clone()); + Ok(reference) + } +} + +fn header(kind: Kind, height: u8, count: u32) -> Vec { + let mut bytes = MAGIC.to_vec(); + bytes.extend_from_slice(&[kind as u8, height]); + bytes.extend_from_slice(&count.to_le_bytes()); + bytes +} + +#[cfg(any(feature = "storage", test))] +pub(super) fn decode( + bytes: &[u8], + reference: &PathStatePageRef, + kind: Kind, + source_bytes: u64, + path: &str, +) -> Result { + reference.validate(kind, source_bytes)?; + if bytes.len() != reference.bytes as usize + || *blake3::hash(bytes).as_bytes() != reference.hash + || bytes.get(..8) != Some(MAGIC.as_slice()) + || bytes.get(8) != Some(&(kind as u8)) + || bytes.get(9) != Some(&reference.height) + { + return corrupt_at(path, "path-state page identity or header mismatch"); + } + let mut input = Input::new(&bytes[10..], path); + let count = input.u32()? as usize; + if count == 0 { + return corrupt_at(path, "empty path-state page"); + } + let page = if reference.height == 0 { + if count > input.remaining() / (kind.key_bytes() + 4) { + return corrupt_at(path, "leaf count exceeds its encoded bytes"); + } + let mut rows = Vec::new(); + for _ in 0..count { + let key = input.take(kind.key_bytes())?.to_vec(); + let length = input.u32()? as usize; + let value = input.take(length)?.to_vec(); + rows.push((key, value)); + } + validate_rows(kind, &rows)?; + if count != reference.count as usize + || rows.first().map(|row| &row.0) != Some(&reference.first) + || rows.last().map(|row| &row.0) != Some(&reference.last) + { + return corrupt_at(path, "leaf coverage does not match its reference"); + } + Page::Leaf(rows) + } else { + if count > input.remaining() / ref_bytes(kind) { + return corrupt_at(path, "directory count exceeds its encoded bytes"); + } + let mut children: Vec = Vec::new(); + let mut total = 0u32; + for _ in 0..count { + let child = decode_ref(&mut input, kind)?; + child.validate(kind, source_bytes)?; + if child.height.checked_add(1) != Some(reference.height) + || child + .offset + .checked_add(u64::from(child.bytes)) + .is_none_or(|end| end > reference.offset) + || children + .last() + .is_some_and(|previous| previous.last >= child.first) + { + return corrupt_at(path, "invalid page directory routing or ancestry"); + } + if matches!(kind, Kind::Records | Kind::Nodes) + && children.last().is_some_and(|previous| { + integer(&previous.last) + .ok() + .and_then(|last| last.checked_add(1)) + != integer(&child.first).ok() + }) + { + return corrupt_at(path, "positional directory has a key gap"); + } + total = total + .checked_add(child.count) + .ok_or_else(|| corruption("directory count overflows"))?; + children.push(child); + } + if total != reference.count + || children.first().map(|child| &child.first) != Some(&reference.first) + || children.last().map(|child| &child.last) != Some(&reference.last) + { + return corrupt_at(path, "directory coverage does not match its reference"); + } + Page::Branch(children) + }; + input.finish()?; + tracing::debug!(target:"crab_metadata::index_decode", index_format="path_state", + decoded_bytes=bytes.len() as u64, "index page decoded"); + Ok(page) +} + +#[cfg(any(feature = "storage", test))] +pub(super) fn scan( + source: &[u8], + root: &PathStatePageRef, + kind: Kind, + path: &str, + visited: &mut std::collections::BTreeMap, +) -> Result { + let mut pending = vec![root.clone()]; + let mut rows = Vec::new(); + while let Some(reference) = pending.pop() { + if let Some((previous_kind, previous)) = + visited.insert(reference.offset, (kind, reference.clone())) + && (previous_kind != kind || previous != reference) + { + return corrupt_at(path, "page has conflicting references"); + } + let page = page_bytes(source, &reference, path)?; + match decode(page, &reference, kind, source.len() as u64, path)? { + Page::Leaf(items) => rows.extend(items), + Page::Branch(children) => pending.extend(children.into_iter().rev()), + } + } + if rows.len() != root.count as usize { + return corrupt_at(path, "tree row count mismatch"); + } + Ok(rows) +} + +#[cfg(any(feature = "storage", test))] +pub(super) fn page_bytes<'a>( + source: &'a [u8], + reference: &PathStatePageRef, + path: &str, +) -> Result<&'a [u8]> { + let start = usize::try_from(reference.offset) + .map_err(|_| corruption("page offset exceeds address space"))?; + let end = start + .checked_add(reference.bytes as usize) + .ok_or_else(|| corruption("page range overflows"))?; + source + .get(start..end) + .ok_or_else(|| crate::validation::corrupt_object(path, "page range is absent")) +} + +fn validate_rows(kind: Kind, rows: &Rows) -> Result<()> { + if rows.is_empty() + || rows.iter().any(|row| row.0.len() != kind.key_bytes()) + || rows.windows(2).any(|pair| pair[0].0 >= pair[1].0) + { + return Err(corruption("page keys are empty, invalid or unordered")); + } + if matches!(kind, Kind::Records | Kind::Nodes) + && rows.windows(2).any(|pair| { + integer(&pair[0].0) + .ok() + .and_then(|value| value.checked_add(1)) + != integer(&pair[1].0).ok() + }) + { + return Err(corruption("positional leaf has a key gap")); + } + Ok(()) +} + +fn integer(key: &[u8]) -> Result { + let bytes = key + .try_into() + .map_err(|_| corruption("invalid ordinal key"))?; + Ok(u32::from_be_bytes(bytes)) +} + +fn ref_bytes(kind: Kind) -> usize { + 49 + 2 * kind.key_bytes() +} + +pub(super) fn encode_ref(bytes: &mut Vec, reference: &PathStatePageRef) { + bytes.extend_from_slice(&reference.offset.to_le_bytes()); + bytes.extend_from_slice(&reference.bytes.to_le_bytes()); + bytes.extend_from_slice(&reference.hash); + bytes.push(reference.height); + bytes.extend_from_slice(&reference.count.to_le_bytes()); + bytes.extend_from_slice(&reference.first); + bytes.extend_from_slice(&reference.last); +} + +#[cfg(any(feature = "storage", test))] +pub(super) fn decode_ref(input: &mut Input<'_, '_>, kind: Kind) -> Result { + Ok(PathStatePageRef { + offset: u64::from_le_bytes(input.array()?), + bytes: input.u32()?, + hash: input.array()?, + height: input.array::<1>()?[0], + count: input.u32()?, + first: input.take(kind.key_bytes())?.to_vec(), + last: input.take(kind.key_bytes())?.to_vec(), + }) +} + +pub(super) fn put_u32(bytes: &mut Vec, value: usize) -> Result<()> { + bytes.extend_from_slice( + &u32::try_from(value) + .map_err(|_| corruption("page scalar overflows"))? + .to_le_bytes(), + ); + Ok(()) +} + +#[cfg(any(feature = "storage", test))] +pub(super) struct Input<'a, 'p> { + bytes: &'a [u8], + path: &'p str, +} +#[cfg(any(feature = "storage", test))] +impl<'a, 'p> Input<'a, 'p> { + pub(super) fn new(bytes: &'a [u8], path: &'p str) -> Self { + Self { bytes, path } + } + pub(super) fn remaining(&self) -> usize { + self.bytes.len() + } + pub(super) fn take(&mut self, count: usize) -> Result<&'a [u8]> { + let (head, tail) = self + .bytes + .split_at_checked(count) + .ok_or_else(|| crate::validation::corrupt_object(self.path, "truncated page row"))?; + self.bytes = tail; + Ok(head) + } + pub(super) fn array(&mut self) -> Result<[u8; N]> { + self.take(N)? + .try_into() + .map_err(|_| corruption("invalid page scalar")) + } + pub(super) fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.array()?)) + } + pub(super) fn finish(self) -> Result<()> { + if !self.bytes.is_empty() { + return corrupt_at(self.path, "page has trailing bytes"); + } + Ok(()) + } +} + +#[cfg(any(feature = "storage", test))] +impl Page { + pub(super) fn resident_bytes(&self) -> usize { + match self { + Self::Leaf(rows) => rows.iter().fold( + rows.capacity() + .saturating_mul(std::mem::size_of::<(Vec, Vec)>()), + |total, (key, value)| { + total + .saturating_add(key.capacity()) + .saturating_add(value.capacity()) + }, + ), + Self::Branch(children) => children.iter().fold( + children + .capacity() + .saturating_mul(std::mem::size_of::()), + |total, child| { + total + .saturating_add(child.first.capacity()) + .saturating_add(child.last.capacity()) + }, + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rows(count: u32, value_bytes: usize) -> Rows { + (0..count) + .map(|index| (index.to_be_bytes().to_vec(), vec![7; value_bytes])) + .collect() + } + + #[test] + fn large_trees_page_their_internal_directories() { + let expected = rows(20_000, 1000); + let mut writer = Writer::default(); + let root = writer.tree(Kind::Records, expected.clone()).unwrap(); + let actual = scan( + &writer.bytes, + &root, + Kind::Records, + "tree", + &mut Default::default(), + ) + .unwrap(); + assert_eq!( + ( + root.height >= 2, + writer + .pages + .values() + .all(|page| page.bytes as usize <= MAX_BYTES), + actual == expected + ), + (true, true, true) + ); + } + + #[test] + fn authenticated_directories_reject_overlapping_routing() { + let mut writer = Writer::default(); + let mut root = writer.tree(Kind::Records, rows(100, 1000)).unwrap(); + let mut body = page_bytes(&writer.bytes, &root, "tree").unwrap().to_vec(); + let first = body[HEADER..HEADER + ref_bytes(Kind::Records)].to_vec(); + body[HEADER + ref_bytes(Kind::Records)..HEADER + 2 * ref_bytes(Kind::Records)] + .copy_from_slice(&first); + root.hash = *blake3::hash(&body).as_bytes(); + assert!( + decode( + &body, + &root, + Kind::Records, + writer.bytes.len() as u64, + "tree" + ) + .is_err() + ); + } + + #[test] + fn page_counts_cannot_allocate_beyond_encoded_bytes() { + let mut writer = Writer::default(); + let mut root = writer.tree(Kind::Records, rows(3, 48)).unwrap(); + let mut body = page_bytes(&writer.bytes, &root, "tree").unwrap().to_vec(); + body[10..14].copy_from_slice(&u32::MAX.to_le_bytes()); + root.hash = *blake3::hash(&body).as_bytes(); + assert!( + decode( + &body, + &root, + Kind::Records, + writer.bytes.len() as u64, + "tree" + ) + .is_err() + ); + } + + #[test] + fn page_hash_lengths_and_bounds_are_checked_before_decoding() { + let mut writer = Writer::default(); + let root = writer.tree(Kind::Records, rows(3, 48)).unwrap(); + let body = page_bytes(&writer.bytes, &root, "tree").unwrap().to_vec(); + let mut invalid = root.clone(); + invalid.first = 1u32.to_be_bytes().to_vec(); + let mut damaged = body.clone(); + damaged[HEADER] ^= 1; + let oversized = PathStatePageRef { + bytes: (MAX_BYTES + 1) as u32, + ..root.clone() + }; + assert_eq!( + ( + decode( + &body, + &invalid, + Kind::Records, + writer.bytes.len() as u64, + "tree" + ) + .is_err(), + decode( + &damaged, + &root, + Kind::Records, + writer.bytes.len() as u64, + "tree" + ) + .is_err(), + oversized.validate(Kind::Records, u64::MAX).is_err() + ), + (true, true, true) + ); + } + + proptest::proptest! { + #[test] + fn ordered_page_trees_round_trip(count in 1u32..1000, value in proptest::collection::vec(proptest::num::u8::ANY, 0..300)) { + let expected = (0..count).map(|index| (index.to_be_bytes().to_vec(), value.clone())).collect::(); + let mut writer = Writer::default(); + let root = writer.tree(Kind::Records, expected.clone()).unwrap(); + let actual = scan(&writer.bytes, &root, Kind::Records, "tree", &mut Default::default()).unwrap(); + proptest::prop_assert_eq!(actual, expected); + } + } +} diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs new file mode 100644 index 000000000..dd9f6df3a --- /dev/null +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -0,0 +1,511 @@ +//! Lazy lookups bound to an authenticated descriptor; full scans remain maintenance operations. +use crab_storage::{Store, StoreLayout}; +use futures_util::{StreamExt, TryStreamExt}; +use std::sync::{Mutex, MutexGuard}; +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + sync::Arc, +}; + +use super::paged::{self, Kind, Page, PathStatePageRef}; +use super::{ + PathStateDescriptor, PathStateNodeRef, PathStateRecord, PathStateSummary, Result, codec, + corrupt, corruption, validate_path, +}; +use crate::split_commit_graph::{CommitGraphLayerRef, SplitCommitGraph}; + +const CACHE_BYTES: usize = 8 * 1024 * 1024; +const CACHE_PAGES: usize = 256; + +/// Verified path attribution with bounded, shared immutable-page caching. +#[derive(Debug)] +pub struct PathStateReader { + descriptor: PathStateDescriptor, + graph_layers: Vec, + paths: Vec, + descriptor_bytes: u64, + max_bytes: u64, + cache: Mutex, +} + +#[derive(Debug, Default)] +struct Cache { + pages: BTreeMap<(usize, u64), (Kind, PathStatePageRef, Arc)>, + order: VecDeque<(usize, u64)>, + bytes: usize, +} + +struct ChildBatch { + root: PathStatePageRef, + requests: Vec<(usize, u32)>, +} + +struct Work { + maximum: u64, + bytes: u64, + seen: BTreeSet<(usize, u64)>, +} + +impl PathStateReader { + /// Load only a descriptor and bind its positional roots to the supplied graph. + pub async fn load( + store: &Store, + layout: &StoreLayout, + hash: &str, + graph: &SplitCommitGraph, + max_bytes: u64, + ) -> Result { + let (descriptor, descriptor_bytes) = + super::storage::load_path_state_descriptor_with_size(store, layout, hash, max_bytes) + .await?; + if !matches_graph(&descriptor, graph) + || descriptor.commit_ordinal_digest != graph.ordinal_digest() + { + return corrupt("path-state descriptor does not match its commit graph"); + } + let paths = descriptor + .layers + .iter() + .map(|layer| layout.repo_path(&layer.path)) + .collect(); + Ok(Self { + descriptor, + graph_layers: graph.descriptor.layers.clone(), + paths, + descriptor_bytes, + max_bytes, + cache: Mutex::new(Cache::default()), + }) + } + + /// Resolve raw paths using the captured graph and an operation-admitted store for the same origin. + pub async fn latest( + &self, + store: &Store, + graph: &SplitCommitGraph, + start: &[u8; 20], + paths: &[&[u8]], + ) -> Result> { + // Scalars can match two rebuilt graphs with different ordinals. The + // immutable layer identities pin the ordering verified during load. + if !matches_graph(&self.descriptor, graph) || self.graph_layers != graph.descriptor.layers { + return corrupt("path-state query graph identity changed"); + } + if paths.is_empty() { + return Ok(Vec::new()); + } + for path in paths { + validate_path(path, false)?; + } + let start_ordinal = graph + .ordinal(start) + .ok_or_else(|| corruption("start commit is absent from graph"))?; + let mut work = Work { + maximum: self.max_bytes, + bytes: self.descriptor_bytes, + seen: BTreeSet::new(), + }; + let start = self + .records(store, graph, &[start_ordinal], &mut work) + .await? + .remove(0); + let mut nodes = vec![start.root; paths.len()]; + let mut remaining = paths + .iter() + .map(|path| path.split(|byte| *byte == b'/')) + .collect::>(); + let mut pending = (0..paths.len()).collect::>(); + while !pending.is_empty() { + let states = self.nodes(store, &nodes, &pending, &mut work).await?; + let mut components = BTreeMap::>::new(); + for (&position, state) in pending.iter().zip(states) { + let Some(component) = remaining[position].next() else { + continue; + }; + let children = state + .1 + .ok_or_else(|| corruption("visible path is absent from its trie"))?; + components + .entry(nodes[position].layer as usize) + .or_default() + .push((position, component, children)); + } + let mut children = BTreeMap::<(usize, u64), ChildBatch>::new(); + for (layer, requests) in components { + let names = self.descriptor.layers[layer] + .pages + .names + .as_ref() + .ok_or_else(|| corruption("children have no component directory"))?; + let keys = requests + .iter() + .map(|(_, name, _)| blake3::hash(name).as_bytes().to_vec()) + .collect::>(); + let values = self + .lookup(store, layer, names, Kind::Names, &keys, &mut work) + .await?; + for ((position, component, root), (key, value)) in + requests.into_iter().zip(keys.into_iter().zip(values)) + { + let value = value.ok_or_else(|| corruption("visible component is absent"))?; + let (ordinal, name) = + codec::decode_name(&key, &value, self.paths[layer].as_ref())?; + if name != component || ordinal >= names.count { + return corrupt("component identity or ordinal mismatch"); + } + let batch = + children + .entry((layer, root.offset)) + .or_insert_with(|| ChildBatch { + root: root.clone(), + requests: Vec::new(), + }); + if batch.root != root { + return corrupt("shared child roots have conflicting routing metadata"); + } + batch.requests.push((position, ordinal)); + } + } + pending.clear(); + for ((layer, _), batch) in children { + let keys = batch + .requests + .iter() + .map(|(_, ordinal)| ordinal.to_be_bytes().to_vec()) + .collect::>(); + let values = self + .lookup(store, layer, &batch.root, Kind::Children, &keys, &mut work) + .await?; + for ((position, _), value) in batch.requests.into_iter().zip(values) { + let value = value.ok_or_else(|| corruption("visible child is absent"))?; + let child = codec::decode_child(&value, self.paths[layer].as_ref())?; + self.validate_node(child, Some(nodes[position]))?; + nodes[position] = child; + pending.push(position); + } + } + } + let positions = (0..paths.len()).collect::>(); + let states = self.nodes(store, &nodes, &positions, &mut work).await?; + let ordinals = states + .into_iter() + .map(|(value, _)| { + value + .filter(|ordinal| *ordinal <= start_ordinal) + .ok_or_else(|| { + corruption("path attribution is absent or points to a future commit") + }) + }) + .collect::>>()?; + Ok(self + .records(store, graph, &ordinals, &mut work) + .await? + .into_iter() + .map(|record| PathStateSummary { + oid: record.oid, + author: record.author, + author_seconds: record.author_seconds, + message: record.message, + }) + .collect()) + } + + async fn records( + &self, + store: &Store, + graph: &SplitCommitGraph, + ordinals: &[u32], + work: &mut Work, + ) -> Result> { + let mut grouped = BTreeMap::>::new(); + for (position, &ordinal) in ordinals.iter().enumerate() { + let layer = self + .descriptor + .layers + .partition_point(|layer| layer.base_ordinal <= ordinal) + .checked_sub(1) + .ok_or_else(|| corruption("record layer is absent"))?; + let reference = &self.descriptor.layers[layer]; + if ordinal - reference.base_ordinal >= reference.commit_count { + return corrupt("record ordinal is out of bounds"); + } + grouped.entry(layer).or_default().push((position, ordinal)); + } + let mut output = vec![None; ordinals.len()]; + for (layer, requests) in grouped { + let reference = &self.descriptor.layers[layer]; + let keys = requests + .iter() + .map(|(_, ordinal)| (ordinal - reference.base_ordinal).to_be_bytes().to_vec()) + .collect::>(); + let values = self + .lookup( + store, + layer, + &reference.pages.records, + Kind::Records, + &keys, + work, + ) + .await?; + for ((position, ordinal), value) in requests.into_iter().zip(values) { + let value = value.ok_or_else(|| corruption("positional record is missing"))?; + let record = codec::decode_record(&value, self.paths[layer].as_ref())?; + let commit = graph + .record(ordinal) + .ok_or_else(|| corruption("commit graph record is missing"))?; + if record.oid != commit.oid + || record.first_parent != commit.parents.first().copied() + { + return corrupt("path-state record does not match its commit ordinal"); + } + self.validate_node(record.root, None)?; + if record.root.layer as usize > layer { + return corrupt("record root points to a future layer"); + } + output[position] = Some(record); + } + } + output + .into_iter() + .map(|record| record.ok_or_else(|| corruption("record result is incomplete"))) + .collect() + } + + async fn nodes( + &self, + store: &Store, + nodes: &[PathStateNodeRef], + positions: &[usize], + work: &mut Work, + ) -> Result, Option)>> { + let mut grouped = BTreeMap::>::new(); + for (output, &position) in positions.iter().enumerate() { + let node = nodes[position]; + self.validate_node(node, None)?; + grouped + .entry(node.layer as usize) + .or_default() + .push((output, node)); + } + let mut output = vec![None; positions.len()]; + for (layer, requests) in grouped { + let reference = &self.descriptor.layers[layer]; + let keys = requests + .iter() + .map(|(_, node)| node.index.to_be_bytes().to_vec()) + .collect::>(); + let values = self + .lookup( + store, + layer, + &reference.pages.nodes, + Kind::Nodes, + &keys, + work, + ) + .await?; + for ((position, _), value) in requests.into_iter().zip(values) { + let value = value.ok_or_else(|| corruption("positional node is missing"))?; + let node = codec::decode_node(&value, reference.bytes, self.paths[layer].as_ref())?; + if node + .0 + .is_some_and(|ordinal| ordinal >= self.descriptor.commit_count) + { + return corrupt("node attribution is out of bounds"); + } + output[position] = Some(node); + } + } + output + .into_iter() + .map(|node| node.ok_or_else(|| corruption("node result is incomplete"))) + .collect() + } + + fn validate_node( + &self, + node: PathStateNodeRef, + parent: Option, + ) -> Result<()> { + let layer = self + .descriptor + .layers + .get(node.layer as usize) + .ok_or_else(|| corruption("node layer is absent"))?; + if node.index >= layer.node_count + || parent.is_some_and(|parent| { + node.layer > parent.layer + || (node.layer == parent.layer && node.index >= parent.index) + }) + { + return corrupt("node reference is out of bounds or cyclic"); + } + Ok(()) + } + + async fn lookup( + &self, + store: &Store, + layer: usize, + root: &PathStatePageRef, + kind: Kind, + keys: &[Vec], + work: &mut Work, + ) -> Result>>> { + if keys.iter().any(|key| key.len() != kind.key_bytes()) { + return corrupt("invalid lookup key"); + } + let mut pending = vec![(root.clone(), (0..keys.len()).collect::>())]; + let mut results = vec![None; keys.len()]; + while !pending.is_empty() { + let wave = std::mem::take(&mut pending); + for (reference, _) in &wave { + self.charge(layer, reference, kind, work)?; + } + let pages = futures_util::stream::iter(wave.into_iter().map( + |(reference, positions)| async move { + let page = self.page(store, layer, &reference, kind).await?; + Ok::<_, crate::error::MetadataError>((page, positions)) + }, + )) + .buffer_unordered(16) + .try_collect::>() + .await?; + for (page, positions) in pages { + match page.as_ref() { + Page::Leaf(rows) => { + for position in positions { + if let Ok(index) = + rows.binary_search_by(|row| row.0.cmp(&keys[position])) + { + results[position] = Some(rows[index].1.clone()); + } + } + } + Page::Branch(children) => { + let mut batches = BTreeMap::>::new(); + for position in positions { + let key = &keys[position]; + let index = children.partition_point(|child| child.last < *key); + if children.get(index).is_some_and(|child| child.first <= *key) { + batches.entry(index).or_default().push(position); + } + } + pending.extend( + batches + .into_iter() + .map(|(index, positions)| (children[index].clone(), positions)), + ); + } + } + } + } + Ok(results) + } + + fn charge( + &self, + layer: usize, + reference: &PathStatePageRef, + kind: Kind, + work: &mut Work, + ) -> Result<()> { + let source_bytes = self.descriptor.layers[layer].bytes; + reference.validate(kind, source_bytes)?; + let key = (layer, reference.offset); + if work.seen.insert(key) { + work.bytes = work + .bytes + .checked_add(u64::from(reference.bytes)) + .ok_or_else(|| corruption("query byte count overflows"))?; + if work.bytes > work.maximum { + return corrupt("path-state query exceeds its byte limit"); + } + } + Ok(()) + } + + // Guards cover local cache work only; origin I/O runs after the guard drops. + fn cache(&self) -> Result> { + self.cache.lock().map_err(|_| { + crate::error::MetadataError::Internal("path-state page cache is poisoned".to_owned()) + }) + } + + async fn page( + &self, + store: &Store, + layer: usize, + reference: &PathStatePageRef, + kind: Kind, + ) -> Result> { + let source_bytes = self.descriptor.layers[layer].bytes; + let key = (layer, reference.offset); + if let Some((previous_kind, previous, page)) = self.cache()?.pages.get(&key) { + if *previous_kind != kind || previous != reference { + return corrupt("cached page routing metadata changed"); + } + return Ok(page.clone()); + } + let bytes = store + .range_get( + &self.paths[layer], + reference.offset..reference.offset + u64::from(reference.bytes), + ) + .await?; + let page = Arc::new(paged::decode( + &bytes, + reference, + kind, + source_bytes, + self.paths[layer].as_ref(), + )?); + let weight = page_weight(reference, &page); + if weight <= CACHE_BYTES { + let mut cache = self.cache()?; + if let Some((previous_kind, previous, existing)) = cache.pages.get(&key) { + if *previous_kind != kind || previous != reference { + return corrupt("concurrent page routing metadata changed"); + } + return Ok(existing.clone()); + } + while cache.bytes.saturating_add(weight) > CACHE_BYTES + || cache.pages.len() >= CACHE_PAGES + { + let Some(oldest) = cache.order.pop_front() else { + break; + }; + if let Some((_, reference, old)) = cache.pages.remove(&oldest) { + cache.bytes = cache.bytes.saturating_sub(page_weight(&reference, &old)); + } + } + cache.bytes += weight; + cache.order.push_back(key); + cache + .pages + .insert(key, (kind, reference.clone(), page.clone())); + } + Ok(page) + } +} + +fn page_weight(reference: &PathStatePageRef, page: &Page) -> usize { + page.resident_bytes() + .saturating_add(std::mem::size_of::()) + .saturating_add(std::mem::size_of::()) + .saturating_add(reference.first.capacity()) + .saturating_add(reference.last.capacity()) + .saturating_add(128) +} + +fn matches_graph(descriptor: &PathStateDescriptor, graph: &SplitCommitGraph) -> bool { + descriptor.generation == graph.descriptor.generation + && descriptor.pack_index_hash == graph.descriptor.pack_index_hash + && descriptor.git_validation_digest == graph.descriptor.git_validation_digest + && descriptor.commit_count == graph.descriptor.commit_count +} + +#[cfg(test)] +#[path = "reader_tests.rs"] +mod tests; diff --git a/crates/crab-metadata/src/path_state/reader_tests.rs b/crates/crab-metadata/src/path_state/reader_tests.rs new file mode 100644 index 000000000..0d3708f4e --- /dev/null +++ b/crates/crab-metadata/src/path_state/reader_tests.rs @@ -0,0 +1,239 @@ +use super::super::{ + DEFAULT_MAX_PATH_STATE_BYTES, PathStateInput, PathStateMutation, append_path_state, + upload_path_state, +}; +use super::*; +use bytes::Bytes; +use object_store::{ObjectStoreExt, memory::InMemory}; +use std::sync::atomic::{AtomicU64, Ordering}; + +async fn fixture() -> ( + Store, + StoreLayout, + SplitCommitGraph, + super::super::PathStateWrite, +) { + let graph = super::super::tests::graph_with_commits(2); + let input = |n| PathStateInput { + oid: [n; 20], + first_parent: (n == 2).then_some([1; 20]), + author: b"author".to_vec(), + author_seconds: i64::from(n), + message: vec![n], + mutations: if n == 1 { + (0..3000) + .map(|index| PathStateMutation { + path: format!("tree/name-{index:04}").into_bytes(), + present: true, + reset: false, + }) + .collect() + } else { + vec![PathStateMutation { + path: b"tree/name-0001".to_vec(), + present: true, + reset: false, + }] + }, + }; + let base = append_path_state(None, &graph, vec![input(1)]).unwrap(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "paged-attribution".to_owned()); + upload_path_state(&store, &layout, &base, DEFAULT_MAX_PATH_STATE_BYTES) + .await + .unwrap(); + let write = append_path_state(Some(base.into_index()), &graph, vec![input(2)]).unwrap(); + upload_path_state(&store, &layout, &write, DEFAULT_MAX_PATH_STATE_BYTES) + .await + .unwrap(); + (store, layout, graph, write) +} + +#[tokio::test] +async fn batched_pages_preserve_old_roots_order_and_repeated_paths() { + let (store, layout, graph, write) = fixture().await; + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let paths = [ + b"tree/name-2999".as_slice(), + b"tree/name-0001", + b"tree/name-2999", + b"tree/name-0000", + ]; + for commit in [[1; 20], [2; 20]] { + let actual = reader + .latest(&store, &graph, &commit, &paths) + .await + .unwrap(); + let expected = write.index.latest(&graph, &commit, &paths).unwrap(); + assert_eq!(actual, expected); + } +} + +#[tokio::test] +async fn query_reads_only_dependency_pages_and_caches_verified_results() { + let (store, layout, graph, write) = fixture().await; + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = store.with_read_byte_observer(Arc::new(move |read| { + observed.fetch_add(read, Ordering::Relaxed); + })); + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + bytes.store(0, Ordering::Relaxed); + let paths = [b"tree/name-0001".as_slice()]; + reader + .latest(&store, &graph, &[2; 20], &paths) + .await + .unwrap(); + let cold = bytes.swap(0, Ordering::Relaxed); + reader + .latest(&store, &graph, &[2; 20], &paths) + .await + .unwrap(); + let full = write + .index + .descriptor + .layers + .iter() + .map(|layer| layer.bytes) + .sum::(); + assert_eq!((cold < full, bytes.load(Ordering::Relaxed)), (true, 0)); +} + +#[tokio::test] +async fn query_rejects_a_rebuilt_graph_with_matching_scalar_identity() { + let (store, layout, mut graph, write) = fixture().await; + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + graph.descriptor.layers[0].hash = "f".repeat(64); + assert!( + reader + .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn required_corrupt_or_missing_pages_never_become_a_lookup_miss() { + let (store, layout, graph, write) = fixture().await; + let reference = &write.index.descriptor.layers[1]; + let path = layout.repo_path(&reference.path); + for corruption in [true, false] { + if corruption { + store + .inner() + .put(&path, Bytes::from(vec![0; reference.bytes as usize]).into()) + .await + .unwrap(); + } else { + store.delete(&path).await.unwrap(); + } + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + assert!( + reader + .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .await + .is_err() + ); + } +} + +#[tokio::test] +async fn cache_identity_includes_the_page_kind() { + let (store, layout, graph, write) = fixture().await; + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let reference = &reader.descriptor.layers[1].pages.records; + reader + .page(&store, 1, reference, Kind::Records) + .await + .unwrap(); + assert!( + reader + .page(&store, 1, reference, Kind::Nodes) + .await + .is_err() + ); +} + +#[tokio::test] +async fn warm_cache_cannot_bypass_the_query_byte_limit() { + let (store, layout, graph, write) = fixture().await; + let mut reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + reader + .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .await + .unwrap(); + reader.max_bytes = reader.descriptor_bytes; + assert!( + reader + .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn page_queries_remain_send_without_holding_cache_guards_across_io() { + fn require_send(future: T) -> T { + future + } + let (store, layout, graph, write) = fixture().await; + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let result = require_send(reader.latest(&store, &graph, &[2; 20], &[b"tree/name-0001"])).await; + assert!(result.is_ok()); +} diff --git a/crates/crab-metadata/src/path_state/storage.rs b/crates/crab-metadata/src/path_state/storage.rs index 238cc46a2..309440638 100644 --- a/crates/crab-metadata/src/path_state/storage.rs +++ b/crates/crab-metadata/src/path_state/storage.rs @@ -21,7 +21,7 @@ pub async fn load_path_state_descriptor( .map(|(descriptor, _)| descriptor) } -async fn load_path_state_descriptor_with_size( +pub(super) async fn load_path_state_descriptor_with_size( store: &Store, layout: &StoreLayout, descriptor_hash: &str, @@ -35,7 +35,11 @@ async fn load_path_state_descriptor_with_size( let descriptor_path = layout.bulk_manifest_path("path-state", descriptor_hash); let expected = decode_hash(descriptor_hash, descriptor_path.as_ref())?; let descriptor_bytes = store - .verify_bounded(&descriptor_path, &expected, max_bytes) + .verify_bounded( + &descriptor_path, + &expected, + max_bytes.min(super::MAX_DESCRIPTOR_BYTES), + ) .await?; let descriptor_bytes_len = descriptor_bytes.len() as u64; let descriptor = serde_json::from_slice(&descriptor_bytes).map_err(|source| { From a29919031f0ca3a7d93e8e33c6a54fbe627d6466 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 11:17:00 -0700 Subject: [PATCH 004/127] fix(git): omit path hints from visibility walks --- crab/src/git/push.rs | 80 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 9 deletions(-) diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index bde57c825..7ad33ce68 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -18290,10 +18290,12 @@ pub(crate) fn enumerate_visibility_difference( maximum: usize, ) -> Result>> { let mut command = std::process::Command::new("git"); + // Visibility needs only object IDs. Optional pathname hints are ambiguous + // and can make the text reader reject otherwise valid raw Git paths. command .arg("--git-dir") .arg(git_dir) - .args(["rev-list", "--objects"]) + .args(["rev-list", "--objects", "--no-object-names"]) .arg(include); if let Some(exclude) = exclude { command.arg("--not").arg(exclude); @@ -18335,14 +18337,7 @@ pub(crate) fn enumerate_visibility_difference( let _ = child.kill(); break; } - let Some(oid) = line.split_whitespace().next() else { - operation_error = Some(CrabError::Internal( - "git rev-list returned an empty visibility row".to_owned(), - )); - let _ = child.kill(); - break; - }; - let oid = match gix_hash::ObjectId::from_hex(oid.as_bytes()) { + let oid = match gix_hash::ObjectId::from_hex(line.as_bytes()) { Ok(oid) => oid, Err(error) => { operation_error = Some(CrabError::Internal(format!( @@ -22479,6 +22474,73 @@ mod tests { drop(fixtures); } + #[test] + fn visibility_and_candidates_preserve_raw_path_reachability() { + let _guard = CleanGitEnvGuard::new(); + let repo = tempfile::tempdir().expect("bare repository"); + let git = |args: &[&str], input: Option<&[u8]>| { + let mut child = Command::new("git") + .args(args) + .current_dir(repo.path()) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn Git"); + if let Some(input) = input { + child.stdin.take().unwrap().write_all(input).unwrap(); + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_owned() + }; + git(&["init", "--bare", "--quiet"], None); + let mut entries = Vec::new(); + let mut expected = Vec::new(); + for path in [ + b"raw-\xff".as_slice(), + b"space name", + b"line\nname", + b"tab\tname", + ] { + let blob = git(&["hash-object", "-w", "--stdin"], Some(path)); + entries.extend_from_slice(format!("100644 blob {blob}\t").as_bytes()); + entries.extend_from_slice(path); + entries.push(0); + expected.push(blob); + } + let tree = git(&["mktree", "-z"], Some(&entries)); + let commit = git(&["commit-tree", &tree], Some(b"raw paths\n")); + let candidates = enumerate_git_object_candidates_blocking( + Some(repo.path().to_path_buf()), + vec![commit.clone()], + Vec::new(), + ) + .unwrap() + .unwrap() + .into_iter() + .map(|oid| oid.to_string()) + .collect::>(); + let visible = enumerate_visibility_difference(repo.path(), &commit, None, 100) + .unwrap() + .unwrap(); + expected.extend([tree, commit]); + expected.sort_unstable(); + assert_eq!((candidates, visible), (expected.clone(), expected)); + } + #[test] fn visibility_edit_enumerates_only_changed_reachability() { let _guard = CleanGitEnvGuard::new(); From 59cddc2a3b9ec896d147bcefa61b4ed0fb9cba84 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 11:44:57 -0700 Subject: [PATCH 005/127] perf(git): read attribution pages lazily --- .../docs/architecture/indexed-remote-reads.md | 41 +++- crates/crab-remote-git/REFERENCE.md | 13 +- crates/crab-remote-git/src/path_state.rs | 124 ++++++----- .../src/repository/tests/snapshot_indexes.rs | 192 +++++++++++++++++- crates/crab-remote-git/src/snapshot.rs | 8 +- crates/crab-remote-git/src/state.rs | 53 ++--- 6 files changed, 351 insertions(+), 80 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 8a284a53a..942296762 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -28,7 +28,7 @@ Xet files and unified local staging are subsequent workstreams. | --- | --- | --- | --- | | 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | | 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Implemented; 23 focused tests, GC retention regression and 1,000/10,000-commit RustFS format qualification passed | -| 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Pending | +| 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Canonical consumer passed 171 owner tests, 60 RustFS scaling samples and raw-path/force-push oracles; projection ingestion and combined restart proof pending | | 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Pending | | 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | | 6 | Indexed historical Xet catalog | Indexed checkpoints and visible deltas; bounded compaction; 10-GiB edits and byte-identical cold historical hydration; protected receive and sibling consumers | Pending | @@ -239,3 +239,42 @@ its descriptor/readiness pointer. No separate page-key scan, new authority objec or old-format reader is introduced. Before GC or readiness accepts an index, its version and descriptor roots must validate; retired derived indexes are rebuilt under existing ownership and CAS rules. + +## Canonical attribution qualification + +The first lazy foreground consumer uses the same captured graph and origin, with +descriptor/page reads admitted by the active operation. Cached metadata never +retains a finished operation's budget or cancellation token. Entry, depth, +logical-object and response work remains charged for warm queries. Missing +required pages are corruption, while absent readiness retains indexing status. +HTTP Cell projection ingestion is still a separate unfinished slice of stage 3. + +RustFS `lazy-attribution-r1` reused the frozen baseline repositories and native +Git oracle. Twenty process-isolated samples per history size passed: 60 cold/warm +attribution comparisons, with complete browse/archive verification also passing +at 10,000 commits. The latter checked 102 entries/file bodies, 1,000 requested +history records and all 50 blame lines. + +| Commits | Cold median | Cold p95 | Maximum path-state decoded bytes | Maximum query backend bytes | Median whole-process RSS | +| --- | --- | --- | --- | --- | --- | +| 100 | 5.197 ms | 5.713 ms | 13,895 | 18,704 | 21,839,872 | +| 1,000 | 4.885 ms | 5.787 ms | 14,425 | 50,741 | 28,426,240 | +| 10,000 | 7.732 ms | 8.305 ms | 77,406 | 105,899 | 107,487,232 | + +Query decode/fetch and latency satisfy the frozen attribution bounds. Process +RSS includes opening and still grows with unrelated history; this does not +establish bounded opening or graph access. Provider HTTP evidence covers each +whole process, separately from its backend query counters. Cloud latency, +projection ingestion and the later history/catalog/lifecycle gates remain open. + +The transition fixture passed both all ten read workloads and complete +browse/archive verification after deletion/recreation, file/tree replacement, +force-push to a side branch and rewind to the original root. It includes raw-byte +names in Git objects, independent of filesystem Unicode support, and checked +300 blame-line origins across six reader runs. Publication first exposed an +existing UTF-8 failure in visibility enumeration; [PR #525](https://github.com/crabbuild/crab/pull/525) +requests OIDs only, matching candidate enumeration. The new regression reproduced +the old failure before the fix, and all 18 visibility tests plus the original live +push passed afterward. These transitions qualify canonical reads; configured +Cell projection, concurrent retained readers and builder restart still need their +own proof. diff --git a/crates/crab-remote-git/REFERENCE.md b/crates/crab-remote-git/REFERENCE.md index 901ee2500..6ad7c85e9 100644 --- a/crates/crab-remote-git/REFERENCE.md +++ b/crates/crab-remote-git/REFERENCE.md @@ -235,8 +235,17 @@ pages do not replay newer commits. A matching complete graph groups bounded raw commit and tree reads for range coalescing without becoming the history authority. All snapshot constructors use this same graph validation when their supplied -metadata names an index. They retain path-state metadata for exact attribution; -a corrupt declared path index, or a missing graph needed to interpret it, +metadata names an index. Attribution captures the path-state descriptor and +graph identity, batches path prefixes, and fetches only required authenticated +pages. Verified page caching is bounded and private to that captured index; +every invocation wraps the same origin store with its current operation's +read admission. The cache retains no operation admission or cancellation token. +Descriptor/page reads obey aggregate request/byte admission, while entry, depth, +logical-object and response limits also apply to warm queries. Required missing +pages fail as corruption rather than becoming indexing readiness or path misses. +A cancelled waiter can abandon shared descriptor initialization; a successful +initializer retains only immutable metadata, so later operations remain usable. +A corrupt declared path index, or a missing graph needed to interpret it, returns `Corrupt(PathState)` instead of fabricating attribution. A journal overlay that changes the Git validation digest discards the base index pointers. Absent or superseded indexes cause no index-origin reads. Graph opening obeys diff --git a/crates/crab-remote-git/src/path_state.rs b/crates/crab-remote-git/src/path_state.rs index 34569d78a..4debb2d44 100644 --- a/crates/crab-remote-git/src/path_state.rs +++ b/crates/crab-remote-git/src/path_state.rs @@ -1,18 +1,20 @@ use std::sync::Arc; use crab_metadata::error::MetadataError; -use crab_metadata::path_state::load_path_state; +use crab_metadata::path_state::PathStateReader; use crab_storage::{StorageError, Store, StoreLayout}; use gix_hash::ObjectId; use tokio_util::sync::CancellationToken; use crate::commit_graph::CommitGraphIndex; -use crate::{CommitSummary, CorruptionStage, Error, GitPath, Result}; +use crate::{ + BudgetDimension, CommitSummary, CorruptionStage, Error, GitPath, OperationContext, Result, +}; /// Validated generation-bound path attribution metadata. #[derive(Debug)] pub(crate) struct PathStateIndex { - index: crab_metadata::path_state::PathStateIndex, + index: PathStateReader, commit_graph: Arc, } @@ -20,72 +22,96 @@ impl PathStateIndex { pub(crate) async fn load( store: &Store, layout: &StoreLayout, - content_hash: Option<&str>, - commit_graph: Option<&Arc>, + content_hash: &str, + commit_graph: &Arc, max_bytes: u64, cancellation: &CancellationToken, runtime_cancellation: &CancellationToken, - ) -> Result> { - let Some(content_hash) = content_hash else { - return Ok(None); - }; - let Some(commit_graph) = commit_graph else { - return Err(Error::Corrupt { - stage: CorruptionStage::PathState, - }); - }; + ) -> Result { let index = tokio::select! { biased; () = cancellation.cancelled() => return Err(Error::Cancelled), () = runtime_cancellation.cancelled() => return Err(Error::Cancelled), - result = load_path_state( + result = PathStateReader::load( store, layout, content_hash, commit_graph.graph(), max_bytes, - ) => match result { - Ok(index) => index, - Err( - MetadataError::CorruptObject { .. } - | MetadataError::Storage { - source: - StorageError::CorruptObject { .. } | StorageError::NotFound { .. }, - }, - ) => { - return Err(Error::Corrupt { - stage: CorruptionStage::PathState, - }); - } - Err(error) => return Err(Error::Metadata(error)), - }, + ) => result.map_err(metadata_error)?, }; - Ok(Some(Self { + Ok(Self { index, commit_graph: Arc::clone(commit_graph), - })) + }) } - pub(crate) fn latest(&self, commit: ObjectId, paths: &[GitPath]) -> Result> { + pub(crate) async fn latest( + &self, + store: &Store, + commit: ObjectId, + paths: &[GitPath], + operation: &OperationContext, + ) -> Result> { + operation.ensure_active()?; let ObjectId::Sha1(commit) = commit else { return Err(Error::UnsupportedObjectFormat); }; + operation + .charge(BudgetDimension::LogicalObjects, paths.len() as u64) + .await?; + operation + .charge(BudgetDimension::Entries, paths.len() as u64) + .await?; + let depth = paths + .iter() + .map(|path| path.components().count()) + .max() + .unwrap_or(0); + operation + .charge(BudgetDimension::Depth, depth as u64) + .await?; let paths = paths.iter().map(GitPath::as_bytes).collect::>(); - self.index - .latest(self.commit_graph.graph(), &commit, &paths) - .map(|summaries| { - summaries - .into_iter() - .map(|summary| CommitSummary { - oid: ObjectId::Sha1(summary.oid), - author: summary.author.into(), - author_seconds: summary.author_seconds, - message: summary.message.into(), - }) - .collect() - }) - .map_err(|_| Error::Corrupt { - stage: CorruptionStage::PathState, - }) + // Keep admission local to this invocation. The cached index must not + // retain a finished operation's budget or cancellation token. + let store = store + .clone() + .with_read_admission(operation.read_admission()); + let summaries = tokio::select! { + biased; + () = operation.cancellation().cancelled() => return Err(Error::Cancelled), + result = self.index.latest(&store, self.commit_graph.graph(), &commit, &paths) => { + result.map_err(metadata_error)? + }, + }; + operation.ensure_active()?; + let mut output = Vec::with_capacity(summaries.len()); + for summary in summaries { + operation + .charge( + BudgetDimension::ResponseBytes, + (summary.author.len() + summary.message.len() + 28) as u64, + ) + .await?; + output.push(CommitSummary { + oid: ObjectId::Sha1(summary.oid), + author: summary.author.into(), + author_seconds: summary.author_seconds, + message: summary.message.into(), + }); + } + Ok(output) + } +} + +fn metadata_error(error: MetadataError) -> Error { + match error { + MetadataError::CorruptObject { .. } + | MetadataError::Storage { + source: StorageError::CorruptObject { .. } | StorageError::NotFound { .. }, + } => Error::Corrupt { + stage: CorruptionStage::PathState, + }, + error => Error::Metadata(error), } } diff --git a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs index 66514724f..7b713ccfe 100644 --- a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs +++ b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs @@ -127,15 +127,28 @@ async fn snapshot_read_indexes_supply_exact_attribution_without_manifest_or_cata .unwrap(); assert!(repository.commit_graph_available()); assert!(repository.path_state_available()); - let index = repository.state.path_state(&cancel).await.unwrap().unwrap(); + let operation = repository + .operation(crate::OperationKind::History, &cancel) + .await + .unwrap(); + let index = repository + .state + .path_state(&operation) + .await + .unwrap() + .unwrap(); let latest = index .latest( + &repository.state.store, ObjectId::Sha1([0x11; 20]), &[crate::GitPath::new(b"README.md".to_vec()).unwrap()], + &operation, ) + .await .unwrap(); assert_eq!(latest[0].oid, ObjectId::Sha1([0x11; 20])); assert_eq!(latest[0].message.as_ref(), b"initial"); + operation.finish(Ok(())).await.unwrap(); } runtime.shutdown().await; } @@ -241,8 +254,13 @@ async fn snapshot_read_indexes_fail_attribution_when_declared_metadata_is_corrup ) .await .unwrap(); + let operation = repository + .operation(crate::OperationKind::History, &cancel) + .await + .unwrap(); + let result = repository.state.path_state(&operation).await.map(|_| ()); assert!(matches!( - repository.state.path_state(&cancel).await, + operation.finish(result).await, Err(Error::Corrupt { stage: crate::CorruptionStage::PathState }) @@ -310,3 +328,173 @@ async fn snapshot_read_indexes_drain_on_cancellation_shutdown_and_deadline() { runtime.shutdown().await; } } + +#[tokio::test] +async fn snapshot_read_indexes_admit_descriptor_pages_and_warm_semantic_results() { + for mode in ["descriptor", "page", "warm"] { + let fixture = open_fixture(1, None).await; + let snapshot = snapshot_with_read_indexes(&fixture).await; + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let repository = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + let paths = [crate::GitPath::new(b"README.md".to_vec()).unwrap()]; + if mode != "descriptor" { + let warm = repository + .operation(crate::OperationKind::History, &cancel) + .await + .unwrap(); + let index = repository.state.path_state(&warm).await.unwrap().unwrap(); + if mode == "warm" { + index + .latest( + &repository.state.store, + ObjectId::Sha1([0x11; 20]), + &paths, + &warm, + ) + .await + .unwrap(); + } + warm.finish(Ok(())).await.unwrap(); + } + let limits = match mode { + "descriptor" => OperationLimits { + max_fetched_bytes: 1, + ..Default::default() + }, + "page" => OperationLimits { + max_storage_requests: 1, + ..Default::default() + }, + _ => OperationLimits { + max_response_bytes: 1, + ..Default::default() + }, + }; + let operation = crate::OperationContext::open( + repository.state.clone(), + crate::OperationKind::History, + &cancel, + limits, + ) + .await + .unwrap(); + let result = async { + let index = repository.state.path_state(&operation).await?.unwrap(); + index + .latest( + &repository.state.store, + ObjectId::Sha1([0x11; 20]), + &paths, + &operation, + ) + .await + } + .await; + let error = operation.finish(result).await.unwrap_err(); + assert!( + match mode { + "warm" => matches!( + error, + Error::LimitExceeded { + limit: "response bytes", + .. + } + ), + _ => admission_rejected(&error), + }, + "{mode}: {error:?}" + ); + runtime.shutdown().await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { + for mode in ["caller", "shutdown", "deadline"] { + let fixture = open_fixture(1, None).await; + let snapshot = snapshot_with_read_indexes(&fixture).await; + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let repository = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + fixture + .backend + .block_path_containing("metadata/path-state/layers/"); + let worker_cancel = cancel.clone(); + let worker = tokio::spawn(async move { + let limits = OperationLimits { + max_duration: if mode == "deadline" { + Duration::from_millis(50) + } else { + Duration::from_secs(60) + }, + ..Default::default() + }; + let operation = crate::OperationContext::open( + repository.state.clone(), + crate::OperationKind::History, + &worker_cancel, + limits, + ) + .await + .unwrap(); + let result = async { + let index = repository.state.path_state(&operation).await?.unwrap(); + index + .latest( + &repository.state.store, + ObjectId::Sha1([0x11; 20]), + &[crate::GitPath::new(b"README.md".to_vec()).unwrap()], + &operation, + ) + .await + } + .await; + operation.finish(result).await + }); + tokio::time::timeout( + Duration::from_secs(2), + fixture.backend.request_started.notified(), + ) + .await + .unwrap(); + if mode == "caller" { + cancel.cancel(); + } + if mode == "shutdown" { + tokio::time::timeout(Duration::from_secs(2), runtime.shutdown()) + .await + .unwrap(); + } + let result = tokio::time::timeout(Duration::from_secs(2), worker) + .await + .unwrap() + .unwrap(); + assert!( + match mode { + "deadline" => matches!(result, Err(Error::Timeout { .. })), + _ => matches!(result, Err(Error::Cancelled)), + }, + "{mode}: {result:?}" + ); + runtime.shutdown().await; + } +} diff --git a/crates/crab-remote-git/src/snapshot.rs b/crates/crab-remote-git/src/snapshot.rs index 25969a6b0..8e656f1bb 100644 --- a/crates/crab-remote-git/src/snapshot.rs +++ b/crates/crab-remote-git/src/snapshot.rs @@ -1163,7 +1163,7 @@ impl RemoteGitSnapshot { /// Find the newest first-parent commit that changed each listed directory entry. /// /// Entries must be immediate children returned by this snapshot. The result - /// preserves their order and shares one bounded history walk across all entries. + /// preserves their order and batches authenticated path-state page reads. pub async fn latest_directory_entry_commits( &self, directory: &GitPath, @@ -1185,12 +1185,14 @@ impl RemoteGitSnapshot { } } - if let Some(index) = self.repository.path_state(operation.cancellation()).await? { + if let Some(index) = self.repository.path_state(operation).await? { let paths = entries .iter() .map(|entry| entry.path.clone()) .collect::>(); - return index.latest(self.commit_oid, &paths); + return index + .latest(&self.repository.store, self.commit_oid, &paths, operation) + .await; } Err(Error::PathStateIndexing { generation: self.repository.generation, diff --git a/crates/crab-remote-git/src/state.rs b/crates/crab-remote-git/src/state.rs index a601b519d..6759e5af5 100644 --- a/crates/crab-remote-git/src/state.rs +++ b/crates/crab-remote-git/src/state.rs @@ -8,14 +8,13 @@ use crab_metadata::shallow_closure::ShallowClosureDescriptor; use crab_storage::{Store, StoreLayout}; use crab_xet::hash::MerkleHash; use tokio::sync::OnceCell; -use tokio_util::sync::CancellationToken; use crate::commit_graph::CommitGraphIndex; use crate::path_state::PathStateIndex; use crate::reader::RemoteGitReader; use crate::{ - CorruptionStage, Error, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, - RepositoryRefs, Result, + CorruptionStage, Error, OperationContext, RemoteGitRuntime, RepositoryIdentity, + RepositoryOptions, RepositoryRefs, Result, }; /// Immutable repository facts shared by handles and snapshots. @@ -54,31 +53,39 @@ impl RepositoryState { pub(crate) async fn path_state( &self, - cancellation: &CancellationToken, + operation: &OperationContext, ) -> Result>> { let Some(hash) = self.path_state_hash.as_deref() else { return Ok(None); }; + operation.ensure_active()?; + let graph = self.commit_graph.as_ref().ok_or(Error::Corrupt { + stage: CorruptionStage::PathState, + })?; + let store = self + .store + .clone() + .with_read_admission(operation.read_admission()); let runtime_cancellation = self.runtime.background_cancellation(); - let index = self - .path_state - .get_or_try_init(|| async { - PathStateIndex::load( - &self.store, - &self.layout, - Some(hash), - self.commit_graph.as_ref(), - self.options.object_limits().max_path_state_bytes, - cancellation, - &runtime_cancellation, - ) - .await? - .map(Arc::new) - .ok_or(Error::Corrupt { - stage: CorruptionStage::PathState, - }) - }) - .await?; + let load = self.path_state.get_or_try_init(|| async { + PathStateIndex::load( + &store, + &self.layout, + hash, + graph, + self.options.object_limits().max_path_state_bytes, + operation.cancellation(), + &runtime_cancellation, + ) + .await + .map(Arc::new) + }); + let index = tokio::select! { + biased; + () = operation.cancellation().cancelled() => return Err(Error::Cancelled), + () = runtime_cancellation.cancelled() => return Err(Error::Cancelled), + result = load => result?, + }; Ok(Some(index)) } } From 909ef2feb568256c3dc72b256886d07ac29a6bb2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 13:45:48 -0700 Subject: [PATCH 006/127] perf(http): stream attribution projection Import authenticated path-state records, nodes and edges without materializing the complete index or rereading retained Git commits. Keep projection actors runtime-owned so cold state reads cannot wait for background ingestion. Scheduler cleanup releases only unused activations; migration and shutdown preserve explicit drain barriers. Append schema migration 3 without changing earlier SQL digests. Preserve summaries and epoch membership; remove unconsumed Git data. --- .../docs/architecture/indexed-remote-reads.md | 51 +- crates/crab-http-server/README.md | 12 +- crates/crab-http-server/REFERENCE.md | 37 +- crates/crab-http-server/src/cells.rs | 98 +++- .../0003_git_attribution_summaries.sql | 10 + .../src/cells/repository/projection.rs | 39 +- crates/crab-http-server/src/cells/router.rs | 29 +- .../crab-http-server/src/cells/scheduler.rs | 5 +- .../src/maintenance_tests/browse.rs | 159 ++++++ crates/crab-http-server/src/projection.rs | 418 ++++++--------- crates/crab-http-server/tests/verify_live.py | 67 ++- crates/crab-metadata/README.md | 18 +- crates/crab-metadata/src/path_state.rs | 2 +- crates/crab-metadata/src/path_state/reader.rs | 4 + .../src/path_state/reader_tests.rs | 2 +- crates/crab-metadata/src/path_state/scan.rs | 479 ++++++++++++++++++ 16 files changed, 1061 insertions(+), 369 deletions(-) create mode 100644 crates/crab-http-server/src/cells/migrations/0003_git_attribution_summaries.sql create mode 100644 crates/crab-metadata/src/path_state/scan.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 942296762..c6705a208 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -28,7 +28,7 @@ Xet files and unified local staging are subsequent workstreams. | --- | --- | --- | --- | | 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | | 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Implemented; 23 focused tests, GC retention regression and 1,000/10,000-commit RustFS format qualification passed | -| 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Canonical consumer passed 171 owner tests, 60 RustFS scaling samples and raw-path/force-push oracles; projection ingestion and combined restart proof pending | +| 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Canonical scaling/transitions passed; streaming Cell ingestion, projected native Git oracle and two process starts passed on RustFS; broader cloud/lifecycle qualification remains open | | 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Pending | | 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | | 6 | Indexed historical Xet catalog | Indexed checkpoints and visible deltas; bounded compaction; 10-GiB edits and byte-identical cold historical hydration; protected receive and sibling consumers | Pending | @@ -247,7 +247,47 @@ descriptor/page reads admitted by the active operation. Cached metadata never retains a finished operation's budget or cancellation token. Entry, depth, logical-object and response work remains charged for warm queries. Missing required pages are corruption, while absent readiness retains indexing status. -HTTP Cell projection ingestion is still a separate unfinished slice of stage 3. +HTTP Cell projection ingestion uses the streaming maintenance consumer below. + +Projection ingestion now has a streaming implementation. Its full +maintenance scan verifies layer hashes, authenticated pages and physical coverage +before EOF, retaining a current-layer dictionary/audit rather than the complete +index. The importer consumes verified summaries instead of resolving retained +ordinals through foreground Git visibility. An append-only application migration +preserves summaries and epoch membership while removing unconsumed commit fields. +Focused proof: 26 path-state tests, 30 HTTP maintenance tests, 17 projection tests +and 73 Cell/router/scheduler/release tests pass; three credential-dependent Cell +tests remain ignored. These slices overlap and their counts must not be summed. +A cancellation +negative control reproduced a skipped record in the old scan cursor; the fixed +cursor advances only after page I/O succeeds. Rewind proof confirms retained +graph records remain unavailable to foreground attribution after force-push. + +Live `stream-projection-r1` published the 1,000-commit fixture through native HTTP +receive into a dedicated RustFS bucket. An initial attribution response succeeded, +but that alone does not prove the configured Cell projection was ready: canonical +attribution is an existing readiness path. Process restart qualification failed +twice at the verifier's attribution timeout. Instrumentation showed the state +query finished and then waited to drain a cold activation while background +ingestion held the Cell operation guard. The second run's metrics showed 30,120 +attribution rows and 192 summaries submitted, zero completed reconciliations and +projection readiness zero. The fix gives projection work the same runtime-owned +Cell lifetime as ordinary collaboration requests. Scheduler cleanup releases a +cold activation only when exclusive operation access is immediately available; +an active borrower leaves ownership with the bounded runtime. Explicit migration +draining and node shutdown retain their close/publication barriers. + +The subsequent single-repository run completed one reconciliation with all 1,000 +summaries and 105,121 attribution rows. Its selected-source readiness gauge was +one before and after independent native Git verification of 102 attribution +entries, author/time/message fields, file bytes, 50 blame lines and 30 sampled +historical commits. One hundred foreground attribution/metrics polls continued +during ingestion; the largest combined poll took 2.084 seconds. This includes a +private metrics subprocess and is not a standalone HTTP latency percentile. +`process-proof-r3` then passed the same native oracle on two process starts, with +clean shutdowns. Failed r1/r2 receipts remain preserved. These runs use a local +debug binary, not a qualified release image; broader CI and cloud gates remain +separate from this configured-projection correctness proof. RustFS `lazy-attribution-r1` reused the frozen baseline repositories and native Git oracle. Twenty process-isolated samples per history size passed: 60 cold/warm @@ -265,7 +305,7 @@ Query decode/fetch and latency satisfy the frozen attribution bounds. Process RSS includes opening and still grows with unrelated history; this does not establish bounded opening or graph access. Provider HTTP evidence covers each whole process, separately from its backend query counters. Cloud latency, -projection ingestion and the later history/catalog/lifecycle gates remain open. +opening and the later history/catalog/lifecycle gates remain open. The transition fixture passed both all ten read workloads and complete browse/archive verification after deletion/recreation, file/tree replacement, @@ -276,5 +316,6 @@ existing UTF-8 failure in visibility enumeration; [PR #525](https://github.com/c requests OIDs only, matching candidate enumeration. The new regression reproduced the old failure before the fix, and all 18 visibility tests plus the original live push passed afterward. These transitions qualify canonical reads; configured -Cell projection, concurrent retained readers and builder restart still need their -own proof. +Cell projection has the separate ingestion/restart proof above and owner tests +cover the same rewind visibility boundary. Concurrent retained readers during +compaction/GC and broader interrupted-build qualification remain later gates. diff --git a/crates/crab-http-server/README.md b/crates/crab-http-server/README.md index b762b3742..841104bfe 100644 --- a/crates/crab-http-server/README.md +++ b/crates/crab-http-server/README.md @@ -28,7 +28,17 @@ Immutable descriptors are verified on identity changes and at the existing hourly scrub boundary; explicit repair invalidates the cached proof. One worker per repository retains completion only for the exact verified source, so idle polling does not reimport the same Cell epoch. Tree APIs use the verified remote -reader; reconciliation imports only consumed ref, commit and attribution rows. +reader; reconciliation imports only consumed ref, summary and attribution rows. +It streams authenticated path-state pages into provisional Cell epochs without +rereading raw Git commits. Retained summaries cannot authorize a revision: the +foreground reader still checks current ref visibility before querying the Cell. +Schema migration 3 preserves existing summaries and epoch membership while +removing unconsumed Git fields. Existing Cells upgrade through release maintenance; +their collaboration data must not be discarded to rebuild browse metadata. +Projection reads and ingestion borrow runtime-owned Cells like collaboration +requests. Cold read completion does not drain the Cell or wait for ingestion. +Scheduler cleanup releases an unused cold activation only when no other operation +holds it; runtime budgets, pressure eviction and node shutdown own retained Cells. The current unshipped path-state codec interns raw components per layer and rejects aggregate writes above the reader budget before publishing any layers. The full Next.js/RustFS idle and mixed-load qualification remains in progress. diff --git a/crates/crab-http-server/REFERENCE.md b/crates/crab-http-server/REFERENCE.md index a74853a64..de9bbb6b2 100644 --- a/crates/crab-http-server/REFERENCE.md +++ b/crates/crab-http-server/REFERENCE.md @@ -759,16 +759,33 @@ requests and source changes reopen maintenance; unchanged completed epochs are not repeatedly imported between scrubs. Cell reconciliation does not preload tree rows: tree/blob responses continue through the verified remote Git reader. Attribution batches obey both row and serialized-byte limits, preserving raw -path bytes. The current path-state codec stores each raw component once per -layer and checks all retained layer bytes against the reader budget before any -write. Its encoded size improvement and full Next.js idle/load gates require -qualification; focused fixed-ref tests do not prove production throughput. - -This changes unshipped derived-index encodings and the application migration -digest. Existing pre-release application Cells need a deliberate rebuild or -operator migration under a selected release descriptor. Rebuilding application -Cells loses collaboration data; never perform that implicitly. Qualification -uses fresh application Cells and preserves canonical Git objects and refs. +path bytes. Reconciliation streams authenticated path-state nodes, edges and +summaries in dependency order, without assembling the complete path-state index +or reading raw Git commits. Each layer's complete hash and physical page closure +must validate before the scan finishes. Partial ingestion remains a provisional +epoch; only successful EOF and a matching source permit promotion. Maintenance +retains the current layer's component dictionary/page audit and a bounded page +cache, and performs a full streaming hash pass before its page traversal. +The commit graph still loads eagerly; bounded opening is a separate milestone. + +Projection summaries include retained graph ordinals, including commits removed +by force-push. Foreground revision resolution checks visibility before consulting +those rows. Their presence never grants access to retained history. + +Projection reads and ingestion borrow the runtime-owned Cell under the same +lifetime policy as collaboration requests. Completing a cold state query must +not drain the actor and wait for a concurrent importer. Scheduler cleanup takes +exclusive operation access only when it is immediately available; if another +operation joined the activation, the bounded runtime retains ownership. Runtime +pressure eviction and node shutdown close retained actors. Explicit migration +draining still waits for exclusive access and completes its publication barrier. + +Schema migration 3 removes unconsumed Git commit fields and parent rows while +preserving summary bytes and epoch membership. Earlier migration SQL and digests +remain unchanged. Upgrade existing application Cells through release maintenance +under the selected descriptor; do not rebuild Cells and lose collaboration data. +The full Next.js idle/load gates still require qualification; focused projection +tests do not establish production throughput. Root commits compare against an empty tree. Path history follows the exact first-parent path. Commit history with `base` instead returns commits reachable from `rev` but not from `base`, across every parent. diff --git a/crates/crab-http-server/src/cells.rs b/crates/crab-http-server/src/cells.rs index 6366d296c..c4a57fbbc 100644 --- a/crates/crab-http-server/src/cells.rs +++ b/crates/crab-http-server/src/cells.rs @@ -65,6 +65,8 @@ pub(crate) use scheduler::{RepositoryCellScheduler, SchedulerStatus}; const REPOSITORY_MIGRATION: &str = include_str!("cells/migrations/0001_repository_identity.sql"); const REPOSITORY_MIGRATION_V2: &str = include_str!("cells/migrations/0002_pull_review_threads.sql"); +const REPOSITORY_MIGRATION_V3: &str = + include_str!("cells/migrations/0003_git_attribution_summaries.sql"); const REPOSITORY_MAX_DATABASE_BYTES: u64 = 5 * 1024 * 1024 * 1024; const REPOSITORY_MAX_CAPTURE_BYTES: u64 = 64 * 1024 * 1024; const REPOSITORY_MAX_FILE_BYTES: u64 = 6 * 1024 * 1024 * 1024; @@ -94,7 +96,8 @@ pub(crate) fn initialize_repository_schema( transaction: &rusqlite::Transaction<'_>, ) -> rusqlite::Result<()> { transaction.execute_batch(REPOSITORY_MIGRATION)?; - transaction.execute_batch(REPOSITORY_MIGRATION_V2) + transaction.execute_batch(REPOSITORY_MIGRATION_V2)?; + transaction.execute_batch(REPOSITORY_MIGRATION_V3) } const MAX_LIVE_NODES: usize = 10_000; @@ -297,7 +300,7 @@ pub(crate) fn compiled_application() -> cellule_runtime::Result &'static ModuleDescriptor { - static MIGRATIONS: OnceLock<[MigrationDescriptor; 2]> = OnceLock::new(); + static MIGRATIONS: OnceLock<[MigrationDescriptor; 3]> = OnceLock::new(); static DESCRIPTOR: OnceLock = OnceLock::new(); let migrations = MIGRATIONS.get_or_init(|| { [ @@ -2098,6 +2101,11 @@ fn repository_descriptor() -> &'static ModuleDescriptor { sql: REPOSITORY_MIGRATION_V2, digest: digest(REPOSITORY_MIGRATION_V2.as_bytes()), }, + MigrationDescriptor { + version: 3, + sql: REPOSITORY_MIGRATION_V3, + digest: digest(REPOSITORY_MIGRATION_V3.as_bytes()), + }, ] }); DESCRIPTOR.get_or_init(|| ModuleDescriptor { @@ -2105,7 +2113,7 @@ fn repository_descriptor() -> &'static ModuleDescriptor { source_digest: repository_source_digest(), retained_codes: &[], schema_min: 1, - schema_max: 2, + schema_max: 3, migrations, commands: REPOSITORY_COMMANDS, queries: REPOSITORY_QUERIES, @@ -2127,6 +2135,7 @@ fn repository_source_digest() -> Digest { hasher.update(b"crab.http.repository.module.v2\0"); hasher.update(REPOSITORY_MIGRATION.as_bytes()); hasher.update(REPOSITORY_MIGRATION_V2.as_bytes()); + hasher.update(REPOSITORY_MIGRATION_V3.as_bytes()); hasher.update(include_bytes!("cells/repository.rs")); hasher.update(include_bytes!("cells/repository/codec.rs")); hasher.update(include_bytes!("cells/repository/checks.rs")); @@ -2142,6 +2151,7 @@ fn repository_source_digest() -> Digest { hasher.update(include_bytes!("cells/repository/releases_codec.rs")); hasher.update(include_bytes!("cells/repository/settings.rs")); hasher.update(include_bytes!("cells/repository/settings_codec.rs")); + hasher.update(include_bytes!("cells/repository/projection.rs")); Digest::from_bytes(*hasher.finalize().as_bytes()) } @@ -2150,7 +2160,7 @@ const fn operation(id: u32, input_limit: u32, output_limit: u32) -> OperationDes id, codec_version: 1, schema_min: 1, - schema_max: 2, + schema_max: 3, input_limit, output_limit, } @@ -2161,7 +2171,7 @@ const fn operation_v2(id: u32, input_limit: u32, output_limit: u32) -> Operation id, codec_version: 1, schema_min: 2, - schema_max: 2, + schema_max: 3, input_limit, output_limit, } @@ -2245,6 +2255,68 @@ mod tests { const ROLLOVER_PREDECESSOR: Digest = Digest::from_bytes([32; 32]); const ROLLOVER_SQL: &str = "CREATE TABLE rollover(value BLOB NOT NULL)"; + #[test] + fn summary_migration_preserves_ready_epoch_membership_and_summary_bytes() { + let connection = rusqlite::Connection::open_in_memory().unwrap(); + connection + .execute_batch("PRAGMA foreign_keys = ON") + .unwrap(); + connection.execute_batch(REPOSITORY_MIGRATION).unwrap(); + connection.execute_batch(REPOSITORY_MIGRATION_V2).unwrap(); + connection.execute_batch( + "INSERT INTO git_projection_epochs(epoch_id, state, source_token, manifest_generation, manifest_etag, journal_state_digest, pack_index_hash, git_validation_digest, head_ref, started_at_ms) VALUES (1, 'ready', 'captured-source', 0, 'etag', 'journal', 'packs', 'validation', X'6d61696e', 1); UPDATE git_projection_state SET ready_epoch = 1 WHERE singleton = 1;" + ).unwrap(); + let oid = [1_u8; 20]; + connection + .execute( + "INSERT INTO git_commits VALUES (?1, ?2, ?3, X'', 9, 0, X'', X'', 9, 0, ?4, 0, 6)", + rusqlite::params![ + oid.as_slice(), + [2_u8; 20].as_slice(), + b"Author".as_slice(), + b"change".as_slice() + ], + ) + .unwrap(); + connection + .execute( + "INSERT INTO git_epoch_commits VALUES (1, 0, ?1)", + [oid.as_slice()], + ) + .unwrap(); + connection + .execute( + "INSERT INTO git_commit_parents VALUES (?1, 0, ?2)", + rusqlite::params![oid.as_slice(), [3_u8; 20].as_slice()], + ) + .unwrap(); + connection.execute_batch(REPOSITORY_MIGRATION_V3).unwrap(); + let actual = connection.query_row( + "SELECT ec.commit_oid, c.author_name, c.author_time, c.message_preview, e.source_token FROM git_projection_state s JOIN git_projection_epochs e ON e.epoch_id = s.ready_epoch JOIN git_epoch_commits ec ON ec.epoch_id = e.epoch_id JOIN git_commits c ON c.oid = ec.commit_oid", + [], + |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?, row.get::<_, i64>(2)?, row.get::<_, Vec>(3)?, row.get::<_, String>(4)?)), + ).unwrap(); + let violations = connection + .prepare("PRAGMA foreign_key_check") + .unwrap() + .query_map([], |_| Ok(())) + .unwrap() + .count(); + assert_eq!( + (actual, violations), + ( + ( + oid.to_vec(), + b"Author".to_vec(), + 9, + b"change".to_vec(), + "captured-source".to_owned() + ), + 0 + ) + ); + } + #[test] fn node_session_parser_accepts_only_nonzero_canonical_hex() { assert_eq!( @@ -2396,10 +2468,15 @@ mod tests { assert_eq!(descriptor["modules"][0]["name"], "repository"); assert_eq!( descriptor["modules"][0]["code"], - "649bd14cf06eb91169510e3828761059fe8b21164cac3044110b2c5fd2160f95" + status_hex( + first + .module_code(RepositoryModule::NAME) + .unwrap() + .as_bytes() + ) ); assert_eq!(descriptor["modules"][0]["schema_min"], 1); - assert_eq!(descriptor["modules"][0]["schema_max"], 2); + assert_eq!(descriptor["modules"][0]["schema_max"], 3); let operation_ids = |kind| { descriptor["modules"][0][kind] .as_array() @@ -3787,7 +3864,10 @@ mod tests { &target, CatalogRole::Application, registry.module_code(RepositoryModule::NAME).unwrap(), - 2, + registry + .current_cell_version(REPOSITORY_NAMESPACE, CatalogRole::Application) + .unwrap() + .1, ) .unwrap(); let proof = releases diff --git a/crates/crab-http-server/src/cells/migrations/0003_git_attribution_summaries.sql b/crates/crab-http-server/src/cells/migrations/0003_git_attribution_summaries.sql new file mode 100644 index 000000000..b342d8921 --- /dev/null +++ b/crates/crab-http-server/src/cells/migrations/0003_git_attribution_summaries.sql @@ -0,0 +1,10 @@ +DROP TABLE git_commit_parents; +ALTER TABLE git_commits DROP COLUMN tree_oid; +ALTER TABLE git_commits DROP COLUMN author_email; +ALTER TABLE git_commits DROP COLUMN author_tz_offset_seconds; +ALTER TABLE git_commits DROP COLUMN committer_name; +ALTER TABLE git_commits DROP COLUMN committer_email; +ALTER TABLE git_commits DROP COLUMN committer_time; +ALTER TABLE git_commits DROP COLUMN committer_tz_offset_seconds; +ALTER TABLE git_commits DROP COLUMN message_truncated; +ALTER TABLE git_commits DROP COLUMN encoded_bytes; diff --git a/crates/crab-http-server/src/cells/repository/projection.rs b/crates/crab-http-server/src/cells/repository/projection.rs index b1cc144b3..71a79a778 100644 --- a/crates/crab-http-server/src/cells/repository/projection.rs +++ b/crates/crab-http-server/src/cells/repository/projection.rs @@ -157,19 +157,9 @@ pub(crate) struct RefRow { pub(crate) struct CommitRow { pub ordinal: u32, pub oid: Vec, - pub tree_oid: Vec, - pub parents: Vec>, pub author_name: Vec, - pub author_email: Vec, pub author_time: i64, - pub author_tz_offset_seconds: i32, - pub committer_name: Vec, - pub committer_email: Vec, - pub committer_time: i64, - pub committer_tz_offset_seconds: i32, pub message_preview: Vec, - pub message_truncated: bool, - pub encoded_bytes: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -662,24 +652,15 @@ fn apply_commits( epoch_id: u64, rows: &[CommitRow], ) -> cellule_runtime::Result<()> { - let mut statements = Vec::with_capacity(rows.len().saturating_mul(3)); + let mut statements = Vec::with_capacity(rows.len().saturating_mul(2)); for row in rows { statements.push(statement( - "INSERT OR IGNORE INTO git_commits(oid, tree_oid, author_name, author_email, author_time, author_tz_offset_seconds, committer_name, committer_email, committer_time, committer_tz_offset_seconds, message_preview, message_truncated, encoded_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT OR IGNORE INTO git_commits(oid, author_name, author_time, message_preview) VALUES (?, ?, ?, ?)", vec![ SqlValue::Blob(row.oid.clone()), - SqlValue::Blob(row.tree_oid.clone()), SqlValue::Blob(row.author_name.clone()), - SqlValue::Blob(row.author_email.clone()), SqlValue::Integer(row.author_time), - SqlValue::Integer(i64::from(row.author_tz_offset_seconds)), - SqlValue::Blob(row.committer_name.clone()), - SqlValue::Blob(row.committer_email.clone()), - SqlValue::Integer(row.committer_time), - SqlValue::Integer(i64::from(row.committer_tz_offset_seconds)), SqlValue::Blob(row.message_preview.clone()), - SqlValue::Integer(i64::from(u8::from(row.message_truncated))), - integer(row.encoded_bytes)?, ], )); statements.push(statement( @@ -690,18 +671,6 @@ fn apply_commits( SqlValue::Blob(row.oid.clone()), ], )); - for (index, parent) in row.parents.iter().enumerate() { - statements.push(statement( - "INSERT OR REPLACE INTO git_commit_parents(commit_oid, parent_index, parent_oid) VALUES (?, ?, ?)", - vec![ - SqlValue::Blob(row.oid.clone()), - integer(u64::try_from(index).map_err(|_| { - cellule_runtime::Error::Command("projection parent index overflowed") - })?)?, - SqlValue::Blob(parent.clone()), - ], - )); - } } execute_statements(context, statements)?; Ok(()) @@ -888,10 +857,6 @@ fn collect(context: &CommandContext<'_, '_>) -> cellule_runtime::Result<()> { ), vec![], ), - statement( - "DELETE FROM git_commit_parents WHERE commit_oid IN (SELECT commit_oid FROM git_commit_parents WHERE commit_oid NOT IN (SELECT commit_oid FROM git_epoch_commits) LIMIT 256)", - vec![], - ), statement( "DELETE FROM git_commits WHERE oid IN (SELECT oid FROM git_commits WHERE oid NOT IN (SELECT commit_oid FROM git_epoch_commits) LIMIT 256)", vec![], diff --git a/crates/crab-http-server/src/cells/router.rs b/crates/crab-http-server/src/cells/router.rs index b1516bd14..18de6b2d6 100644 --- a/crates/crab-http-server/src/cells/router.rs +++ b/crates/crab-http-server/src/cells/router.rs @@ -431,18 +431,18 @@ impl RepositoryCellRouter { .await } - pub(crate) async fn route_projection( - &self, - repository: Uuid, - ) -> crate::Result { + pub(crate) async fn route_projection(&self, repository: Uuid) -> crate::Result { let target = CellTarget::new( self.identity.tenant(), self.identity.application(), REPOSITORY_NAMESPACE, repository.as_bytes(), )?; + // Like collaboration requests, browse work borrows a runtime-owned + // Cell. Draining a cold read here would wait for concurrent ingestion. self.route_runtime(target, self.runtime_principal(&["repository.projection"])) .await + .map(|scheduled| scheduled.cell) } pub(crate) async fn route_runtime( @@ -1186,10 +1186,29 @@ impl RepositoryCellRouter { } pub(crate) async fn drain_local_target(&self, target: &CellTarget) -> crate::Result<()> { - let _operation: OwnedRwLockWriteGuard<()> = + let operation: OwnedRwLockWriteGuard<()> = Arc::clone(&self.operation[activation_shard(target)]) .write_owned() .await; + self.drain_local_target_locked(target, operation).await + } + + pub(crate) async fn drain_unused_local_target(&self, target: &CellTarget) -> crate::Result<()> { + // A cold scheduler activation may have gained foreground or importer + // borrowers. Leave that Cell with the bounded runtime instead of + // queuing a writer that blocks every subsequent reader until they exit. + let Ok(operation) = Arc::clone(&self.operation[activation_shard(target)]).try_write_owned() + else { + return Ok(()); + }; + self.drain_local_target_locked(target, operation).await + } + + async fn drain_local_target_locked( + &self, + target: &CellTarget, + _operation: OwnedRwLockWriteGuard<()>, + ) -> crate::Result<()> { let proof = self .catalog .lookup(target.cell_id()) diff --git a/crates/crab-http-server/src/cells/scheduler.rs b/crates/crab-http-server/src/cells/scheduler.rs index 7a455f1f2..ef5897495 100644 --- a/crates/crab-http-server/src/cells/scheduler.rs +++ b/crates/crab-http-server/src/cells/scheduler.rs @@ -751,7 +751,8 @@ impl RepositoryCellScheduler { } else { drop(cell); } - if release_after && let Err(error) = router.drain_local_target(&target).await { + if release_after && let Err(error) = router.drain_unused_local_target(&target).await + { tracing::warn!(error = %error, "Workflow scheduler Cell release failed"); } drop(activity_cell); @@ -797,7 +798,7 @@ impl RepositoryCellScheduler { let target = cell.target.clone(); drop(cell); if release_after { - self.router.drain_local_target(&target).await + self.router.drain_unused_local_target(&target).await } else { Ok(()) } diff --git a/crates/crab-http-server/src/maintenance_tests/browse.rs b/crates/crab-http-server/src/maintenance_tests/browse.rs index c7d722234..23417c15f 100644 --- a/crates/crab-http-server/src/maintenance_tests/browse.rs +++ b/crates/crab-http-server/src/maintenance_tests/browse.rs @@ -2,6 +2,165 @@ use super::*; use crab_metadata::capsule_protocol::{BrowseIndexes, load_browse_indexes}; use crab_remote_git::OperationKind; +#[tokio::test] +async fn projection_reads_reuse_the_runtime_cell_during_background_ingestion() { + let server = fixture().await; + let repo = repository(&server); + let router = server.repository_cells.as_ref().unwrap(); + crate::projection::state(router, repo.id).await.unwrap(); + // A read-only cold activation is owned but not yet Residency::Resident. + // Count retained actors rather than using that narrower routing probe. + let retained = router.runtime().unreleased_cell_count().await.unwrap(); + let importer = router.route_projection(repo.id).await.unwrap(); + let state = tokio::time::timeout( + Duration::from_secs(1), + crate::projection::state(router, repo.id), + ) + .await; + drop(importer); + close(&server).await; + assert_eq!((retained, matches!(state, Ok(Ok(_)))), (1, true)); +} + +#[tokio::test] +async fn scheduler_cleanup_preserves_borrowed_projection_and_releases_unused_cell() { + let server = fixture().await; + let repo = repository(&server); + let router = server.repository_cells.as_ref().unwrap(); + let importer = router.route_projection(repo.id).await.unwrap(); + let target = importer.target.clone(); + let release = tokio::time::timeout( + Duration::from_secs(1), + router.drain_unused_local_target(&target), + ) + .await; + let state = tokio::time::timeout( + Duration::from_secs(1), + crate::projection::state(router, repo.id), + ) + .await; + drop(importer); + router.drain_unused_local_target(&target).await.unwrap(); + let retained = router.runtime().unreleased_cell_count().await.unwrap(); + close(&server).await; + assert_eq!( + ( + matches!(release, Ok(Ok(()))), + matches!(state, Ok(Ok(_))), + retained + ), + (true, true, 0) + ); +} + +#[tokio::test] +async fn projected_attribution_survives_force_push_rewind_with_retained_graph_records() { + let server = fixture().await; + let repo = repository(&server); + let (history, _) = crate::test_git::merge_history_with_nested_tag(); + let mut root = crab_write::capsule_protocol::open_root(&repo.layout) + .await + .unwrap(); + let mut old = None; + let mut responses = Vec::new(); + for index in [3, 0] { + let published = publish_next( + &repo, + root, + old, + history.oids[index].clone(), + (index == 3).then(|| history.pack.clone()), + ) + .await; + root = published.0; + old = Some(published.1); + let result = crate::maintenance::run( + repo.layout.clone(), + Arc::clone(&server.maintenance_admission), + None, + server.cancellation.clone(), + crate::maintenance::Pass::BackgroundMaintenance, + Some(crate::maintenance::ProjectionContext { + repository_id: repo.id, + router: server.repository_cells.as_ref().cloned(), + metrics: server.metrics.clone(), + identity: repo.identity.clone(), + runtime: Arc::clone(&server.runtime), + options: server.options, + }), + ) + .await; + if !matches!(result, Ok(Some(_))) { + close(&server).await; + panic!("projection did not promote after publishing commit {index}: {result:?}"); + } + repo.invalidate().await; + let response = router(Arc::clone(&server)) + .oneshot( + Request::builder() + .uri(format!( + "/api/repos/team/repo/tree-attribution?rev={}&limit=100", + history.oids[index] + )) + .header("host", "localhost:8788") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + responses.push((index, status, value)); + } + let indexes = load_browse_indexes(&repo.layout).await.unwrap().unwrap(); + let graph = crab_metadata::split_commit_graph::load_split_commit_graph( + &repo.store, + &repo.layout, + indexes.commit_graph_hash(), + server.options.object_limits().max_commit_graph_bytes, + ) + .await + .unwrap(); + let merge = gix_hash::ObjectId::from_hex(history.oids[3].as_bytes()).unwrap(); + let retained = graph + .ordinal(merge.as_bytes().try_into().unwrap()) + .is_some(); + // Projection summaries include retained ordinals. The foreground revision + // boundary must still reject history removed by the current ref rewind. + let hidden = router(Arc::clone(&server)) + .oneshot( + Request::builder() + .uri(format!( + "/api/repos/team/repo/tree-attribution?rev={}&limit=100", + history.oids[3] + )) + .header("host", "localhost:8788") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap() + .status(); + close(&server).await; + assert_eq!((retained, hidden), (true, StatusCode::NOT_FOUND)); + assert!( + responses + .iter() + .all(|(index, status, value)| *status == StatusCode::OK + && value["items"] + .as_array() + .is_some_and(|items| !items.is_empty() + && (*index != 0 + || items + .iter() + .all(|entry| entry["last_commit"]["oid"] == history.oids[0])))), + "{responses:?}" + ); +} + #[tokio::test] async fn capsule_merge_and_nested_tag_preserve_graph_and_first_parent_attribution() { let mut server = fixture_without_cells().await; diff --git a/crates/crab-http-server/src/projection.rs b/crates/crab-http-server/src/projection.rs index b87522a4d..53f8738d3 100644 --- a/crates/crab-http-server/src/projection.rs +++ b/crates/crab-http-server/src/projection.rs @@ -8,9 +8,9 @@ use cellule_runtime::client::{CellClient, Committed, InvocationError}; use cellule_runtime::identity::CellTarget; use cellule_runtime::identity::RequestId; use crab_metadata::manifest_store::RepositorySnapshot; -use crab_metadata::path_state::{PathStateIndex, load_path_state}; +use crab_metadata::path_state::{PathStateItem, PathStateReader, PathStateRecord}; use crab_metadata::split_commit_graph::{SplitCommitGraph, load_split_commit_graph}; -use crab_remote_git::{Commit, OperationKind, RemoteGitRepository, RepositoryOptions, Revision}; +use crab_remote_git::{RemoteGitRepository, RepositoryOptions}; use crab_storage::{Store, StoreLayout}; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -57,10 +57,9 @@ pub(crate) async fn reconcile( cancellation, ) .await?; - let scheduled = router.route_projection(context.repository_id).await?; - let target = scheduled.cell.target.clone(); - let client = scheduled.cell.client.clone(); - let release_after = scheduled.should_release(); + let cell = router.route_projection(context.repository_id).await?; + let target = cell.target.clone(); + let client = cell.client.clone(); let result = build_epoch( store, layout, @@ -94,17 +93,7 @@ pub(crate) async fn reconcile( build_result, build_started.elapsed(), ); - drop(scheduled.cell); - let drained = if release_after { - router.drain_local_target(&target).await - } else { - Ok(()) - }; - match (result, drained) { - (Err(error), _) => Err(error), - (Ok(_), Err(error)) => Err(error), - (Ok(source), Ok(())) => Ok(source), - } + result } async fn collect_epochs( @@ -222,27 +211,12 @@ pub(crate) async fn state( router: &RepositoryCellRouter, repository_id: Uuid, ) -> crate::Result { - let scheduled = router.route_projection(repository_id).await?; - let target = scheduled.cell.target.clone(); - let result = scheduled - .cell - .client - .query::(&target, None, projection::ProjectionState) + let cell = router.route_projection(repository_id).await?; + cell.client + .query::(&cell.target, None, projection::ProjectionState) .await .map(|observed| observed.output) - .map_err(invocation_error); - let release_after = scheduled.should_release(); - drop(scheduled.cell); - let drained = if release_after { - router.drain_local_target(&target).await - } else { - Ok(()) - }; - match (result, drained) { - (Err(error), _) => Err(error), - (Ok(_), Err(error)) => Err(error), - (Ok(output), Ok(())) => Ok(output), - } + .map_err(invocation_error) } pub(crate) async fn attribution( @@ -253,47 +227,31 @@ pub(crate) async fn attribution( paths: Vec>, ) -> crate::Result { let batches = attribution_batches(source_token, commit_oid, paths)?; - let scheduled = router.route_projection(repository_id).await?; - let target = scheduled.cell.target.clone(); - let result = async { - let mut items = Vec::new(); - for paths in batches { - let input = serde_json::to_vec(&projection::AttributionQuery { - source_token: source_token.to_owned(), - commit_oid: commit_oid.to_vec(), - paths, - })?; - let observed = scheduled - .cell - .client - .query::(&target, None, input) - .await - .map_err(invocation_error)?; - let response: projection::AttributionResponse = - serde_json::from_slice(&observed.output).map_err(crate::Error::Json)?; - if response.state != "ready" { - return Ok(response); - } - items.extend(response.items); + let cell = router.route_projection(repository_id).await?; + let target = cell.target.clone(); + let mut items = Vec::new(); + for paths in batches { + let input = serde_json::to_vec(&projection::AttributionQuery { + source_token: source_token.to_owned(), + commit_oid: commit_oid.to_vec(), + paths, + })?; + let observed = cell + .client + .query::(&target, None, input) + .await + .map_err(invocation_error)?; + let response: projection::AttributionResponse = + serde_json::from_slice(&observed.output).map_err(crate::Error::Json)?; + if response.state != "ready" { + return Ok(response); } - Ok(projection::AttributionResponse { - state: "ready".to_owned(), - items, - }) - } - .await; - let release_after = scheduled.should_release(); - drop(scheduled.cell); - let drained = if release_after { - router.drain_local_target(&target).await - } else { - Ok(()) - }; - match (result, drained) { - (Err(error), _) => Err(error), - (Ok(_), Err(error)) => Err(error), - (Ok(output), Ok(())) => Ok(output), + items.extend(response.items); } + Ok(projection::AttributionResponse { + state: "ready".to_owned(), + items, + }) } fn attribution_batches( @@ -483,7 +441,7 @@ async fn build_epoch_contents( ) .await .map_err(metadata_error)?; - let path_state = load_path_state( + let path_state = PathStateReader::load( store, layout, path_hash, @@ -492,20 +450,9 @@ async fn build_epoch_contents( ) .await .map_err(metadata_error)?; - project_commits( - repository, - &graph, - client, - target, - source, - epoch, - metrics, - cancellation, - ) - .await?; - // Tree APIs use the verified remote reader. Preloading unconsumed tree rows - // would add recursive origin reads and let a wide tree block attribution. + project_attribution( + store, &graph, &path_state, client, @@ -518,70 +465,14 @@ async fn build_epoch_contents( .await } -async fn project_commits( - repository: &RemoteGitRepository, - graph: &SplitCommitGraph, - client: &CellClient, - target: &CellTarget, - source: &projection::SourceIdentity, - epoch: u64, - metrics: &crate::metrics::Metrics, - cancellation: &CancellationToken, -) -> crate::Result<()> { - let mut start = 0_u32; - while start < graph.descriptor.commit_count { - check_cancelled(cancellation)?; - let end = start - .saturating_add(COMMIT_BATCH) - .min(graph.descriptor.commit_count); - let operation = repository - .operation(OperationKind::History, cancellation) - .await?; - let result = async { - let mut rows = Vec::with_capacity((end - start) as usize); - for ordinal in start..end { - if cancellation.is_cancelled() { - return Err(crab_remote_git::Error::Cancelled); - } - let record = - graph - .record(ordinal) - .ok_or(crab_remote_git::Error::InternalInvariant { - invariant: "projection commit graph ordinal is missing", - })?; - let oid = object_id(&record.oid).map_err(|_| { - crab_remote_git::Error::InternalInvariant { - invariant: "projection commit graph has a non-SHA-1 object ID", - } - })?; - let snapshot = repository - .snapshot(&Revision::Commit(oid), &operation) - .await?; - let commit = snapshot.commit(&operation).await?; - rows.push(commit_row(ordinal, &commit)); - } - Ok::<_, crab_remote_git::Error>(rows) - } - .await; - let rows = operation.finish(result).await?; - send_json_batch( - client, - target, - projection::OP_COMMITS, - epoch, - source, - &rows, - metrics, - ) - .await?; - start = end; - } - Ok(()) -} - +#[expect( + clippy::too_many_arguments, + reason = "the importer binds origin, graph, Cell, epoch, metrics and cancellation" +)] async fn project_attribution( + store: &Store, graph: &SplitCommitGraph, - path_state: &PathStateIndex, + path_state: &PathStateReader, client: &CellClient, target: &CellTarget, source: &projection::SourceIdentity, @@ -589,130 +480,147 @@ async fn project_attribution( metrics: &crate::metrics::Metrics, cancellation: &CancellationToken, ) -> crate::Result<()> { - if cancellation.is_cancelled() { - return Err(crate::Error::Remote(crab_remote_git::Error::Cancelled)); - } - // Commit nodes before edges and roots to preserve their foreign keys. Build - // only the current batch; collecting all edges duplicates a large index. - for (layer_index, layer) in path_state.layers().iter().enumerate() { - for (batch_index, nodes) in layer.nodes.chunks(ATTRIBUTION_BATCH_NODES).enumerate() { - check_cancelled(cancellation)?; - let nodes = nodes - .iter() - .enumerate() - .map(|(offset, node)| projection::AttributionNodeRow { - node_hash: node_hash( - source, - layer_index as u32, - (batch_index * ATTRIBUTION_BATCH_NODES + offset) as u32, - ), - last_change_ordinal: node.value, - encoded_bytes: 0, - }) - .collect(); - let batch = projection::AttributionBatch { - nodes, - edges: Vec::new(), - roots: Vec::new(), - }; - send_json_batch( - client, - target, - projection::OP_ATTRIBUTION, - epoch, - source, - &batch, - metrics, - ) - .await?; - } - } - let mut batch = projection::AttributionBatch { + let mut scan = path_state.scan(store, graph).map_err(metadata_error)?; + let mut commits = Vec::new(); + let mut commit_bytes = 2; + let mut attribution = projection::AttributionBatch { nodes: Vec::new(), edges: Vec::new(), roots: Vec::new(), }; - let empty_bytes = serde_json::to_vec(&batch)?.len(); - let mut encoded_bytes = empty_bytes; - for (layer_index, layer) in path_state.layers().iter().enumerate() { - for (node_index, node) in layer.nodes.iter().enumerate() { - let parent_hash = node_hash(source, layer_index as u32, node_index as u32); - for (component, child) in &node.children { - check_cancelled(cancellation)?; - let edge = projection::AttributionEdgeRow { - parent_hash: parent_hash.clone(), - component: component.to_vec(), + let empty_bytes = serde_json::to_vec(&attribution)?.len(); + let mut attribution_bytes = empty_bytes; + loop { + let item = tokio::select! { + biased; + () = cancellation.cancelled() => return Err(crab_remote_git::Error::Cancelled.into()), + item = scan.next() => item.map_err(metadata_error)?, + }; + let Some(item) = item else { + break; + }; + check_cancelled(cancellation)?; + let (nodes, edges, roots) = match item { + PathStateItem::Node { reference, value } => ( + Some(projection::AttributionNodeRow { + node_hash: node_hash(source, reference.layer, reference.index), + last_change_ordinal: value, + encoded_bytes: 0, + }), + None, + None, + ), + PathStateItem::Edge { + parent, + name, + child, + } => ( + None, + Some(projection::AttributionEdgeRow { + parent_hash: node_hash(source, parent.layer, parent.index), + component: name.to_vec(), child_hash: node_hash(source, child.layer, child.index), - }; - let edge_bytes = serde_json::to_vec(&edge)?.len() + 1; - if empty_bytes + edge_bytes > MAX_BATCH_PAYLOAD_BYTES { + }), + None, + ), + PathStateItem::Record { ordinal, record } => { + // Cache retained ordinals from verified metadata, without + // resolving them as visible revisions or rereading Git bodies. + let row = commit_row(ordinal, &record); + let bytes = serde_json::to_vec(&row)?.len() + 1; + if bytes + 2 > MAX_BATCH_PAYLOAD_BYTES { return Err(crate::Error::Config( - "one attribution edge exceeds the projection wire limit", + "one commit summary exceeds the projection wire limit", )); } - if !batch.edges.is_empty() - && (batch.edges.len() == ATTRIBUTION_BATCH_NODES * 8 - || encoded_bytes + edge_bytes > MAX_BATCH_PAYLOAD_BYTES) + if !commits.is_empty() + && (commits.len() == COMMIT_BATCH as usize + || commit_bytes + bytes > MAX_BATCH_PAYLOAD_BYTES) { send_json_batch( client, target, - projection::OP_ATTRIBUTION, + projection::OP_COMMITS, epoch, source, - &batch, + &commits, metrics, ) .await?; - batch.edges.clear(); - encoded_bytes = empty_bytes; + commits.clear(); + commit_bytes = 2; } - encoded_bytes += edge_bytes; - batch.edges.push(edge); + commit_bytes += bytes; + commits.push(row); + ( + None, + None, + Some(projection::AttributionRootRow { + commit_ordinal: ordinal, + root_hash: node_hash(source, record.root.layer, record.root.index), + }), + ) } + }; + let bytes = match (&nodes, &edges, &roots) { + (Some(row), _, _) => serde_json::to_vec(row)?.len(), + (_, Some(row), _) => serde_json::to_vec(row)?.len(), + (_, _, Some(row)) => serde_json::to_vec(row)?.len(), + _ => return Err(crate::Error::Config("empty attribution scan item")), + } + 1; + if bytes + empty_bytes > MAX_BATCH_PAYLOAD_BYTES { + return Err(crate::Error::Config( + "one attribution row exceeds the projection wire limit", + )); + } + if attribution.nodes.len() == ATTRIBUTION_BATCH_NODES + || attribution.edges.len() == ATTRIBUTION_BATCH_NODES * 8 + || attribution.roots.len() == ATTRIBUTION_BATCH_NODES + || attribution_bytes + bytes > MAX_BATCH_PAYLOAD_BYTES + { + send_json_batch( + client, + target, + projection::OP_ATTRIBUTION, + epoch, + source, + &attribution, + metrics, + ) + .await?; + attribution.nodes.clear(); + attribution.edges.clear(); + attribution.roots.clear(); + attribution_bytes = empty_bytes; } + attribution.nodes.extend(nodes); + attribution.edges.extend(edges); + attribution.roots.extend(roots); + attribution_bytes += bytes; } - if !batch.edges.is_empty() { + if !commits.is_empty() { send_json_batch( client, target, - projection::OP_ATTRIBUTION, + projection::OP_COMMITS, epoch, source, - &batch, + &commits, metrics, ) .await?; } - for start in (0..graph.descriptor.commit_count).step_by(ATTRIBUTION_BATCH_NODES) { - check_cancelled(cancellation)?; - let end = start - .saturating_add(ATTRIBUTION_BATCH_NODES as u32) - .min(graph.descriptor.commit_count); - let roots = (start..end) - .map(|ordinal| { - let root = path_state - .record(ordinal) - .ok_or(crate::Error::Config("path-state record is missing"))? - .root; - Ok(projection::AttributionRootRow { - commit_ordinal: ordinal, - root_hash: node_hash(source, root.layer, root.index), - }) - }) - .collect::>>()?; - let batch = projection::AttributionBatch { - nodes: Vec::new(), - edges: Vec::new(), - roots, - }; + if !attribution.nodes.is_empty() + || !attribution.edges.is_empty() + || !attribution.roots.is_empty() + { send_json_batch( client, target, projection::OP_ATTRIBUTION, epoch, source, - &batch, + &attribution, metrics, ) .await?; @@ -747,29 +655,14 @@ fn reference_rows(snapshot: &RepositorySnapshot) -> crate::Result projection::CommitRow { - let message = commit.message.as_ref(); - let preview = message[..message.len().min(MAX_MESSAGE_PREVIEW_BYTES)].to_vec(); +fn commit_row(ordinal: u32, record: &PathStateRecord) -> projection::CommitRow { + let message = &record.message; projection::CommitRow { ordinal, - oid: commit.oid.as_slice().to_vec(), - tree_oid: commit.tree.as_slice().to_vec(), - parents: commit - .parents - .iter() - .map(|parent| parent.as_slice().to_vec()) - .collect(), - author_name: commit.author.name.to_vec(), - author_email: commit.author.email.to_vec(), - author_time: commit.author.seconds, - author_tz_offset_seconds: commit.author.offset_seconds, - committer_name: commit.committer.name.to_vec(), - committer_email: commit.committer.email.to_vec(), - committer_time: commit.committer.seconds, - committer_tz_offset_seconds: commit.committer.offset_seconds, - message_preview: preview, - message_truncated: message.len() > MAX_MESSAGE_PREVIEW_BYTES, - encoded_bytes: u64::try_from(message.len()).unwrap_or(u64::MAX), + oid: record.oid.to_vec(), + author_name: record.author.clone(), + author_time: record.author_seconds, + message_preview: message[..message.len().min(MAX_MESSAGE_PREVIEW_BYTES)].to_vec(), } } @@ -860,11 +753,6 @@ fn invocation_error(error: InvocationError) -> crate::Error { } } -fn object_id(bytes: &[u8]) -> crate::Result { - gix_hash::ObjectId::try_from(bytes) - .map_err(|_| crate::Error::Config("Git object ID is not a SHA-1")) -} - fn decode_sha1(value: &str) -> crate::Result> { if value.len() != 40 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(crate::Error::Config("manifest ref is not a SHA-1")); diff --git a/crates/crab-http-server/tests/verify_live.py b/crates/crab-http-server/tests/verify_live.py index 1f07a15d8..791b93983 100644 --- a/crates/crab-http-server/tests/verify_live.py +++ b/crates/crab-http-server/tests/verify_live.py @@ -16,6 +16,11 @@ import urllib.request +def require(condition, context): + if not condition: + raise AssertionError(context) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--url', required=True, help='Server origin, e.g. http://127.0.0.1:8788') @@ -59,11 +64,11 @@ def http(path, status=200, method='GET', headers=None, sample_class=None): response = error with response: body = response.read() - assert response.status == status, (path, response.status, body[:300]) + require(response.status == status, (path, response.status, body[:300])) if status == 200 and path.startswith(api): - assert response.headers['Server-Timing'], path - assert response.headers['X-Crab-Generation'], path - assert response.headers['Cache-Control'] == 'no-store', path + require(response.headers['Server-Timing'], path) + require(response.headers['X-Crab-Generation'], path) + require(response.headers['Cache-Control'] == 'no-store', path) action = sample_class or path.split('?', 1)[0].rsplit('/', 1)[-1] elapsed = (time.perf_counter() - started) * 1000 with samples_lock: @@ -77,14 +82,14 @@ def read(action, **params): def verify_content(content, revision, path): body = git('show', f'{revision}:{path}') oid = hashlib.sha1(b'blob ' + str(len(body)).encode() + b'\0' + body).hexdigest() - assert content['oid'] == oid and content['size'] == len(body), path + require(content['oid'] == oid and content['size'] == len(body), path) if content['text'] is not None: - assert content['text'].encode() == body, path + require(content['text'].encode() == body, path) return body - assert git('rev-parse', args.revision).decode().strip() == args.revision + require(git('rev-parse', args.revision).decode().strip() == args.revision, 'HTTP qualification failed') refs = read('refs') - assert refs['head']['oid'] == args.revision + require(refs['head']['oid'] == args.revision, 'HTTP qualification failed') for directory in args.directory: expected = git('ls-tree', '-z', args.revision + (':' + directory if directory else '')) expected = [record.split(b'\t', 1) for record in expected.split(b'\0') if record] @@ -95,14 +100,14 @@ def verify_content(content, revision, path): if cursor: params['cursor'] = cursor page = read('tree', **params) - assert page['commit'] == args.revision + require(page['commit'] == args.revision, 'HTTP qualification failed') items.extend(page['items']) cursor = page['next'] if not cursor: break actual = [(item['mode'].lstrip('0'), item['oid'], bytes.fromhex(item['path_hex']).rsplit(b'/', 1)[-1]) for item in items] oracle = [(meta.split()[0].decode().lstrip('0'), meta.split()[2].decode(), name) for meta, name in expected] - assert actual == oracle, directory + require(actual == oracle, directory) checks['tree_entries'] += len(items) if args.attribution: attributed = [] @@ -112,37 +117,45 @@ def verify_content(content, revision, path): if cursor: params['cursor'] = cursor page = read('tree-attribution', **params) - assert page['commit'] == args.revision + require(page['commit'] == args.revision, 'HTTP qualification failed') attributed.extend(page['items']) cursor = page['next'] if not cursor: break - assert [(item['path_hex'], item['oid'], item['mode']) for item in attributed] == [ + require([(item['path_hex'], item['oid'], item['mode']) for item in attributed] == [ (item['path_hex'], item['oid'], item['mode']) for item in items - ], directory + ], directory) for item in attributed: path = bytes.fromhex(item['path_hex']) expected_commit = git('--literal-pathspecs', 'log', '--first-parent', '--max-count=1', '--format=%H', args.revision, '--', os.fsdecode(path)).decode().strip() - assert item['last_commit']['oid'] == expected_commit, item['path_hex'] + require(item['last_commit']['oid'] == expected_commit, item['path_hex']) + headers, message = git('cat-file', 'commit', expected_commit).split(b'\n\n', 1) + author = next(line[7:] for line in headers.splitlines() if line.startswith(b'author ')) + identity, timestamp = author.rsplit(b'> ', 1) + name = identity.rsplit(b' <', 1)[0] + summary = item['last_commit'] + require(summary['author'] == name.decode('utf-8', 'replace'), item['path_hex']) + require(summary['author_seconds'] == int(timestamp.split()[0]), item['path_hex']) + require(summary['message'] == message[:8192].decode('utf-8', 'replace'), item['path_hex']) checks['tree_attribution_entries'] = checks.get('tree_attribution_entries', 0) + len(attributed) page = read('commits', limit=10) - assert [item['oid'] for item in page['items']] == git('rev-list', '--first-parent', '--max-count=10', args.revision).decode().splitlines() + require([item['oid'] for item in page['items']] == git('rev-list', '--first-parent', '--max-count=10', args.revision).decode().splitlines(), 'HTTP qualification failed') for commit in page['items']: raw = git('cat-file', 'commit', commit['oid']) - assert bytes.fromhex(commit['message_hex']) == raw.split(b'\n\n', 1)[1] - assert read('commit', rev=commit['oid'])['tree'] == commit['tree'] + require(bytes.fromhex(commit['message_hex']) == raw.split(b'\n\n', 1)[1], 'HTTP qualification failed') + require(read('commit', rev=commit['oid'])['tree'] == commit['tree'], 'HTTP qualification failed') checks['commits'] += 1 for path in args.file or ['README.md']: expected = verify_content(read('file', path=path), args.revision, path) actual, headers = http(api + '/blob?' + urllib.parse.urlencode({'rev': args.revision, 'path': path})) - assert actual == expected and headers['Content-Disposition'] == 'attachment', path + require(actual == expected and headers['Content-Disposition'] == 'attachment', path) checks['blobs'] += 1 changes = read('changes') - assert changes['base'] == git('rev-parse', args.revision + '^1').decode().strip() + require(changes['base'] == git('rev-parse', args.revision + '^1').decode().strip(), 'HTTP qualification failed') expected_paths = git('diff-tree', '--no-commit-id', '--no-renames', '--name-only', '-r', '-z', changes['base'], args.revision).split(b'\0')[:-1] - assert [bytes.fromhex(change['path_hex']) for change in changes['changes']] == expected_paths + require([bytes.fromhex(change['path_hex']) for change in changes['changes']] == expected_paths, 'HTTP qualification failed') for change in changes['changes']: diff = read('diff', path_hex=change['path_hex']) path = bytes.fromhex(change['path_hex']).decode() @@ -156,19 +169,19 @@ def verify_content(content, revision, path): actual = [entry['commit']['oid'] for entry in attribution['ranges'] for _ in range(entry['lines'])] lines = git('blame', '--first-parent', '--line-porcelain', args.revision, '--', args.blame).splitlines() expected = [line.split()[0].decode() for line in lines if len(line.split()) in (3, 4) and len(line.split()[0]) == 40 and all(byte in b'0123456789abcdef' for byte in line.split()[0])] - assert actual == expected, args.blame + require(actual == expected, args.blame) checks['blame_lines'] = len(actual) for suffix, status in [('tree?limit=0', 400), ('tree?path=../secret', 404), ('file?path=missing-file', 404), ('file?path=.', 404), ('tree?path_hex=ff0', 400), ('tree?cursor=bad', 400)]: body, _ = http(api + '/' + suffix, status) - assert json.loads(body)['error']['message'] + require(json.loads(body)['error']['message'], 'HTTP qualification failed') checks['errors'] += 1 http('/api/repos/unknown/missing/refs', 404) http('/api/repos', 403, headers={'Host': 'untrusted.invalid'}) http('/assets/absent.js', 404) shell, headers = http('/' + args.repository) - assert b'id="root"' in shell and "frame-ancestors 'none'" in headers['Content-Security-Policy'] + require(b'id="root"' in shell and "frame-ancestors 'none'" in headers['Content-Security-Policy'], 'HTTP qualification failed') head, headers = http('/' + args.repository, method='HEAD') - assert not head and int(headers['Content-Length']) == len(shell) + require(not head and int(headers['Content-Length']) == len(shell), 'HTTP qualification failed') historical = None if args.historical_commits: ancestors = git('rev-list', '--first-parent', f'--max-count={args.history_window + 1}', args.revision).decode().splitlines()[1:] @@ -180,9 +193,9 @@ def verify_historical(oid): raw = git('cat-file', 'commit', oid) path = api + '/commit?' + urllib.parse.urlencode({'rev': oid}) observed = json.loads(http(path, sample_class='historical_commit')[0]) - assert observed['oid'] == oid, oid - assert observed['tree'].encode() == raw.splitlines()[0].split()[1], oid - assert bytes.fromhex(observed['message_hex']) == raw.split(b'\n\n', 1)[1], oid + require(observed['oid'] == oid, oid) + require(observed['tree'].encode() == raw.splitlines()[0].split()[1], oid) + require(bytes.fromhex(observed['message_hex']) == raw.split(b'\n\n', 1)[1], oid) with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: list(pool.map(verify_historical, selected)) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 4e8df01b5..569143dd9 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -61,13 +61,19 @@ budget before buffering, then admit each layer against its authenticated size and the remaining aggregate budget. All bodies still require hash and exact length verification; malformed provider sizes cannot bypass the encoded-byte ceiling. Decoded index structures require separate memory qualification. -Path-state descriptor version 4 requires exact-name tree comparison semantics; -earlier descriptors can authenticate incorrect mutations and are rejected. +Path-state descriptor version 5 requires exact-name tree comparison semantics +and authenticated pages; retired descriptors are rejected. Maintenance rebuilds completed indexes and resumable checkpoints through the -existing conditional publication path. The unchanged `CRABPS03` layer codec -stores each raw path component once per layer in a sorted canonical table, -with checked component ordinals on edges. Decoded edges share immutable component -bytes. No release tag contains a path-state format; maintenance rebuilds +existing conditional publication path. Record, node, component and child pages +have bounded authenticated directories. Raw components are stored once per layer, +with checked component ordinals on edges. `PathStateReader` fetches only the pages +required by a point/batch query. Its maintenance scan emits nodes before dependent +edges and records, verifies complete layer hashes and physical page coverage, and +retains only the current layer's dictionary/audit plus the bounded page cache. +Only successful EOF proves a complete scan; callers must not publish readiness +after partial output or an error. Full-layer hashing followed by page traversal +adds maintenance I/O while avoiding complete layer/index buffering. +No release tag contains a path-state format; maintenance rebuilds unshipped derived indexes rather than reading retired versions. Validated index descriptors/layers expose only immutable accessors, so checkpoint append reuses the verified prefix instead of rescanning diff --git a/crates/crab-metadata/src/path_state.rs b/crates/crab-metadata/src/path_state.rs index fb0f10a47..ade719b29 100644 --- a/crates/crab-metadata/src/path_state.rs +++ b/crates/crab-metadata/src/path_state.rs @@ -16,7 +16,7 @@ mod paged; mod reader; pub use paged::{PathStateLayerPages, PathStatePageRef}; #[cfg(feature = "storage")] -pub use reader::PathStateReader; +pub use reader::{PathStateItem, PathStateReader, PathStateScan}; #[cfg(feature = "storage")] mod storage; diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index dd9f6df3a..2b1505a1c 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -17,6 +17,10 @@ use crate::split_commit_graph::{CommitGraphLayerRef, SplitCommitGraph}; const CACHE_BYTES: usize = 8 * 1024 * 1024; const CACHE_PAGES: usize = 256; +#[path = "scan.rs"] +mod scan; +pub use scan::{PathStateItem, PathStateScan}; + /// Verified path attribution with bounded, shared immutable-page caching. #[derive(Debug)] pub struct PathStateReader { diff --git a/crates/crab-metadata/src/path_state/reader_tests.rs b/crates/crab-metadata/src/path_state/reader_tests.rs index 0d3708f4e..72c68fc42 100644 --- a/crates/crab-metadata/src/path_state/reader_tests.rs +++ b/crates/crab-metadata/src/path_state/reader_tests.rs @@ -7,7 +7,7 @@ use bytes::Bytes; use object_store::{ObjectStoreExt, memory::InMemory}; use std::sync::atomic::{AtomicU64, Ordering}; -async fn fixture() -> ( +pub(super) async fn fixture() -> ( Store, StoreLayout, SplitCommitGraph, diff --git a/crates/crab-metadata/src/path_state/scan.rs b/crates/crab-metadata/src/path_state/scan.rs new file mode 100644 index 000000000..973137941 --- /dev/null +++ b/crates/crab-metadata/src/path_state/scan.rs @@ -0,0 +1,479 @@ +use super::*; + +/// One verified record, node or edge emitted in dependency order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PathStateItem { + Record { + ordinal: u32, + record: PathStateRecord, + }, + Node { + reference: PathStateNodeRef, + value: Option, + }, + Edge { + parent: PathStateNodeRef, + name: Arc<[u8]>, + child: PathStateNodeRef, + }, +} + +/// Exhaustive maintenance traversal retaining only the current layer's dictionary and page audit. +pub struct PathStateScan<'a> { + reader: &'a PathStateReader, + store: &'a Store, + graph: &'a SplitCommitGraph, + layer: usize, + phase: Phase, + names: Vec>, + used: Vec, + audit: BTreeMap, + nodes: Cursor, + records: Cursor, + children: Option<(PathStateNodeRef, Cursor)>, +} + +enum Phase { + Begin, + Nodes, + Records, + Finish, + Failed, +} + +#[derive(Default)] +struct Cursor { + pending: Vec, + leaf: Option>, + index: usize, +} + +impl PathStateReader { + /// Scan every layer through authenticated pages; errors invalidate the unfinished traversal. + pub fn scan<'a>( + &'a self, + store: &'a Store, + graph: &'a SplitCommitGraph, + ) -> Result> { + if !matches_graph(&self.descriptor, graph) || self.graph_layers != graph.descriptor.layers { + return corrupt("path-state scan graph identity changed"); + } + let bytes = + self.descriptor + .layers + .iter() + .try_fold(self.descriptor_bytes, |bytes, layer| { + bytes + .checked_add(layer.bytes) + .ok_or_else(|| corruption("scan byte count overflows")) + })?; + if bytes > self.max_bytes { + return corrupt("path-state scan exceeds its byte limit"); + } + Ok(PathStateScan { + reader: self, + store, + graph, + layer: 0, + phase: Phase::Begin, + names: Vec::new(), + used: Vec::new(), + audit: BTreeMap::new(), + nodes: Cursor::default(), + records: Cursor::default(), + children: None, + }) + } +} + +impl PathStateScan<'_> { + /// Advance the scan; only EOF establishes complete physical and semantic closure. + pub async fn next(&mut self) -> Result> { + if matches!(self.phase, Phase::Failed) { + return corrupt("path-state scan previously failed"); + } + let result = self.advance().await; + if result.is_err() { + self.phase = Phase::Failed; + } + result + } + + async fn advance(&mut self) -> Result> { + while self.layer < self.reader.descriptor.layers.len() { + let reference = &self.reader.descriptor.layers[self.layer]; + let path = &self.reader.paths[self.layer]; + if let Some((parent, cursor)) = self.children.as_mut() { + if let Some((key, value)) = cursor + .row( + self.reader, + self.store, + self.layer, + Kind::Children, + &mut self.audit, + ) + .await? + { + let ordinal = integer(&key)? as usize; + let name = self + .names + .get(ordinal) + .ok_or_else(|| corruption("component ordinal is out of bounds"))?; + let child = codec::decode_child(&value, path.as_ref())?; + self.reader.validate_node(child, Some(*parent))?; + self.used[ordinal] = true; + return Ok(Some(PathStateItem::Edge { + parent: *parent, + name: name.clone(), + child, + })); + } + self.children = None; + } + match self.phase { + Phase::Begin => { + // Verify the whole object without retaining it. Page reads + // then validate their own identities even if origin changes. + let (meta, _, mut stream) = self.store.get_stream(path, None).await?; + if meta.size != reference.bytes { + return corrupt("layer length mismatch"); + } + let mut hasher = blake3::Hasher::new(); + while let Some(bytes) = stream.next().await { + hasher.update(&bytes?); + } + if hasher.finalize().to_hex().as_str() != reference.hash { + return corrupt("layer content identity mismatch"); + } + let mut ordered = BTreeMap::new(); + if let Some(root) = &reference.pages.names { + let mut names = Cursor::new(root.clone()); + while let Some((key, value)) = names + .row( + self.reader, + self.store, + self.layer, + Kind::Names, + &mut self.audit, + ) + .await? + { + let (ordinal, name) = codec::decode_name(&key, &value, path.as_ref())?; + if ordered.insert(ordinal, Arc::<[u8]>::from(name)).is_some() { + return corrupt("duplicate component ordinal"); + } + } + } + for (index, (ordinal, name)) in ordered.into_iter().enumerate() { + if ordinal as usize != index + || self + .names + .last() + .is_some_and(|previous| previous.as_ref() >= name.as_ref()) + { + return corrupt("component ordinals are incomplete or unordered"); + } + self.names.push(name); + } + self.used = vec![false; self.names.len()]; + self.nodes = Cursor::new(reference.pages.nodes.clone()); + self.records = Cursor::new(reference.pages.records.clone()); + self.phase = Phase::Nodes; + } + Phase::Nodes => { + if let Some((key, value)) = self + .nodes + .row( + self.reader, + self.store, + self.layer, + Kind::Nodes, + &mut self.audit, + ) + .await? + { + let node = PathStateNodeRef { + layer: self.layer as u32, + index: integer(&key)?, + }; + self.reader.validate_node(node, None)?; + let (value, children) = + codec::decode_node(&value, reference.bytes, path.as_ref())?; + if value + .is_some_and(|ordinal| ordinal >= self.reader.descriptor.commit_count) + { + return corrupt("node attribution is out of bounds"); + } + self.children = children.map(|root| (node, Cursor::new(root))); + return Ok(Some(PathStateItem::Node { + reference: node, + value, + })); + } + self.phase = Phase::Records; + } + Phase::Records => { + if let Some((key, value)) = self + .records + .row( + self.reader, + self.store, + self.layer, + Kind::Records, + &mut self.audit, + ) + .await? + { + let ordinal = reference + .base_ordinal + .checked_add(integer(&key)?) + .ok_or_else(|| corruption("record ordinal overflows"))?; + let record = codec::decode_record(&value, path.as_ref())?; + let commit = self + .graph + .record(ordinal) + .ok_or_else(|| corruption("commit graph record is missing"))?; + if record.oid != commit.oid + || record.first_parent != commit.parents.first().copied() + { + return corrupt("path-state record does not match its commit ordinal"); + } + self.reader.validate_node(record.root, None)?; + if record.root.layer as usize > self.layer { + return corrupt("record root points to a future layer"); + } + return Ok(Some(PathStateItem::Record { ordinal, record })); + } + self.phase = Phase::Finish; + } + Phase::Finish => { + if self.used.contains(&false) { + return corrupt("unused path-state component"); + } + let mut offset = 0u64; + for (start, (_, page)) in &self.audit { + if *start != offset { + return corrupt("layer pages overlap or leave unused bytes"); + } + offset = offset + .checked_add(u64::from(page.bytes)) + .ok_or_else(|| corruption("layer coverage overflows"))?; + } + if offset != reference.bytes { + return corrupt("layer contains trailing unreferenced bytes"); + } + self.layer += 1; + self.names.clear(); + self.used.clear(); + self.audit.clear(); + self.phase = Phase::Begin; + } + Phase::Failed => return corrupt("path-state scan previously failed"), + } + } + Ok(None) + } +} + +impl Cursor { + fn new(root: PathStatePageRef) -> Self { + Self { + pending: vec![root], + ..Self::default() + } + } + + async fn row( + &mut self, + reader: &PathStateReader, + store: &Store, + layer: usize, + kind: Kind, + audit: &mut BTreeMap, + ) -> Result, Vec)>> { + loop { + if let Some(page) = &self.leaf { + let Page::Leaf(rows) = page.as_ref() else { + return corrupt("scan leaf changed kind"); + }; + if let Some(row) = rows.get(self.index) { + self.index += 1; + return Ok(Some(row.clone())); + } + self.leaf = None; + } + let Some(reference) = self.pending.last().cloned() else { + return Ok(None); + }; + reference.validate(kind, reader.descriptor.layers[layer].bytes)?; + if let Some((previous_kind, previous)) = audit.get(&reference.offset) + && (*previous_kind != kind || *previous != reference) + { + return corrupt("page has conflicting references"); + } + let page = reader.page(store, layer, &reference, kind).await?; + // Keep the cursor unchanged across I/O. A dropped next() future + // must not skip a page and later claim complete scan coverage. + self.pending.pop(); + audit.insert(reference.offset, (kind, reference)); + match page.as_ref() { + Page::Leaf(_) => { + self.leaf = Some(page); + self.index = 0; + } + Page::Branch(children) => self.pending.extend(children.iter().rev().cloned()), + } + } + } +} + +fn integer(key: &[u8]) -> Result { + Ok(u32::from_be_bytes( + key.try_into() + .map_err(|_| corruption("invalid ordinal key"))?, + )) +} + +#[cfg(test)] +mod tests { + use super::super::tests::fixture; + use super::*; + use crate::path_state::{DEFAULT_MAX_PATH_STATE_BYTES, PathStateLayer, PathStateNode}; + + #[tokio::test] + async fn cancelled_page_read_can_resume_without_skipping_records() { + let (store, layout, graph, write) = fixture().await; + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let throttled = Arc::new(object_store::throttle::ThrottledStore::new( + store.inner().clone(), + object_store::throttle::ThrottleConfig { + wait_get_per_call: std::time::Duration::from_secs(60), + ..Default::default() + }, + )); + let slow = Store::new(throttled.clone()); + let mut cursor = Cursor::new(reader.descriptor.layers[0].pages.records.clone()); + let mut audit = BTreeMap::new(); + { + let pending = cursor.row(&reader, &slow, 0, Kind::Records, &mut audit); + tokio::pin!(pending); + assert!(futures_util::poll!(pending.as_mut()).is_pending()); + } + throttled.config_mut(|config| config.wait_get_per_call = std::time::Duration::ZERO); + let row = cursor + .row(&reader, &slow, 0, Kind::Records, &mut audit) + .await + .unwrap(); + assert_eq!( + row.map(|(key, bytes)| ( + integer(&key).unwrap(), + codec::decode_record(&bytes, "test").unwrap() + )), + Some((0, write.index.record(0).unwrap().clone())) + ); + } + + #[tokio::test] + async fn scan_reconstructs_the_full_index_in_dependency_order() { + let (store, layout, graph, write) = fixture().await; + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + &graph, + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let mut scan = reader.scan(&store, &graph).unwrap(); + let mut layers = reader + .descriptor + .layers + .iter() + .map(|reference| PathStateLayer { + base_ordinal: reference.base_ordinal, + records: Vec::new(), + nodes: Vec::new(), + }) + .collect::>(); + while let Some(item) = scan.next().await.unwrap() { + match item { + PathStateItem::Node { reference, value } => { + assert_eq!( + reference.index as usize, + layers[reference.layer as usize].nodes.len() + ); + layers[reference.layer as usize].nodes.push(PathStateNode { + value, + children: BTreeMap::new(), + }); + } + PathStateItem::Edge { + parent, + name, + child, + } => { + assert!( + layers[child.layer as usize] + .nodes + .get(child.index as usize) + .is_some() + ); + layers[parent.layer as usize].nodes[parent.index as usize] + .children + .insert(name, child); + } + PathStateItem::Record { ordinal, record } => { + let layer = layers.partition_point(|layer| layer.base_ordinal <= ordinal) - 1; + layers[layer].records.push(record); + } + } + } + assert_eq!(layers, write.index.layers()); + } + + #[tokio::test] + async fn full_scan_rejects_unreferenced_bytes_and_cannot_resume_after_error() { + let (store, layout, graph, write) = fixture().await; + let mut descriptor = write.index.descriptor().clone(); + let layer = descriptor.layers.last_mut().unwrap(); + let mut bytes = store + .get_with_etag_bounded(&layout.repo_path(&layer.path), layer.bytes) + .await + .unwrap() + .0 + .to_vec(); + bytes.push(0); + layer.hash = blake3::hash(&bytes).to_hex().to_string(); + layer.path = crate::path_state::path_state_layer_path(&layer.hash); + layer.bytes = bytes.len() as u64; + store + .put(&layout.repo_path(&layer.path), bytes::Bytes::from(bytes)) + .await + .unwrap(); + let bytes = serde_json::to_vec(&descriptor).unwrap(); + let hash = blake3::hash(&bytes).to_hex().to_string(); + store + .put( + &layout.bulk_manifest_path("path-state", &hash), + bytes::Bytes::from(bytes), + ) + .await + .unwrap(); + let reader = + PathStateReader::load(&store, &layout, &hash, &graph, DEFAULT_MAX_PATH_STATE_BYTES) + .await + .unwrap(); + let mut scan = reader.scan(&store, &graph).unwrap(); + while matches!(scan.next().await, Ok(Some(_))) {} + assert!(scan.next().await.is_err()); + } +} From a55b7a38ed3231938b68f830e75ce25e81e47542 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 15:18:20 -0700 Subject: [PATCH 007/127] docs: plan indexed reads and shared staging --- ...ndexed-remote-reads-and-unified-staging.md | 481 ++++++++++++++++++ advisor-plans/README.md | 12 + 2 files changed, 493 insertions(+) create mode 100644 advisor-plans/037-indexed-remote-reads-and-unified-staging.md diff --git a/advisor-plans/037-indexed-remote-reads-and-unified-staging.md b/advisor-plans/037-indexed-remote-reads-and-unified-staging.md new file mode 100644 index 000000000..b80b74b5b --- /dev/null +++ b/advisor-plans/037-indexed-remote-reads-and-unified-staging.md @@ -0,0 +1,481 @@ +# Plan 037: Bound remote reads and unify large-file staging + +Planned at `51435f76bd21142bc20a94745984a48b6c7db43b`, 2026-10-13. +Priority P1; effort XL; correctness and publication risk HIGH. +Status: implementation in progress. Existing indexed-read stages 1–3 are partially +qualified. Slice A's paged producer/component reader is implemented and qualified; +full foreground conversion and the remaining slices are incomplete. The canonical +indexed-read ledger records the evidence and remaining gates. + +## Executor contract + +Read this complete plan and the root and nearest scoped `AGENTS.md` before work. +One reviewable PR per vertical slice. Preserve the existing stacked branch base; +inspect parent PR status before choosing the next base. Use `codex/` branches. +The user authorized implementation and PR creation, but this document is a plan, +not completion evidence. Attach every created PR to the chat. + +First run: + +```sh +git diff --stat 51435f76bd21142bc20a94745984a48b6c7db43b..HEAD -- \ + crates/crab-metadata crates/crab-remote-git crates/crab-read crates/crab-write \ + crates/crab-staging crates/crab-remote crab/src/git crab/src/cmd/add.rs +``` + +Reconcile changed contracts before execution. Do not redo independently completed +work. Update `crab/docs/architecture/indexed-remote-reads.md` with actual receipts +when implementing; do not replace its incomplete gates with assertions. + +## Outcome and scope + +Keep Git commit/tree/blob identity and Xet content formats intact. Use immutable, +authenticated indexes in object storage to answer fixed queries without decoding +unrelated history. Keep ref visibility independent of physical object retention. +Native add and Git clean retain their different entry points and publication +policies, but share one streaming staging engine, recipe authority and payload +ownership model. + +Include canonical reads, HTTP attribution projections, native push, protected +receive, hydration, checkpoint/compaction, fsck and GC. No new Crab data server, +second authoritative SQL database, dependency override, or speculative config. +HTTP projection remains derived and bound to the captured repository state. + +Do not change chunking policy, Xet hash algorithms, xorb/shard wire format, or +public CLI flags in this program. Full logical line blame on 10-GiB Xet text is +a later capability: provide version attribution without hydrating the payload. +Ordinary Git blobs continue to use the existing validated pack locator and +range reader; change that locator only after measurements identify excess work. + +## Current state and evidence + +The canonical ledger is `crab/docs/architecture/indexed-remote-reads.md`. +It records baseline, format, native Git oracle and RustFS receipts. Cloud latency, +broader lifecycle and full blame parity remain incomplete. + +| Surface | Current implementation | Change owner | +| --- | --- | --- | +| Path attribution | `PathStateReader` authenticates bounded range pages; foreground uses it, maintenance streams it | `crab-metadata`, `crab-remote-git`, HTTP projection | +| Graph opening | `load_split_commit_graph` reads every layer; `CommitGraphIndex::load` builds full pinned reachability | `crab-metadata`, `crab-remote-git` | +| Graph/path binding | `SplitCommitGraph::ordinal_digest` hashes every positional record | `crab-metadata` | +| Repository control opening | `open_view_from_root_with_control` loads complete controls/frontier; selected-ref view already exists | `crab-read` | +| File history/blame | `snapshot.rs` visits first-parent commits and trees in batches; existing result cache is bounded | `crab-remote-git` | +| Historical Xet lookup | `xet_publication::prepare_delta` reconstructs the catalog, then scans historical xorb chunks | CLI publication, shared read/write/catalog owners | +| Native staging | `crab add` uses `crab_staging::stream`, prepared plans, batch rollback and close-before-index | `crab-staging`, CLI add | +| Git clean | Separate CDC/staging loop, recipe lease publication and flush-before-pointer | `crab-staging`, CLI clean/filter | + +Drift anchors, copied from current source: + +```rust +// crates/crab-remote-git/src/commit_graph.rs +struct CommitGraphIndex { + graph: SplitCommitGraph, + pinned_reachable: Vec, +} +// crates/crab-metadata/src/split_commit_graph.rs:772 +for reference in &descriptor.layers { /* fetch and decode each layer */ } +// crab/src/git/xet_publication.rs:69 +let base_catalog = view.pointer_catalog()?; +// crates/crab-metadata/src/capsule_protocol/pointer.rs:188 +pub fn apply(&mut self, delta: &Self) -> Result<()> { /* validate closure */ } +``` + +Important catalog semantics: immutable xorb/shard identity conflicts are errors; +a newer file-to-shard mapping may replace an older one only with the same declared +file size. Reproduce `PointerCatalog::apply`, not generic last-write-wins. + +Current local baseline: cold attribution rises from 8.001 ms at 100 commits to +142.636 ms at 10,000; decoded bytes rise from 137,704 to 13,052,443. Lazy path +attribution already reduced 10,000-commit query p95 to 8.305 ms with at most +77,406 decoded bytes. Opening still scales and must be measured independently. +These are recorded local receipts, not new tests run for this proposal. + +## Architecture decisions + +### Capture authority, then resolve indexes + +Every operation holds an immutable captured view: repository/storage identity, +root identity, checked ref positions and activations, relevant inventory/catalog +roots, graph identity and attribution semantics. These are conceptual fields; +extend existing view types rather than introducing a parallel snapshot framework. + +An index hit supplies a location or summary. It never supplies authorization. +Selected-ref capture is authoritative only for the selected names. A missing +selected ref says nothing about another ref. Arbitrary-OID queries requiring +membership in any visible ref need a complete visibility proof. + +Do not claim constant-cost arbitrary-OID visibility from independently mutable +per-ref heads. Initially charge its ref capture explicitly; selected-ref reads +must avoid unrelated heads. Removing that ref-count cost requires an additional +publication protocol with an atomic visibility-directory fence. Design and prove +that separately before changing the current per-ref publication authority. + +### Reuse authenticated page mechanics + +Use the path-state format as the concrete starting point: immutable object, +typed range references, hashes, exact key bounds/counts, bounded internal pages, +bounded height, strict decoding and range validation. Target 32-KiB pages; retain +the existing 2-MiB hard page bound unless measured graph/catalog needs justify a +smaller limit. Bound descriptors, level manifests and caches as well as leaves. + +Extract only mechanisms actually shared by graph and path-state consumers; keep +semantic validation in each owner. Do not build a general database abstraction. +Batch keys by common pages, deduplicate concurrent reads and charge bytes before +allocation/I/O. Cache verified immutable pages by repository/origin identity, +object hash and complete page reference; misses also bind the captured state. +Cache locks must not cover network waits or retain operation admission tokens. + +Authenticated means bound to the captured, verified root. Hashes alone do not +prove that arbitrary client-provided graph/catalog data describes valid Git. +Writers must construct indexes from verified objects under the existing authority +boundary; maintenance validates whole closure. Point reads validate the dependency +closure they consume. Corruption is an error, never an ordinary missing-key result. + +### Write immutable dependencies before publishing authority + +Order: payload/recipe dependencies → index pages → verified descriptor or capsule +→ existing ref/root CAS and activation → acknowledgment. Use existing storage +conditional-write contracts. Never advertise partial coverage. A CAS conflict +replans against fresh captured state; it must not silently reuse a stale closure. + +Amazon S3 conditional writes compare the object's ETag; they do not provide a +multi-object transaction. Verify equivalent behavior on each supported provider. +Reference: https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html + +## Execution order + +Existing stages 1–3 establish baseline and path-state foundations. Finish their +outstanding qualification without rebuilding them. Execute A → B → C → D → E; +F may proceed after the shared recipe/ownership audit and joins G. H follows D. +Lifecycle tests accompany every slice; G is the combined release gate. + +### A. Page graph lookup and make positional identity immutable + +Scope: `crates/crab-metadata/src/split_commit_graph.rs`, path-state readers/codecs, +their tests, and graph builders in `crates/crab-write/src/generation*`. + +1. Audit released graph producers/readers, not just format file presence. The + ledger lists six upstream release commits containing graph version 1. Define + explicit rebuild/migration behavior before replacing a published reader. +2. Encode paged OID→ordinal and ordinal→record indexes. Keep parent ordinals and + corrected generation in records; put variable summaries in admitted pages. + Use a bounded directory over layers, not an ever-growing inline layer list. +3. Introduce narrow async batched lookup with caller-owned `Store` admission. + Preserve duplicate input positions. Verify OID↔ordinal agreement, ordered + parent edges, counts, page routing and generation/inventory bindings. +4. Publish a writer-verified positional identity in the authenticated descriptor. + Path-state binds that identity without hashing all graph records on opening. + Do not memoize a digest behind publicly mutable `descriptor`/`layers` fields. +5. Keep streaming complete validation for maintenance, rebuild and compaction; + migrate foreground callers only when slice B supplies visibility proof. + +Tests: equivalence with eager graph; duplicate OIDs; mismatched OID directory; +future/cyclic parents; malformed routing/hash/count/length; overflow/allocation +limits; cancellation during page fetch; compaction preserving positional identity. +Negative controls must make the corresponding regression fail. + +Verify: metadata commands V1/V2 below. Done: point lookup instrumentation shows +only the bounded descriptor/directory and requested pages; no historical layer +decode. Codec-only success does not complete the graph/opening milestone. + +### B. Replace full reachability materialization with explicit proof + +Scope: metadata graph builder and `crab-remote-git` graph, visibility, repository, +revision, operation and snapshot consumers; shared write publication. + +1. Prototype an immutable ancestor-membership set per commit/tip, keyed by ordinal. + Use authenticated copy-on-write radix/bitmap pages and structural sharing: + membership(child) = union(membership(parents)) + child. Reuse identical pages + and short-circuit identical unions. Represent shallow/external boundaries + explicitly; absent closure is not proof of non-reachability. +2. Build sets from verified Git DAGs. Bind each root to graph positional identity, + tip OID and coverage. Full scanner verifies the set/DAG recurrence before + trusted publication. Reader verifies selected membership page chains. +3. Capture current visible tips and activations, peel annotated tags through the + existing verified object path, then check candidate membership in those roots. + A retained old tip must remain rejected after rewind/force-push. +4. Replace `pinned_reachable: Vec` and synchronous graph-dependent readers + with batched async operations. Keep complete traversal output-sized when asked + for complete ancestry, fetch negotiation or shallow output. + +This representation is a design candidate with a mandatory build-cost gate. +Persistent sets can amplify storage on merge-heavy/disconnected DAGs; do not hide +that cost. Measure linear, branch-heavy and criss-cross merge fixtures. If sharing +does not bound incremental cost acceptably, stop this slice and revise the proof +representation. Proposed prototype rejection gate: on fixed-shape fixtures at +1k/2k/4k/8k commits, doubling history must not exceed 3× total membership-index +bytes or verified build CPU at the two largest sizes. Freeze the fixture and +limits before coding; report absolute costs too. This detects quadratic growth, +but is not a worst-case storage-complexity proof. Do not substitute an unbounded +foreground DFS or Bloom filter. + +Tests: every ancestry answer against native `git merge-base --is-ancestor`; +retained/invisible commits; both merge parents; duplicate candidates; tags; +shallow boundaries; forged membership; stale root; interrupted publication. +Verify V1/V3 plus RustFS visibility transition tests. Done: fixed membership +lookups avoid full graph traversal; build bytes/time/amplification are reported. + +### C. Remove eager repository control opening + +Scope: `crab-read/src/capsule_protocol*`, metadata capsule/checkpoint/control +shapes, `crab-remote-git/src/repository.rs`, shared writers and locator consumers. + +1. Reuse `open_ref_view_from_root_for_refs` for operations naming refs. Preserve + exact transaction/activation checks; never treat this partial view as global. +2. Add authenticated bounded directories for capsule control dependencies and + inventory/index roots. Read footer/control ranges without retaining complete + pre-checkpoint capsules. Both frontier and compacted opening must be bounded. +3. Bound the number of delta runs inspected by a point read. Compact under the + existing owner and CAS boundaries; use explicit admission/backpressure when + the bound cannot be maintained. Do not silently exceed limits. +4. Wire graph/path opening to these roots. Preserve verified pack locator lookup, + pack ranges and Git object hash checks. Explicitly finish every operation and + close all locator/SlateDB instances on success, error and cancellation. +5. For the broader goal of ref-count-independent arbitrary-OID reads, create a + separately reviewed protocol slice: authenticated visibility directory plus + an atomic publication fence tying it to ref positions/activations. Model + concurrent disjoint-ref writers, force-push and multi-ref transactions first. + Existing independently mutable heads cannot validate a stale aggregate root + without additional work. Do not enable this path before its authority, + migration and writer-contention gates pass. Until then, report the remaining + O(visible refs) capture cost as an open goal, not a completed optimization. + +Tests: pre-checkpoint and compacted sources; named refs among many unrelated refs; +partial multi-ref activation; deleted refs; history rewind; long frontier; +cancelled opening; malformed footer/root; zero unfinished work after close. +Verify V3/V4 and read scaling below. Done: selected opening has no full graph, +catalog or capsule decode. Report complete-ref enumeration separately and expose +any remaining arbitrary-OID ref-count cost; do not conceal it in setup time. + +### D. Use change points for file history and first-parent blame + +Scope: `crab-remote-git/src/snapshot.rs`, path-state query, traversal cursors, +runtime caches, object-reader call sites and native oracle fixtures. + +1. Find the latest path change at the selected commit. Jump to that change's + first parent and query its preceding change. Keep path/mode changes distinct + from content changes; handle deletion, recreation and file/tree replacement. +2. History returns requested change records with a cursor bound to snapshot, + raw path and semantics. A small history page never walks unrelated commits. +3. Blame reads the selected blob, carries unresolved line mappings through each + relevant blob comparison, and stops at origins or explicit boundaries. + Identical blobs skip comparison while preserving origin semantics. +4. Reuse existing bounded result caches. Add comparison reuse only for measured + repeated pairs; key by parent/child blob OIDs, algorithm/options version and + identity scope. Revalidate visibility before returning cached results. Charge + warm output/work and preserve complete-or-error resource behavior. + +Tests: current first-parent semantics and matching native Git options on linear +and merge fixtures; repeated lines; empty files; no final newline; mode-only +changes; recreation; UTF-8/binary behavior; cursor/cache identity; cancellation. +Native Git follows whole-file renames automatically; first-parent fixed-path +behavior is not default-blame parity. Reference: https://git-scm.com/docs/git-blame + +Verify V3/V4 and 20-sample scaling. Done: fixed 20-edit file with growing unrelated +history visits bounded relevant changes; all origins match the selected semantics. + +### E. Index historical Xet catalog through publication and hydration + +Scope: metadata capsule `pointer.rs` and checkpoint/run descriptors; read dependency +proof, term resolution/hydration; shared write/checkpoint/compaction; CLI +`git/xet_publication.rs`; protected receive, fsck, GC and SDK/VFS consumers. + +1. Implement authenticated indexes for file hash→binding, chunk hash→placement, + shard hash→dependencies and xorb hash→descriptor. Paginate variable recipes + and chunk tables too; a single 10-GiB recipe must not require a huge leaf. +2. Store indexed checkpoint plus visible immutable delta runs. Use a bounded + leveled manifest with key ranges and compaction; cap overlapping runs per + lookup. An index on every delta without bounded run count is insufficient. +3. Batch outgoing file hashes in `prepare_delta`, then stream recipe pages and + batch only their distinct needed chunk hashes. Fetch referenced descriptors, + validate placements/sizes/hashes and preserve file-binding precedence. + Do not rebuild `view.pointer_catalog()` on the normal push/read path. +4. Construct the new delta from verified selected closure. Origin-verify new + payloads and pages before capsule/ref publication. Preserve external-chunk + authority and protected receive checks. Stream full closure for fsck/GC. +5. Switch hydration, protected receive, maintenance and sibling consumers in the + same format migration; do not leave one path with an unchecked old authority. + +Tests: indexed lookup versus `PointerCatalog::apply`; duplicate chunks/occurrences; +mapping replacement with same size; conflicting sizes/xorb/shard descriptors; +missing dependencies; corrupted range; CAS retry; compaction with pinned readers. +Verify V1/V5/V6, then the 10-GiB harness. Done: fixed recipe/edit does not decode +unrelated catalog history; every cold historical version reconstructs byte-for-byte +with all occurrence positions preserved. Count recipe-sized work honestly. + +### F. Unify native-add and Git-clean staging mechanics + +Scope: `crab-staging/src/stream.rs`, recipe, index, batch/lease and push-plan owners; +CLI `cmd/add.rs`, `git/clean.rs`, `git/filter_process.rs`; relevant push consumers. + +1. Inventory native and filter lifecycle states, durable payload references, + prepared xorbs, provisional file identity and recipe leases. Add crossover + tests before deleting either loop. Both paths already share `StagingArea`; + sharing its type alone does not unify payload storage or plan ownership. +2. Refactor the existing streaming engine to accept a bounded byte source/sink + without requiring seek, reopening a path or trusting stat metadata. Native + source retains descriptor stability checks; filter source is the exact packet + stream Git supplies. Handle backpressure outside locks; preserve CPU admission. +3. One canonical engine owns CDC, hashes, chunk occurrence recording, payload + deduplication and recipe/prepared-plan creation. A chunk may have one payload + location and many file-version occurrences. Keep every occurrence in recipes. +4. Native policy owns its multi-file batch and rollback; clean owns one file and + emits a pointer only after its durable recipe/payload barrier. The engine + returns a durable staged version, not a pointer before durability. +5. Native publication writes Git ODB/index; clean returns bytes to Git. Since clean + cannot know whether Git later writes its index successfully, retain the durable + lease until existing reconciliation establishes release safety. A failed native + batch releases only its leases, never payload still referenced by clean/history. +6. Adopt reusable prepared plans across both paths. Verify streamed input hashes + even when reusing a recipe; a path/stat match is not content proof. Delete + duplicated clean/native chunking policy and synthetic identity plumbing once + crossover and failure tests pass. Do not spool a second complete file to adapt + clean to a path-only API. + +Tests: native→clean→native and clean→native→clean on real 10-GiB files; multiple +small edits; repeated/zero/random chunks; mid-read cancellation; fsync failure; +native Git-index failure; filter termination; old committed version after rollback; +concurrent paths sharing chunks; recipe and chunking-policy identity conflicts. +Exercise both incompressible deterministic content and sparse/repeated content; +prove 10 GiB of logical bytes passed through hashing/reconstruction. A file length +or sparse allocation measurement alone does not qualify the large-file pipeline. +Verify V6/V7 and large-file E2E. Done: switching paths creates no second full local +payload representation; each pointer has durable reconstruction; rollback and +restart preserve published versions. Separately report hash/CDC CPU, local bytes, +fsyncs, planning and upload so add latency reduction is measured, not assumed. + +### G. Qualify lifecycle, migration and provider behavior together + +Scope: touched readers/writers, `cmd/gc`, fsck/doctor/repack, retained roots, +maintenance checkpoints and dedicated CI/E2E fixtures. + +1. GC traces every retained descriptor/page and payload dependency, including + reader-pinned roots and resumable builds; grace remains enforced. Use existing + GC protection mechanism, not local Arc lifetime as cloud retention proof. +2. Inject process termination at each durability/publication boundary. Verify + acknowledged state reconstructs, partial state is not visible, builders resume, + lock owners release/recover and stale publishers cannot promote readiness. +3. Hold readers while concurrent pushes, ref rewind, compaction and GC execute. + Confirm stable captured results; new readers observe new complete state. +4. Audit exact released contracts. Rebuild derived browse data; migrate any changed + authoritative catalog shape explicitly. Test representative tagged repositories + and interrupted/resumed migration. No indefinite full-scan runtime reader. +5. Run RustFS correctness and representative S3 latency/conditional-write cells. + Missing cloud credentials or infrastructure are a named open gate, never a pass. + +Verify full changed-crate CI, native HTTP/process oracles, provider fault tests and +the matrix below. Done: receipts bind source, binary, provider, fixtures and failed +negative controls; all required cells pass without ignored credential-dependent proof. + +### H. Extend blame semantics after efficient history is stable + +Scope: remote Git blame types/algorithm/cache identity, HTTP attribution consumers +and native line-porcelain fixtures. Preserve the performance contract from D. + +Implement in separate PRs: source path/source line positions; multiple-parent line +ownership; whole-file rename following; each explicitly supported whitespace and +ignored-revision option. Account for rename candidate work; expose defined budget +errors rather than silently dropping origins. Move/copy detection is a separately +scoped capability. Version semantics and caches whenever results can change. + +Verify: native `git blame --line-porcelain` under identical options on merges, +renames, repeated lines and delete/recreate fixtures. Compare origin OID, original +path, source line, final line and displayed author data. Done: supported semantics +have exact oracle parity and documented unsupported options; no blanket Git parity +claim from first-parent tests. + +## Commands and verification gates + +Set the external target on EVERY compiling invocation. Before long compilation: + +```sh +test -d "$HOME/Workspace" && test -w "$HOME/Workspace" +mkdir -p "$HOME/Workspace/crabbuild-target/crab-ios-abf8fc00-push-audit" +export CARGO_TARGET_DIR="$HOME/Workspace/crabbuild-target/crab-ios-abf8fc00-push-audit" +``` + +The export applies only to this shell. New tools/shells require it again. Another +worktree requires its own directory. Cargo here is `/Users/haipingfu/.cargo/bin/cargo`. + +| Gate | Command, after setting external target | Required result | +| --- | --- | --- | +| V1 | `cargo test -p crab-metadata --features storage --locked` | Relevant codec/page/property tests execute and pass | +| V2 | `cargo clippy -p crab-metadata --features storage --lib --locked -- -D warnings` | No new warnings; payload-only feature compilation also passes | +| V3 | `cargo test -p crab-remote-git --locked` | Visibility, traversal, history, blame and cleanup regressions execute and pass | +| V4 | `cargo test -p crab-read -p crab-remote --features crab-remote/publication --locked` | Capture/opening/publication/dependency tests pass | +| V5 | `cargo test -p crab-write --locked` | Catalog/checkpoint/compaction tests pass | +| V6 | `cargo test -p crab-staging --locked` | Recipe/lease/rollback/durability tests pass | +| V7 | `cargo test -p crab --locked` | Native add, clean/filter and push regressions pass | +| Build | `cargo build -p crab-remote --features publication --release --examples --locked` | Qualifier examples build | +| Install CLI | `make -C crab CARGO=/Users/haipingfu/.cargo/bin/cargo install` | Builds/installs via external target; inspect Makefile artifact lookup first | +| Format | `cargo fmt --all -- --check` | Exit zero for touched source before commit | + +Use the absolute Cargo executable if PATH lacks cargo. Relevant small tests locally; +full/broad suites in CI/dedicated environment. Do not count a filter running zero +tests as proof. Record unrelated baseline lint failures separately; do not suppress +source lints to obtain green output. Python verifier comparisons remain active under +`python -O`. Existing HTTP oracles: `crates/crab-http-server/tests/verify_live.py` +and `verify_process.py`; inspect their CLI before running dedicated-server fixtures. + +Read performance, existing harness (replace dedicated bucket/run placeholders): + +```sh +python3 crab/scripts/e2e/run_remote_read_scaling_rustfs.py \ + --root "$HOME/Workspace/CrabData/indexed-read-abf8fc00" \ + --endpoint-url http://127.0.0.1:19090 \ + --bucket --run-id \ + --crab-bin "$CARGO_TARGET_DIR/release/crab" \ + --qualifier "$CARGO_TARGET_DIR/release/examples/qualify_remote" \ + --history 100 1000 10000 --repeats 20 + +python3 crab/scripts/e2e/run_large_file_edits_rustfs.py \ + --root "$HOME/Workspace/CrabData/indexed-read-abf8fc00" \ + --endpoint-url http://127.0.0.1:19090 \ + --bucket --run-id \ + --crab-bin "$CARGO_TARGET_DIR/release/crab" --file-mib 10240 +``` + +Do not reuse another chat's server/dataset or overwrite a receipt. Source external +qualification repositories are read-only. Keep large fixtures, build artifacts and +raw receipts on the workspace volume. Never print ambient credentials. + +## Final qualification matrix + +| Fixed workload | Grow independently | Acceptance | +| --- | --- | --- | +| One commit/object batch | Graph commits and repository objects | Bounded pages, exact Git object hash | +| 100-entry directory attribution | Unrelated commits/paths | ≤512 KiB decoded and ≤1 MiB query bytes at 10k fixture | +| One selected ref | Unrelated ref count | No unchecked sibling-ref/control capture | +| Arbitrary-OID visibility | Visible refs and disconnected history | Exact membership; report remaining ref-count cost | +| File with 20 edits | Unrelated commits | Change-point work, exact selected blame semantics | +| Same text size | Actual content edits/merges | Report unavoidable comparison/build growth | +| Fixed 10-GiB recipe/edit | Distinct historical chunks | No proportional historical catalog decode | +| Point reads | Uncompacted frontier/runs | Enforced lookup amplification bound | +| Captured reader | Concurrent push/rebuild/compaction/GC | No mixed snapshot; no deleted required dependency | +| Native/clean crossover | Number of edits and shared versions | No duplicate full payload; safe lease/rollback | + +Measure opening, query and maintenance separately. Include cold/warm p50/p95, +HTTP request/bytes by object class, page decodes, CPU, RSS, local write amplification, +index build/compaction time and storage amplification. At least 20 independent +samples per final latency cell. For the frozen local fixture, largest-history +attribution/blame cold p95 must be ≤3× smallest-history p95 +20 ms. Establish cloud +targets from cloud baseline first; local RustFS timing is not cloud proof. + +## Stop conditions and review checklist + +- If a new index changes visibility/ref transaction authority, isolate the protocol + decision and prove publication ordering before implementation proceeds. +- If membership sets or compaction exceed measured build/storage/admission budgets, + revise the representation; do not move unlimited work into opening or hide it. +- If a shipped format is found, define and test its migration boundary before removal. +- If a common staging engine needs a second full-file spool or weakens rollback, + revise the source/sink and ownership boundary before switching callers. +- If a proof needs secrets, cloud access or destructive external mutation unavailable + to this task, record the precise missing gate and continue independent work. + +Before each PR verdict ask: **Is this the best fix, or only a plausible fix?** +Evidence map must include entry point, owner, caller, callee, sibling consumers, +tests, released contract and base behavior. Explain non-test LOC growth. No stage +is complete from codec tests alone; require user operation → actual store/Git side +effect → independently verified visible result. The full initiative is complete +only after all applicable slices and qualification cells are proven. diff --git a/advisor-plans/README.md b/advisor-plans/README.md index 6d7c884db..27b3b88c3 100644 --- a/advisor-plans/README.md +++ b/advisor-plans/README.md @@ -1,5 +1,17 @@ # Crate quality implementation plans +## Indexed remote reads and unified staging + +Planned on 2026-10-13 against `51435f76bd2`. [037](037-indexed-remote-reads-and-unified-staging.md) +specifies the remaining graph/visibility/opening, change-point history, historical +Xet catalog, staging ownership and qualification work. It consumes the existing +path-state implementation instead of rebuilding it. Execute its named slices in +dependency order; update the canonical indexed-read ledger with actual proof. + +| Plan | Priority | Effort | Depends on | Status | +| --- | --- | --- | --- | --- | +| [037](037-indexed-remote-reads-and-unified-staging.md) | P1 performance / P0 correctness gates | XL | Indexed-read stages 1–3; existing recipe/lease authority | IN PROGRESS — paged graph producer/component reader qualified; visibility/opening, stages 5–8 and staging remain open | + ## crab-ltx foundation hardening Planned on 2026-09-25 against `3b8d3b3614c`. [035](035-crab-ltx-foundation-durability-and-performance.md) From 9797a798a5ec1dd0cc7fff8909c21d6f31db6d82 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 15:18:20 -0700 Subject: [PATCH 008/127] feat(metadata)!: page capsule browse graphs Keep released manifest encoding; page capsule lookup dependencies. BREAKING CHANGE: Rust descriptor literals require optional page and ordinal-identity fields. Unshipped path page refs use the shared PageRef type. Version-1 stored graph bytes remain unchanged. --- .../docs/architecture/indexed-remote-reads.md | 64 ++- crab/scripts/e2e/verify_remote_browse.py | 14 + crates/crab-metadata/README.md | 16 + crates/crab-metadata/src/lib.rs | 2 + .../src/{path_state/paged.rs => pages.rs} | 137 +++--- crates/crab-metadata/src/pages/cache.rs | 93 +++++ crates/crab-metadata/src/path_state.rs | 16 +- crates/crab-metadata/src/path_state/codec.rs | 9 +- crates/crab-metadata/src/path_state/reader.rs | 106 +---- crates/crab-metadata/src/path_state/scan.rs | 8 +- .../crab-metadata/src/split_commit_graph.rs | 136 ++++-- .../src/split_commit_graph/paged.rs | 164 ++++++++ .../src/split_commit_graph/paged_tests.rs | 393 ++++++++++++++++++ .../src/split_commit_graph/reader.rs | 291 +++++++++++++ crates/crab-remote-git/src/commit_graph.rs | 2 + crates/crab-remote/README.md | 5 + crates/crab-remote/examples/qualify_remote.rs | 108 ++++- crates/crab-write/README.md | 8 +- crates/crab-write/src/generation/browse.rs | 6 +- 19 files changed, 1386 insertions(+), 192 deletions(-) rename crates/crab-metadata/src/{path_state/paged.rs => pages.rs} (84%) create mode 100644 crates/crab-metadata/src/pages/cache.rs create mode 100644 crates/crab-metadata/src/split_commit_graph/paged.rs create mode 100644 crates/crab-metadata/src/split_commit_graph/paged_tests.rs create mode 100644 crates/crab-metadata/src/split_commit_graph/reader.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index c6705a208..bf7ba469d 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -29,7 +29,7 @@ Xet files and unified local staging are subsequent workstreams. | 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | | 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Implemented; 23 focused tests, GC retention regression and 1,000/10,000-commit RustFS format qualification passed | | 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Canonical scaling/transitions passed; streaming Cell ingestion, projected native Git oracle and two process starts passed on RustFS; broader cloud/lifecycle qualification remains open | -| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Pending | +| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Paged producer/component reader implemented; explicit visibility and bounded foreground opening remain pending | | 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | | 6 | Indexed historical Xet catalog | Indexed checkpoints and visible deltas; bounded compaction; 10-GiB edits and byte-identical cold historical hydration; protected receive and sibling consumers | Pending | | 7 | Lifecycle and migration | Interrupted publication/builds, concurrent owners, GC retained roots/readers, stale caches and released-format migration qualification | Pending; also required in every earlier implementation | @@ -55,7 +55,8 @@ SST/manifest access. It reports logical backend invocations and consumed bytes; provider HTTP retries and listing pages require the separate HTTP meter. Decoder observations count verified graph/path-state blocks and their encoded input bytes. The frozen baseline counted layers. The paged path-state format counts individual -verified pages; the graph still counts layers. Do not relabel decoded bytes as +verified pages; paged graphs count verified pages too, while the released eager +graph format counts layers. Do not relabel decoded bytes as network transfer or resident memory. Decode events belong at the actual decoding boundary. @@ -174,6 +175,16 @@ alone is not proof that every related serialized shape is unshipped. ## Ownership and consumer coverage +The released graph producer/consumer audit also reads CLI push's +`load_base_split_commit_graph`, remote-helper manifest loading and the published +metadata crate manifest at upstream v1.2.4. Both consumers bind the graph to the +manifest generation, pack index, validation digest and committed roots. Manifest +maintenance retains its version-1 encoding. Capsule browse has no released graph +contract and now writes paged descriptor version 2 through its generation owner. +Public Rust descriptor structs gain optional identity/page fields; workspace +literal consumers are updated, while version-1 serialization omits the additions. +Published Rust API versioning remains part of the release gate. + - `crab-metadata`: canonical codecs, index formats, validation and visibility. - `crab-storage`: range I/O, retries and backend observation. - `crab-read`: captured capsule views, Xet dependency resolution and hydration. @@ -319,3 +330,52 @@ push passed afterward. These transitions qualify canonical reads; configured Cell projection has the separate ingestion/restart proof above and owner tests cover the same rewind visibility boundary. Concurrent retained readers during compaction/GC and broader interrupted-build qualification remain later gates. + +## Paged graph producer and component qualification + +Capsule browse construction writes authenticated ordinal-record and OID-directory +pages, with the existing positional digest in its descriptor. Maintenance full +loading checks that digest, OID/record agreement, parent ordering and corrected +generations, layer hashes and contiguous physical page coverage. Unchanged layers +are reused; carries also enforce the 32-layer directory bound for decreasing +append sizes. Removing this carry reproduces a rejected 33-layer descriptor in +the regression test; restoring it passes. The source was restored before final +qualification. Lazy graph loading rejects the released eager format rather than +falling back to it. The full loader retains that format for the shipped manifest +protocol and for owner-controlled rebuilding of unshipped derived browse data. + +Graph and path-state readers share the page codec and bounded cache. Point and +batch graph queries retain input duplicates, validate selected OID/ordinal +agreement and parent generations, and charge unique page bytes before I/O or +cache lookup. Cancellation may discard a query without advancing persistent +reader state. Record presence supplies no visibility proof. + +Focused proof: 19 graph tests, 21 path-state tests and five shared-codec properties +pass; a ten-test page filter overlaps those slices. Capsule browse publication +and non-commit-tag maintenance pass the existing integration test. Metadata +payload-only compilation and strict storage-feature library Clippy pass; the +remote qualifier diagnostic Clippy run permits only the existing coordinator +`unfulfilled-lint-expectations` baseline. Remote-Git test targets compile, Python +scaling tests pass and architecture guardrails pass. The native verifier rejects +a deliberately wrong graph tree with Python optimization enabled. + +Final-source RustFS `paged-graph-r2` reuses the immutable baseline source fixtures. +At 100/1,000/10,000 commits, twenty isolated processes per size each compare cold +and warm sampled ordinal/parent records and duplicate OID batches with native Git. +All 60 comparisons and separate attribution comparisons pass. Opening captures +complete current controls and is reported separately; these component measurements +are not a bounded-opening or release-image qualification. + +| Commits | Cold graph component median | Cold p95 | Maximum query backend/decoded bytes | Maximum warm query backend bytes | +| --- | --- | --- | --- | --- | +| 100 | 1.916 ms | 6.994 ms | 10,024 | 0 | +| 1,000 | 18.653 ms | 55.610 ms | 100,237 | 0 | +| 10,000 | 8.698 ms | 24.208 ms | 166,656 | 0 | + +This debug-host run has substantial latency variation, including the 1,000-commit +cell; deterministic page/work limits are the format proof. Final foreground, +process-resource, release-image and representative S3 latency gates remain open. +Receipts, failed negative controls, source hashes and binary hashes remain outside +the checkout. Stage 4 still requires explicit visible ancestry proof, lazy +foreground conversion, removal of full positional-digest work and bounded control +opening. Stages 5–8 remain incomplete. diff --git a/crab/scripts/e2e/verify_remote_browse.py b/crab/scripts/e2e/verify_remote_browse.py index 9c7979c34..a07fd2d9f 100644 --- a/crab/scripts/e2e/verify_remote_browse.py +++ b/crab/scripts/e2e/verify_remote_browse.py @@ -100,6 +100,20 @@ def check_object(value: dict) -> None: require(item["commit"] == expected, "last-change commit differs") elif name == "blame": lines_checked += check_line_origins(git, head, path, value["ranges"]) + elif name == "graph-pages": + rows = value["records"] + require(len(rows) == 4 and rows[1] == rows[3], "graph batch order/duplicates differ") + for item in rows: + body = git("cat-file", "commit", item["oid"]) + headers = body.split(b"\n\n", 1)[0].splitlines() + tree = next(line[5:].decode() for line in headers if line.startswith(b"tree ")) + parents = [line[7:].decode() for line in headers if line.startswith(b"parent ")] + committer = next(line for line in headers if line.startswith(b"committer ")) + require(item["tree"] == tree and item["parents"] == parents + and item["committer_seconds"] == int(committer.rsplit(b" ", 2)[1]), + "graph record differs from native Git") + else: + raise AssertionError("unverified qualification operation: " + name) return {"status":"passed", "head":head, "queries_checked":len(results), "blame_lines_checked":lines_checked, "measurements":measurements} diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 569143dd9..09218deaf 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -61,6 +61,22 @@ budget before buffering, then admit each layer against its authenticated size and the remaining aggregate budget. All bodies still require hash and exact length verification; malformed provider sizes cannot bypass the encoded-byte ceiling. Decoded index structures require separate memory qualification. +Capsule browse construction writes split-graph descriptor version 2, with +authenticated ordinal-record and OID-directory pages and a verified positional +digest. Its carry rule caps the layer directory at 32 entries even for decreasing +append batch sizes; internal page directories remain bounded. `CommitGraphReader` +loads only the descriptor and dependency ranges, preserves duplicate query +positions and validates selected parent ordering/corrected generations. It does +not grant ref visibility. Complete maintenance loading verifies OID/record +agreement, layer hashes, positional digest and physical byte coverage. +The released manifest producer/consumer path retains version 1 and its exact +encoding: upstream v1.0.1 through v1.2.4 publish `Manifest::commit_graph_hash` from +CLI push and read it in the remote helper. Lazy capsule readers reject that +format; maintenance may rebuild the unreleased eager browse index into pages. +Path and graph readers share the authenticated page codec and an 8-MiB/256-page +cache; each cache belongs to one immutable captured layer directory. The current +foreground graph handshake still materializes reachability and is not yet a +bounded-opening implementation. Path-state descriptor version 5 requires exact-name tree comparison semantics and authenticated pages; retired descriptors are rejected. Maintenance rebuilds completed indexes and resumable checkpoints through the diff --git a/crates/crab-metadata/src/lib.rs b/crates/crab-metadata/src/lib.rs index 24a2445b9..b12b656f4 100644 --- a/crates/crab-metadata/src/lib.rs +++ b/crates/crab-metadata/src/lib.rs @@ -28,6 +28,8 @@ pub mod manifests; pub mod pack_metadata; #[cfg(feature = "storage")] pub mod pack_origin; +mod pages; +pub use pages::PageRef; pub mod path_state; #[cfg(feature = "local-index")] pub mod persistent_chunk_index; diff --git a/crates/crab-metadata/src/path_state/paged.rs b/crates/crab-metadata/src/pages.rs similarity index 84% rename from crates/crab-metadata/src/path_state/paged.rs rename to crates/crab-metadata/src/pages.rs index 1997c67cc..522d53847 100644 --- a/crates/crab-metadata/src/path_state/paged.rs +++ b/crates/crab-metadata/src/pages.rs @@ -1,20 +1,34 @@ -//! Authenticated, bounded trees inside one immutable path-state layer. +//! Authenticated bounded pages shared by immutable metadata indexes. use serde::{Deserialize, Serialize}; +#[cfg(feature = "storage")] +mod cache; +#[cfg(feature = "storage")] +pub(crate) use cache::PageCache; + +use crate::error::{MetadataError, Result}; +#[cfg(any(feature = "storage", test))] +use crate::validation::corrupt_object; + +fn corruption(reason: &str) -> MetadataError { + crate::validation::corrupt_object("metadata page", reason) +} + #[cfg(any(feature = "storage", test))] -use super::corrupt_at; -use super::{Result, corruption}; +fn corrupt_at(path: &str, reason: &str) -> Result { + Err(corrupt_object(path, reason)) +} const MAGIC: &[u8; 8] = b"CRABPP01"; const HEADER: usize = 14; -pub(super) const TARGET_BYTES: usize = 32 * 1024; -pub(super) const MAX_BYTES: usize = 2 * 1024 * 1024; +pub(crate) const TARGET_BYTES: usize = 32 * 1024; +pub(crate) const MAX_BYTES: usize = 2 * 1024 * 1024; const MAX_HEIGHT: u8 = 16; /// Authenticated location and key coverage of one immutable page. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct PathStatePageRef { +pub struct PageRef { pub offset: u64, pub bytes: u32, pub hash: [u8; 32], @@ -24,40 +38,37 @@ pub struct PathStatePageRef { pub last: Vec, } -/// Bounded page roots committed by an immutable layer descriptor. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PathStateLayerPages { - pub records: PathStatePageRef, - pub nodes: PathStatePageRef, - pub names: Option, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] -pub(super) enum Kind { +pub(crate) enum Kind { Records, Nodes, Children, Names, + CommitRecords, + CommitOids, } impl Kind { - pub(super) fn key_bytes(self) -> usize { - if self == Self::Names { 32 } else { 4 } + pub(crate) fn key_bytes(self) -> usize { + match self { + Self::Names => 32, + Self::CommitOids => 20, + _ => 4, + } } } -pub(super) type Rows = Vec<(Vec, Vec)>; +pub(crate) type Rows = Vec<(Vec, Vec)>; #[cfg(any(feature = "storage", test))] #[derive(Debug)] -pub(super) enum Page { +pub(crate) enum Page { Leaf(Rows), - Branch(Vec), + Branch(Vec), } -impl PathStatePageRef { - pub(super) fn validate(&self, kind: Kind, source_bytes: u64) -> Result<()> { +impl PageRef { + pub(crate) fn validate(&self, kind: Kind, source_bytes: u64) -> Result<()> { if self.first.len() != kind.key_bytes() || self.last.len() != kind.key_bytes() || self.first > self.last @@ -70,9 +81,9 @@ impl PathStatePageRef { .checked_add(u64::from(self.bytes)) .is_none_or(|end| end > source_bytes) { - return Err(corruption("invalid path-state page reference")); + return Err(corruption("invalid metadata page reference")); } - if matches!(kind, Kind::Records | Kind::Nodes) { + if matches!(kind, Kind::Records | Kind::Nodes | Kind::CommitRecords) { let first = integer(&self.first)?; let last = integer(&self.last)?; if last.checked_sub(first).and_then(|span| span.checked_add(1)) != Some(self.count) { @@ -84,13 +95,13 @@ impl PathStatePageRef { } #[derive(Default)] -pub(super) struct Writer { - pub(super) bytes: Vec, - pages: std::collections::BTreeMap<[u8; 32], PathStatePageRef>, +pub(crate) struct Writer { + pub(crate) bytes: Vec, + pages: std::collections::BTreeMap<[u8; 32], PageRef>, } impl Writer { - pub(super) fn tree(&mut self, kind: Kind, rows: Rows) -> Result { + pub(crate) fn tree(&mut self, kind: Kind, rows: Rows) -> Result { validate_rows(kind, &rows)?; let mut level = Vec::new(); let mut pending = Vec::new(); @@ -100,9 +111,9 @@ impl Writer { .key_bytes() .checked_add(4) .and_then(|n| n.checked_add(row.1.len())) - .ok_or_else(|| corruption("path-state row size overflows"))?; + .ok_or_else(|| corruption("metadata row size overflows"))?; if row_bytes > MAX_BYTES - HEADER { - return Err(corruption("path-state row exceeds the page bound")); + return Err(corruption("metadata row exceeds the page bound")); } if !pending.is_empty() && size + row_bytes > TARGET_BYTES { level.push(self.leaf(kind, std::mem::take(&mut pending))?); @@ -124,10 +135,10 @@ impl Writer { } level .pop() - .ok_or_else(|| corruption("empty path-state page tree")) + .ok_or_else(|| corruption("empty metadata page tree")) } - fn leaf(&mut self, kind: Kind, rows: Rows) -> Result { + fn leaf(&mut self, kind: Kind, rows: Rows) -> Result { let first = rows .first() .ok_or_else(|| corruption("empty page"))? @@ -149,7 +160,7 @@ impl Writer { self.append(bytes, 0, count, first, last) } - fn branch(&mut self, kind: Kind, children: &[PathStatePageRef]) -> Result { + fn branch(&mut self, kind: Kind, children: &[PageRef]) -> Result { let first = children .first() .ok_or_else(|| corruption("empty directory"))?; @@ -184,7 +195,7 @@ impl Writer { count: u32, first: Vec, last: Vec, - ) -> Result { + ) -> Result { if bytes.len() > MAX_BYTES { return Err(corruption("page exceeds its byte bound")); } @@ -192,7 +203,7 @@ impl Writer { if let Some(reference) = self.pages.get(&hash) { return Ok(reference.clone()); } - let reference = PathStatePageRef { + let reference = PageRef { offset: self.bytes.len() as u64, bytes: bytes.len() as u32, hash, @@ -215,9 +226,9 @@ fn header(kind: Kind, height: u8, count: u32) -> Vec { } #[cfg(any(feature = "storage", test))] -pub(super) fn decode( +pub(crate) fn decode( bytes: &[u8], - reference: &PathStatePageRef, + reference: &PageRef, kind: Kind, source_bytes: u64, path: &str, @@ -229,12 +240,12 @@ pub(super) fn decode( || bytes.get(8) != Some(&(kind as u8)) || bytes.get(9) != Some(&reference.height) { - return corrupt_at(path, "path-state page identity or header mismatch"); + return corrupt_at(path, "metadata page identity or header mismatch"); } let mut input = Input::new(&bytes[10..], path); let count = input.u32()? as usize; if count == 0 { - return corrupt_at(path, "empty path-state page"); + return corrupt_at(path, "empty metadata page"); } let page = if reference.height == 0 { if count > input.remaining() / (kind.key_bytes() + 4) { @@ -259,7 +270,7 @@ pub(super) fn decode( if count > input.remaining() / ref_bytes(kind) { return corrupt_at(path, "directory count exceeds its encoded bytes"); } - let mut children: Vec = Vec::new(); + let mut children: Vec = Vec::new(); let mut total = 0u32; for _ in 0..count { let child = decode_ref(&mut input, kind)?; @@ -275,7 +286,7 @@ pub(super) fn decode( { return corrupt_at(path, "invalid page directory routing or ancestry"); } - if matches!(kind, Kind::Records | Kind::Nodes) + if matches!(kind, Kind::Records | Kind::Nodes | Kind::CommitRecords) && children.last().is_some_and(|previous| { integer(&previous.last) .ok() @@ -299,18 +310,18 @@ pub(super) fn decode( Page::Branch(children) }; input.finish()?; - tracing::debug!(target:"crab_metadata::index_decode", index_format="path_state", + tracing::debug!(target:"crab_metadata::index_decode", index_format=if matches!(kind, Kind::CommitRecords | Kind::CommitOids) { "commit_graph" } else { "path_state" }, decoded_bytes=bytes.len() as u64, "index page decoded"); Ok(page) } #[cfg(any(feature = "storage", test))] -pub(super) fn scan( +pub(crate) fn scan( source: &[u8], - root: &PathStatePageRef, + root: &PageRef, kind: Kind, path: &str, - visited: &mut std::collections::BTreeMap, + visited: &mut std::collections::BTreeMap, ) -> Result { let mut pending = vec![root.clone()]; let mut rows = Vec::new(); @@ -334,9 +345,9 @@ pub(super) fn scan( } #[cfg(any(feature = "storage", test))] -pub(super) fn page_bytes<'a>( +pub(crate) fn page_bytes<'a>( source: &'a [u8], - reference: &PathStatePageRef, + reference: &PageRef, path: &str, ) -> Result<&'a [u8]> { let start = usize::try_from(reference.offset) @@ -356,7 +367,7 @@ fn validate_rows(kind: Kind, rows: &Rows) -> Result<()> { { return Err(corruption("page keys are empty, invalid or unordered")); } - if matches!(kind, Kind::Records | Kind::Nodes) + if matches!(kind, Kind::Records | Kind::Nodes | Kind::CommitRecords) && rows.windows(2).any(|pair| { integer(&pair[0].0) .ok() @@ -380,7 +391,7 @@ fn ref_bytes(kind: Kind) -> usize { 49 + 2 * kind.key_bytes() } -pub(super) fn encode_ref(bytes: &mut Vec, reference: &PathStatePageRef) { +pub(crate) fn encode_ref(bytes: &mut Vec, reference: &PageRef) { bytes.extend_from_slice(&reference.offset.to_le_bytes()); bytes.extend_from_slice(&reference.bytes.to_le_bytes()); bytes.extend_from_slice(&reference.hash); @@ -391,8 +402,8 @@ pub(super) fn encode_ref(bytes: &mut Vec, reference: &PathStatePageRef) { } #[cfg(any(feature = "storage", test))] -pub(super) fn decode_ref(input: &mut Input<'_, '_>, kind: Kind) -> Result { - Ok(PathStatePageRef { +pub(crate) fn decode_ref(input: &mut Input<'_, '_>, kind: Kind) -> Result { + Ok(PageRef { offset: u64::from_le_bytes(input.array()?), bytes: input.u32()?, hash: input.array()?, @@ -403,7 +414,7 @@ pub(super) fn decode_ref(input: &mut Input<'_, '_>, kind: Kind) -> Result, value: usize) -> Result<()> { +pub(crate) fn put_u32(bytes: &mut Vec, value: usize) -> Result<()> { bytes.extend_from_slice( &u32::try_from(value) .map_err(|_| corruption("page scalar overflows"))? @@ -413,19 +424,19 @@ pub(super) fn put_u32(bytes: &mut Vec, value: usize) -> Result<()> { } #[cfg(any(feature = "storage", test))] -pub(super) struct Input<'a, 'p> { +pub(crate) struct Input<'a, 'p> { bytes: &'a [u8], path: &'p str, } #[cfg(any(feature = "storage", test))] impl<'a, 'p> Input<'a, 'p> { - pub(super) fn new(bytes: &'a [u8], path: &'p str) -> Self { + pub(crate) fn new(bytes: &'a [u8], path: &'p str) -> Self { Self { bytes, path } } - pub(super) fn remaining(&self) -> usize { + pub(crate) fn remaining(&self) -> usize { self.bytes.len() } - pub(super) fn take(&mut self, count: usize) -> Result<&'a [u8]> { + pub(crate) fn take(&mut self, count: usize) -> Result<&'a [u8]> { let (head, tail) = self .bytes .split_at_checked(count) @@ -433,15 +444,15 @@ impl<'a, 'p> Input<'a, 'p> { self.bytes = tail; Ok(head) } - pub(super) fn array(&mut self) -> Result<[u8; N]> { + pub(crate) fn array(&mut self) -> Result<[u8; N]> { self.take(N)? .try_into() .map_err(|_| corruption("invalid page scalar")) } - pub(super) fn u32(&mut self) -> Result { + pub(crate) fn u32(&mut self) -> Result { Ok(u32::from_le_bytes(self.array()?)) } - pub(super) fn finish(self) -> Result<()> { + pub(crate) fn finish(self) -> Result<()> { if !self.bytes.is_empty() { return corrupt_at(self.path, "page has trailing bytes"); } @@ -451,7 +462,7 @@ impl<'a, 'p> Input<'a, 'p> { #[cfg(any(feature = "storage", test))] impl Page { - pub(super) fn resident_bytes(&self) -> usize { + pub(crate) fn resident_bytes(&self) -> usize { match self { Self::Leaf(rows) => rows.iter().fold( rows.capacity() @@ -465,7 +476,7 @@ impl Page { Self::Branch(children) => children.iter().fold( children .capacity() - .saturating_mul(std::mem::size_of::()), + .saturating_mul(std::mem::size_of::()), |total, child| { total .saturating_add(child.first.capacity()) @@ -561,7 +572,7 @@ mod tests { invalid.first = 1u32.to_be_bytes().to_vec(); let mut damaged = body.clone(); damaged[HEADER] ^= 1; - let oversized = PathStatePageRef { + let oversized = PageRef { bytes: (MAX_BYTES + 1) as u32, ..root.clone() }; diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs new file mode 100644 index 000000000..761a48a20 --- /dev/null +++ b/crates/crab-metadata/src/pages/cache.rs @@ -0,0 +1,93 @@ +//! Each cache belongs to one captured immutable layer directory. +use super::*; +use crab_storage::Store; +use std::collections::{BTreeMap, VecDeque}; +use std::sync::{Arc, Mutex, MutexGuard}; + +const CACHE_BYTES: usize = 8 * 1024 * 1024; +const CACHE_PAGES: usize = 256; + +#[derive(Debug, Default)] +pub(crate) struct PageCache(Mutex); + +#[derive(Debug, Default)] +struct State { + pages: BTreeMap<(usize, u64), (Kind, PageRef, Arc)>, + order: VecDeque<(usize, u64)>, + bytes: usize, +} + +impl PageCache { + // Guards protect local cache work only, never origin I/O. + fn state(&self) -> Result> { + self.0 + .lock() + .map_err(|_| MetadataError::Internal("metadata page cache is poisoned".to_owned())) + } + pub(crate) async fn fetch( + &self, + store: &Store, + path: &object_store::path::Path, + layer: usize, + source_bytes: u64, + reference: &PageRef, + kind: Kind, + ) -> Result> { + reference.validate(kind, source_bytes)?; + let key = (layer, reference.offset); + if let Some((previous_kind, previous, page)) = self.state()?.pages.get(&key) { + if *previous_kind != kind || previous != reference { + return Err(corruption("cached page routing metadata changed")); + } + return Ok(page.clone()); + } + let bytes = store + .range_get( + path, + reference.offset..reference.offset + u64::from(reference.bytes), + ) + .await?; + let page = Arc::new(decode( + &bytes, + reference, + kind, + source_bytes, + path.as_ref(), + )?); + let weight = page_weight(reference, &page); + if weight <= CACHE_BYTES { + let mut cache = self.state()?; + if let Some((previous_kind, previous, existing)) = cache.pages.get(&key) { + if *previous_kind != kind || previous != reference { + return Err(corruption("concurrent page routing metadata changed")); + } + return Ok(existing.clone()); + } + while cache.bytes.saturating_add(weight) > CACHE_BYTES + || cache.pages.len() >= CACHE_PAGES + { + let Some(oldest) = cache.order.pop_front() else { + break; + }; + if let Some((_, reference, old)) = cache.pages.remove(&oldest) { + cache.bytes = cache.bytes.saturating_sub(page_weight(&reference, &old)); + } + } + cache.bytes += weight; + cache.order.push_back(key); + cache + .pages + .insert(key, (kind, reference.clone(), page.clone())); + } + Ok(page) + } +} + +fn page_weight(reference: &PageRef, page: &Page) -> usize { + page.resident_bytes() + .saturating_add(std::mem::size_of::()) + .saturating_add(std::mem::size_of::()) + .saturating_add(reference.first.capacity()) + .saturating_add(reference.last.capacity()) + .saturating_add(128) +} diff --git a/crates/crab-metadata/src/path_state.rs b/crates/crab-metadata/src/path_state.rs index ade719b29..250a8a49d 100644 --- a/crates/crab-metadata/src/path_state.rs +++ b/crates/crab-metadata/src/path_state.rs @@ -11,10 +11,20 @@ use crate::{ }; mod codec; -mod paged; +use crate::pages as paged; #[cfg(feature = "storage")] mod reader; -pub use paged::{PathStateLayerPages, PathStatePageRef}; +pub use crate::PageRef; + +/// Bounded page roots committed by an immutable layer descriptor. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PathStateLayerPages { + pub records: PageRef, + pub nodes: PageRef, + pub names: Option, +} + #[cfg(feature = "storage")] pub use reader::{PathStateItem, PathStateReader, PathStateScan}; #[cfg(feature = "storage")] @@ -1118,6 +1128,7 @@ mod tests { SplitCommitGraph::new( CommitGraphDescriptor { version: 1, + commit_ordinal_digest: None, generation: 7, pack_index_hash: "a".repeat(64), git_validation_digest: "b".repeat(64), @@ -1128,6 +1139,7 @@ mod tests { base_ordinal: 0, commit_count: u32::from(count), bytes: 24, + pages: None, }], }, vec![layer], diff --git a/crates/crab-metadata/src/path_state/codec.rs b/crates/crab-metadata/src/path_state/codec.rs index 39c837b6d..9ef1545d0 100644 --- a/crates/crab-metadata/src/path_state/codec.rs +++ b/crates/crab-metadata/src/path_state/codec.rs @@ -2,14 +2,15 @@ use std::collections::BTreeMap; #[cfg(any(feature = "storage", test))] use std::sync::Arc; -#[cfg(any(feature = "storage", test))] -use super::paged::Input; -use super::paged::{self, Kind, PathStateLayerPages, Writer}; +use super::PathStateLayerPages; use super::{ MAX_AUTHOR_BYTES, MAX_MESSAGE_BYTES, MAX_PATH_BYTES, PathStateLayer, Result, corruption, }; #[cfg(any(feature = "storage", test))] use super::{PathStateLayerRef, PathStateNode, PathStateNodeRef, PathStateRecord, corrupt_at}; +#[cfg(any(feature = "storage", test))] +use crate::pages::Input; +use crate::pages::{self as paged, Kind, Writer}; pub(super) struct EncodedLayer { pub(super) bytes: Vec, @@ -247,7 +248,7 @@ pub(super) fn decode_node( bytes: &[u8], source_bytes: u64, path: &str, -) -> Result<(Option, Option)> { +) -> Result<(Option, Option)> { let mut input = Input::new(bytes, path); let value = optional(input.u32()?); let children = match input.array::<1>()?[0] { diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index 2b1505a1c..cb9a7d2cc 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -1,22 +1,18 @@ //! Lazy lookups bound to an authenticated descriptor; full scans remain maintenance operations. use crab_storage::{Store, StoreLayout}; use futures_util::{StreamExt, TryStreamExt}; -use std::sync::{Mutex, MutexGuard}; use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet}, sync::Arc, }; -use super::paged::{self, Kind, Page, PathStatePageRef}; use super::{ PathStateDescriptor, PathStateNodeRef, PathStateRecord, PathStateSummary, Result, codec, corrupt, corruption, validate_path, }; +use crate::pages::{self as paged, Kind, Page, PageRef}; use crate::split_commit_graph::{CommitGraphLayerRef, SplitCommitGraph}; -const CACHE_BYTES: usize = 8 * 1024 * 1024; -const CACHE_PAGES: usize = 256; - #[path = "scan.rs"] mod scan; pub use scan::{PathStateItem, PathStateScan}; @@ -29,18 +25,11 @@ pub struct PathStateReader { paths: Vec, descriptor_bytes: u64, max_bytes: u64, - cache: Mutex, -} - -#[derive(Debug, Default)] -struct Cache { - pages: BTreeMap<(usize, u64), (Kind, PathStatePageRef, Arc)>, - order: VecDeque<(usize, u64)>, - bytes: usize, + cache: paged::PageCache, } struct ChildBatch { - root: PathStatePageRef, + root: PageRef, requests: Vec<(usize, u32)>, } @@ -78,7 +67,7 @@ impl PathStateReader { paths, descriptor_bytes, max_bytes, - cache: Mutex::new(Cache::default()), + cache: paged::PageCache::default(), }) } @@ -121,7 +110,7 @@ impl PathStateReader { let mut pending = (0..paths.len()).collect::>(); while !pending.is_empty() { let states = self.nodes(store, &nodes, &pending, &mut work).await?; - let mut components = BTreeMap::>::new(); + let mut components = BTreeMap::>::new(); for (&position, state) in pending.iter().zip(states) { let Some(component) = remaining[position].next() else { continue; @@ -282,7 +271,7 @@ impl PathStateReader { nodes: &[PathStateNodeRef], positions: &[usize], work: &mut Work, - ) -> Result, Option)>> { + ) -> Result, Option)>> { let mut grouped = BTreeMap::>::new(); for (output, &position) in positions.iter().enumerate() { let node = nodes[position]; @@ -352,7 +341,7 @@ impl PathStateReader { &self, store: &Store, layer: usize, - root: &PathStatePageRef, + root: &PageRef, kind: Kind, keys: &[Vec], work: &mut Work, @@ -408,13 +397,7 @@ impl PathStateReader { Ok(results) } - fn charge( - &self, - layer: usize, - reference: &PathStatePageRef, - kind: Kind, - work: &mut Work, - ) -> Result<()> { + fn charge(&self, layer: usize, reference: &PageRef, kind: Kind, work: &mut Work) -> Result<()> { let source_bytes = self.descriptor.layers[layer].bytes; reference.validate(kind, source_bytes)?; let key = (layer, reference.offset); @@ -430,79 +413,26 @@ impl PathStateReader { Ok(()) } - // Guards cover local cache work only; origin I/O runs after the guard drops. - fn cache(&self) -> Result> { - self.cache.lock().map_err(|_| { - crate::error::MetadataError::Internal("path-state page cache is poisoned".to_owned()) - }) - } - async fn page( &self, store: &Store, layer: usize, - reference: &PathStatePageRef, + reference: &PageRef, kind: Kind, ) -> Result> { - let source_bytes = self.descriptor.layers[layer].bytes; - let key = (layer, reference.offset); - if let Some((previous_kind, previous, page)) = self.cache()?.pages.get(&key) { - if *previous_kind != kind || previous != reference { - return corrupt("cached page routing metadata changed"); - } - return Ok(page.clone()); - } - let bytes = store - .range_get( + self.cache + .fetch( + store, &self.paths[layer], - reference.offset..reference.offset + u64::from(reference.bytes), + layer, + self.descriptor.layers[layer].bytes, + reference, + kind, ) - .await?; - let page = Arc::new(paged::decode( - &bytes, - reference, - kind, - source_bytes, - self.paths[layer].as_ref(), - )?); - let weight = page_weight(reference, &page); - if weight <= CACHE_BYTES { - let mut cache = self.cache()?; - if let Some((previous_kind, previous, existing)) = cache.pages.get(&key) { - if *previous_kind != kind || previous != reference { - return corrupt("concurrent page routing metadata changed"); - } - return Ok(existing.clone()); - } - while cache.bytes.saturating_add(weight) > CACHE_BYTES - || cache.pages.len() >= CACHE_PAGES - { - let Some(oldest) = cache.order.pop_front() else { - break; - }; - if let Some((_, reference, old)) = cache.pages.remove(&oldest) { - cache.bytes = cache.bytes.saturating_sub(page_weight(&reference, &old)); - } - } - cache.bytes += weight; - cache.order.push_back(key); - cache - .pages - .insert(key, (kind, reference.clone(), page.clone())); - } - Ok(page) + .await } } -fn page_weight(reference: &PathStatePageRef, page: &Page) -> usize { - page.resident_bytes() - .saturating_add(std::mem::size_of::()) - .saturating_add(std::mem::size_of::()) - .saturating_add(reference.first.capacity()) - .saturating_add(reference.last.capacity()) - .saturating_add(128) -} - fn matches_graph(descriptor: &PathStateDescriptor, graph: &SplitCommitGraph) -> bool { descriptor.generation == graph.descriptor.generation && descriptor.pack_index_hash == graph.descriptor.pack_index_hash diff --git a/crates/crab-metadata/src/path_state/scan.rs b/crates/crab-metadata/src/path_state/scan.rs index 973137941..a850f99ed 100644 --- a/crates/crab-metadata/src/path_state/scan.rs +++ b/crates/crab-metadata/src/path_state/scan.rs @@ -27,7 +27,7 @@ pub struct PathStateScan<'a> { phase: Phase, names: Vec>, used: Vec, - audit: BTreeMap, + audit: BTreeMap, nodes: Cursor, records: Cursor, children: Option<(PathStateNodeRef, Cursor)>, @@ -43,7 +43,7 @@ enum Phase { #[derive(Default)] struct Cursor { - pending: Vec, + pending: Vec, leaf: Option>, index: usize, } @@ -276,7 +276,7 @@ impl PathStateScan<'_> { } impl Cursor { - fn new(root: PathStatePageRef) -> Self { + fn new(root: PageRef) -> Self { Self { pending: vec![root], ..Self::default() @@ -289,7 +289,7 @@ impl Cursor { store: &Store, layer: usize, kind: Kind, - audit: &mut BTreeMap, + audit: &mut BTreeMap, ) -> Result, Vec)>> { loop { if let Some(page) = &self.leaf { diff --git a/crates/crab-metadata/src/split_commit_graph.rs b/crates/crab-metadata/src/split_commit_graph.rs index 14ba6d999..f9e74ab12 100644 --- a/crates/crab-metadata/src/split_commit_graph.rs +++ b/crates/crab-metadata/src/split_commit_graph.rs @@ -10,6 +10,16 @@ use crate::validation::validate_content_hash; #[cfg(feature = "storage")] use crab_storage::{Store, StoreLayout}; +mod paged; +pub use paged::CommitGraphLayerPages; +#[cfg(feature = "storage")] +mod reader; +#[cfg(feature = "storage")] +pub use reader::CommitGraphReader; + +const PAGED_VERSION: u32 = 2; +const MAX_PAGED_LAYERS: usize = 32; + const LAYER_MAGIC: &[u8; 8] = b"CRABCG01"; const LAYER_VERSION: u32 = 1; const LAYER_HEADER_BYTES: usize = 24; @@ -54,12 +64,16 @@ pub struct CommitGraphLayerRef { pub base_ordinal: u32, pub commit_count: u32, pub bytes: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pages: Option, } /// Small immutable object named by `Manifest::commit_graph_hash`. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CommitGraphDescriptor { pub version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub commit_ordinal_digest: Option, pub generation: u64, pub pack_index_hash: String, pub git_validation_digest: String, @@ -121,6 +135,14 @@ impl SplitCommitGraph { oid_to_ordinal, }; validate_graph_records(&graph)?; + if graph + .descriptor + .commit_ordinal_digest + .as_ref() + .is_some_and(|digest| *digest != graph.ordinal_digest()) + { + return corrupt("commit graph positional identity mismatch"); + } Ok(graph) } @@ -150,20 +172,7 @@ impl SplitCommitGraph { /// Return a layer-independent digest of the positional commit records. #[must_use] pub fn ordinal_digest(&self) -> String { - let mut hasher = blake3::Hasher::new(); - hasher.update(b"crab commit ordinal identity\0"); - hasher.update(&self.descriptor.commit_count.to_le_bytes()); - for layer in &self.layers { - for record in &layer.records { - hasher.update(&record.oid); - hasher.update(&record.tree_oid); - hasher.update(&(record.parents.len() as u32).to_le_bytes()); - for parent in &record.parents { - hasher.update(&parent.to_le_bytes()); - } - } - } - hasher.finalize().to_hex().to_string() + ordinal_digest(&self.layers, self.descriptor.commit_count) } /// Return an exact ancestry answer when both commits are in this complete graph. @@ -360,6 +369,28 @@ pub fn append_split_commit_graph( roots, additions, false, + LAYER_VERSION, + ) +} + +/// Append authenticated graph pages for the unreleased capsule browse protocol. +pub fn append_paged_commit_graph( + base: Option, + generation: u64, + pack_index_hash: String, + git_validation_digest: String, + roots: &[[u8; 20]], + additions: Vec, +) -> Result> { + update_split_commit_graph( + base, + generation, + pack_index_hash, + git_validation_digest, + roots, + additions, + false, + PAGED_VERSION, ) } @@ -374,6 +405,7 @@ pub fn compact_split_commit_graph(graph: SplitCommitGraph) -> Result Result { let descriptor = CommitGraphDescriptor { - version: LAYER_VERSION, + version: graph.descriptor.version, + commit_ordinal_digest: graph.descriptor.commit_ordinal_digest.clone(), generation, pack_index_hash, git_validation_digest, @@ -428,6 +461,7 @@ fn update_split_commit_graph( roots: &[[u8; 20]], additions: Vec, compact_layers: bool, + encoding: u32, ) -> Result> { validate_content_hash( &pack_index_hash, @@ -565,6 +599,9 @@ fn update_split_commit_graph( } let new_commit_count = u32::try_from(new_records.len()) .map_err(|_| MetadataError::Internal("commit graph exceeds u32 ordinals".to_owned()))?; + let commit_count = first_new_ordinal + .checked_add(new_commit_count) + .ok_or_else(|| MetadataError::Internal("commit graph exceeds u32 ordinals".to_owned()))?; let existing_layer_refs = base .as_ref() @@ -581,10 +618,12 @@ fn update_split_commit_graph( if new_commit_count > 0 { changed_layers.push(layers.len() - 1); } - while compact_layers && layers.len() >= 2 { + while (compact_layers || encoding == PAGED_VERSION) && layers.len() >= 2 { let newer = layers.last().map_or(0, |layer| layer.records.len()); let older = layers[layers.len() - 2].records.len(); - if newer < older { + // Variable append sizes can leave a long decreasing layer sequence. + // Carry the smallest layers when needed to keep lazy OID lookup bounded. + if newer < older && !(encoding == PAGED_VERSION && layers.len() > MAX_PAGED_LAYERS) { break; } let newer = layers @@ -608,11 +647,17 @@ fn update_split_commit_graph( for (index, layer) in layers.iter().enumerate() { if !changed_layers.contains(&index) && let Some(existing) = base_layer_ref_at(layer, index, &existing_layer_refs) + && existing.pages.is_some() == (encoding == PAGED_VERSION) { layer_refs.push(existing.clone()); continue; } - let bytes = encode_commit_graph_layer(layer)?; + let (bytes, pages) = if encoding == PAGED_VERSION { + let (bytes, pages) = paged::encode_layer(layer)?; + (bytes, Some(pages)) + } else { + (encode_commit_graph_layer(layer)?, None) + }; let hash = blake3::hash(&bytes).to_hex().to_string(); let reference = CommitGraphLayerRef { path: format!("metadata/commit-graph/layers/{hash}.bin"), @@ -620,20 +665,19 @@ fn update_split_commit_graph( base_ordinal: layer.base_ordinal, commit_count: layer.records.len() as u32, bytes: bytes.len() as u64, + pages, }; layer_refs.push(reference.clone()); layer_objects.push(CommitGraphLayerObject { reference, bytes }); } let descriptor = CommitGraphDescriptor { - version: LAYER_VERSION, + version: encoding, + commit_ordinal_digest: (encoding == PAGED_VERSION) + .then(|| ordinal_digest(&layers, commit_count)), generation, pack_index_hash, git_validation_digest, - commit_count: first_new_ordinal - .checked_add(new_commit_count) - .ok_or_else(|| { - MetadataError::Internal("commit graph exceeds u32 ordinals".to_owned()) - })?, + commit_count, layers: layer_refs, }; validate_descriptor(&descriptor, &layers)?; @@ -803,7 +847,10 @@ pub async fn load_split_commit_graph( reason: "commit graph layer byte length mismatch".to_owned(), }); } - layers.push(decode_commit_graph_layer(&bytes, path.as_ref())?); + layers.push(match &reference.pages { + Some(pages) => paged::decode_layer(&bytes, reference, pages, path.as_ref())?, + None => decode_commit_graph_layer(&bytes, path.as_ref())?, + }); } SplitCommitGraph::new(descriptor, layers) } @@ -932,7 +979,7 @@ fn validate_graph_records(graph: &SplitCommitGraph) -> Result<()> { } fn validate_descriptor_shape(descriptor: &CommitGraphDescriptor, path: &str) -> Result<()> { - if descriptor.version != LAYER_VERSION { + if !matches!(descriptor.version, LAYER_VERSION | PAGED_VERSION) { return corrupt_at(path, "unsupported commit graph descriptor version"); } validate_content_hash( @@ -945,6 +992,17 @@ fn validate_descriptor_shape(descriptor: &CommitGraphDescriptor, path: &str) -> "commit graph validation digest", path, )?; + if descriptor.version == PAGED_VERSION { + let digest = descriptor.commit_ordinal_digest.as_deref().ok_or_else(|| { + crate::validation::corrupt_object(path, "paged graph positional identity is absent") + })?; + validate_content_hash(digest, "graph positional identity", path)?; + if descriptor.layers.len() > MAX_PAGED_LAYERS { + return corrupt_at(path, "paged graph layer directory exceeds its bound"); + } + } else if descriptor.commit_ordinal_digest.is_some() { + return corrupt_at(path, "released graph has paged identity fields"); + } let mut next = 0u32; for layer in &descriptor.layers { validate_content_hash(&layer.hash, "commit graph layer hash", path)?; @@ -955,6 +1013,11 @@ fn validate_descriptor_shape(descriptor: &CommitGraphDescriptor, path: &str) -> { return corrupt_at(path, "invalid commit graph layer reference"); } + match (&layer.pages, descriptor.version) { + (Some(pages), PAGED_VERSION) => paged::validate_roots(layer, pages, path)?, + (None, LAYER_VERSION) => {} + _ => return corrupt_at(path, "graph layer format does not match descriptor"), + } next = next.checked_add(layer.commit_count) .ok_or_else(|| MetadataError::CorruptObject { @@ -1034,6 +1097,23 @@ fn parse_sha1_hex(value: &str) -> Option<[u8; 20]> { Some(oid) } +fn ordinal_digest(layers: &[CommitGraphLayer], commit_count: u32) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"crab commit ordinal identity\0"); + hasher.update(&commit_count.to_le_bytes()); + for layer in layers { + for record in &layer.records { + hasher.update(&record.oid); + hasher.update(&record.tree_oid); + hasher.update(&(record.parents.len() as u32).to_le_bytes()); + for parent in &record.parents { + hasher.update(&parent.to_le_bytes()); + } + } + } + hasher.finalize().to_hex().to_string() +} + fn corrupt(reason: &str) -> Result { corrupt_at("commit graph", reason) } @@ -1353,6 +1433,7 @@ mod tests { fn descriptor_compaction_due_only_when_newest_layer_reaches_older_layer() { let descriptor = |counts: &[u32]| CommitGraphDescriptor { version: 1, + commit_ordinal_digest: None, generation: 1, pack_index_hash: hash(1), git_validation_digest: hash(2), @@ -1366,6 +1447,7 @@ mod tests { base_ordinal: 0, commit_count: *count, bytes: 1, + pages: None, }) .collect(), }; @@ -1381,6 +1463,7 @@ mod tests { fn rejects_parent_that_is_not_earlier_than_child() { let descriptor = CommitGraphDescriptor { version: 1, + commit_ordinal_digest: None, generation: 1, pack_index_hash: hash(1), git_validation_digest: hash(2), @@ -1391,6 +1474,7 @@ mod tests { base_ordinal: 0, commit_count: 1, bytes: 88, + pages: None, }], }; let layer = CommitGraphLayer { diff --git a/crates/crab-metadata/src/split_commit_graph/paged.rs b/crates/crab-metadata/src/split_commit_graph/paged.rs new file mode 100644 index 000000000..26a468343 --- /dev/null +++ b/crates/crab-metadata/src/split_commit_graph/paged.rs @@ -0,0 +1,164 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::*; +use crate::PageRef; +#[cfg(any(feature = "storage", test))] +use crate::pages::Input; +use crate::pages::{self, Kind, Writer}; + +/// Authenticated positional records and OID directory for one graph layer. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CommitGraphLayerPages { + pub records: PageRef, + pub oids: PageRef, +} + +pub(super) fn encode_layer(layer: &CommitGraphLayer) -> Result<(Vec, CommitGraphLayerPages)> { + let mut writer = Writer::default(); + let mut records = Vec::with_capacity(layer.records.len()); + let mut oids = BTreeMap::new(); + for (index, record) in layer.records.iter().enumerate() { + let ordinal = layer + .base_ordinal + .checked_add(u32::try_from(index).map_err(|_| { + crate::validation::corrupt_object("commit graph", "ordinal exceeds u32") + })?) + .ok_or_else(|| crate::validation::corrupt_object("commit graph", "ordinal overflow"))?; + if record.parents.len() > MAX_PARENTS + || record.parents.iter().any(|parent| *parent >= ordinal) + { + return corrupt("invalid paged graph parents"); + } + let mut value = Vec::new(); + value.extend_from_slice(&record.oid); + value.extend_from_slice(&record.tree_oid); + value.extend_from_slice(&record.commit_time.to_le_bytes()); + value.extend_from_slice(&record.corrected_generation.to_le_bytes()); + pages::put_u32(&mut value, record.parents.len())?; + for parent in &record.parents { + value.extend_from_slice(&parent.to_le_bytes()); + } + records.push((ordinal.to_be_bytes().to_vec(), value)); + if oids + .insert(record.oid.to_vec(), ordinal.to_le_bytes().to_vec()) + .is_some() + { + return corrupt("duplicate paged graph OID"); + } + } + let records = writer.tree(Kind::CommitRecords, records)?; + let oids = writer.tree(Kind::CommitOids, oids.into_iter().collect())?; + Ok((writer.bytes, CommitGraphLayerPages { records, oids })) +} + +pub(super) fn validate_roots( + reference: &CommitGraphLayerRef, + roots: &CommitGraphLayerPages, + path: &str, +) -> Result<()> { + roots + .records + .validate(Kind::CommitRecords, reference.bytes)?; + roots.oids.validate(Kind::CommitOids, reference.bytes)?; + let end = reference + .base_ordinal + .checked_add(reference.commit_count) + .and_then(|end| end.checked_sub(1)); + if roots.records.count != reference.commit_count + || roots.oids.count != reference.commit_count + || roots.records.first != reference.base_ordinal.to_be_bytes() + || end.is_none_or(|end| roots.records.last != end.to_be_bytes()) + { + return corrupt_at(path, "paged graph root coverage mismatch"); + } + Ok(()) +} + +#[cfg(any(feature = "storage", test))] +pub(super) fn record(bytes: &[u8], ordinal: u32, path: &str) -> Result { + let mut input = Input::new(bytes, path); + let oid = input.array()?; + let tree_oid = input.array()?; + let commit_time = i64::from_le_bytes(input.array()?); + let corrected_generation = u64::from_le_bytes(input.array()?); + let count = input.u32()? as usize; + if count > MAX_PARENTS || count > input.remaining() / 4 { + return corrupt_at(path, "paged graph parent count exceeds its bounds"); + } + let mut parents = Vec::with_capacity(count); + for _ in 0..count { + let parent = input.u32()?; + if parent >= ordinal { + return corrupt_at(path, "paged graph parent is not earlier than child"); + } + parents.push(parent); + } + input.finish()?; + Ok(CommitGraphRecord { + oid, + tree_oid, + commit_time, + corrected_generation, + parents, + }) +} + +#[cfg(any(feature = "storage", test))] +pub(super) fn decode_layer( + bytes: &[u8], + reference: &CommitGraphLayerRef, + roots: &CommitGraphLayerPages, + path: &str, +) -> Result { + validate_roots(reference, roots, path)?; + let mut visited = BTreeMap::new(); + let rows = pages::scan( + bytes, + &roots.records, + Kind::CommitRecords, + path, + &mut visited, + )?; + let mut expected_oids = BTreeMap::new(); + let mut records = Vec::with_capacity(rows.len()); + for (key, value) in rows { + let ordinal = + u32::from_be_bytes(key.as_slice().try_into().map_err(|_| { + crate::validation::corrupt_object(path, "invalid graph ordinal key") + })?); + let record = record(&value, ordinal, path)?; + if expected_oids + .insert(record.oid.to_vec(), ordinal.to_le_bytes().to_vec()) + .is_some() + { + return corrupt_at(path, "duplicate graph OID"); + } + records.push(record); + } + let actual_oids = pages::scan(bytes, &roots.oids, Kind::CommitOids, path, &mut visited)?; + if actual_oids != expected_oids.into_iter().collect::>() { + return corrupt_at( + path, + "graph OID directory disagrees with positional records", + ); + } + let mut offset = 0u64; + for (start, (_, page)) in visited { + if start != offset { + return corrupt_at(path, "graph pages overlap or leave unreferenced bytes"); + } + offset = offset.checked_add(u64::from(page.bytes)).ok_or_else(|| { + crate::validation::corrupt_object(path, "graph page coverage overflow") + })?; + } + if offset != bytes.len() as u64 { + return corrupt_at(path, "graph has trailing unreferenced bytes"); + } + Ok(CommitGraphLayer { + base_ordinal: reference.base_ordinal, + records, + }) +} diff --git a/crates/crab-metadata/src/split_commit_graph/paged_tests.rs b/crates/crab-metadata/src/split_commit_graph/paged_tests.rs new file mode 100644 index 000000000..85322a6ad --- /dev/null +++ b/crates/crab-metadata/src/split_commit_graph/paged_tests.rs @@ -0,0 +1,393 @@ +use super::super::paged::decode_layer; +use super::*; +use object_store::ObjectStoreExt; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +fn oid(index: u32) -> [u8; 20] { + let mut oid = [7; 20]; + oid[..4].copy_from_slice(&index.to_be_bytes()); + oid +} + +fn inputs(start: u32, count: u32) -> Vec { + (start..start + count) + .map(|index| CommitGraphInput { + oid: oid(index), + tree_oid: oid(index + 100_000), + commit_time: i64::from(index), + parents: if index == 0 { + Vec::new() + } else { + vec![oid(index - 1)] + }, + }) + .collect() +} + +async fn fixture(count: u32) -> (Store, StoreLayout, CommitGraphWrite, Arc) { + let observed = Arc::new(AtomicU64::new(0)); + let counter = Arc::clone(&observed); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_read_byte_observer(Arc::new(move |bytes| { + counter.fetch_add(bytes, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store.clone(), "paged-graph".to_owned()); + let write = append_paged_commit_graph( + None, + 1, + "a".repeat(64), + "b".repeat(64), + &[oid(count - 1)], + inputs(0, count), + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &write) + .await + .unwrap(); + observed.store(0, Ordering::Relaxed); + (store, layout, write, observed) +} + +#[tokio::test] +async fn graph_point_queries_preserve_duplicates_without_reading_whole_layers() { + let (store, layout, write, observed) = fixture(10_000).await; + let reader = CommitGraphReader::load(&store, &layout, &write.descriptor_hash, 512 * 1024) + .await + .unwrap(); + assert_eq!( + observed.load(Ordering::Relaxed), + write.descriptor_bytes.len() as u64 + ); + let requested = [oid(9_999), oid(0), oid(9_999), oid(20_000)]; + assert_eq!( + reader.ordinals(&store, &requested).await.unwrap(), + vec![Some(9_999), Some(0), Some(9_999), None] + ); + let records = reader + .records(&store, &[9_999, 0, 9_999, 10_000]) + .await + .unwrap(); + assert_eq!( + records + .iter() + .map(|record| record.as_ref().map(|record| record.oid)) + .collect::>(), + vec![Some(oid(9_999)), Some(oid(0)), Some(oid(9_999)), None] + ); + assert!(observed.load(Ordering::Relaxed) < write.layers[0].bytes.len() as u64 / 2); + observed.store(0, Ordering::Relaxed); + reader.ordinals(&store, &requested).await.unwrap(); + reader + .records(&store, &[9_999, 0, 9_999, 10_000]) + .await + .unwrap(); + assert_eq!(observed.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn full_graph_validation_preserves_ordinals_across_paged_compaction_and_rebind() { + let (store, layout, initial, _) = fixture(8).await; + let base = load_split_commit_graph( + &store, + &layout, + &initial.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let first_digest = base.ordinal_digest(); + let write = append_paged_commit_graph( + Some(base), + 2, + "a".repeat(64), + "b".repeat(64), + &[oid(15)], + inputs(8, 8), + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &write) + .await + .unwrap(); + let graph = load_split_commit_graph( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + assert_eq!(graph.descriptor.layers.len(), 1); + assert_ne!(graph.ordinal_digest(), first_digest); + let rebind = rebind_split_commit_graph(&graph, 3, "c".repeat(64), "d".repeat(64)).unwrap(); + let descriptor = decode_commit_graph_descriptor(&rebind.descriptor_bytes, "rebind").unwrap(); + assert_eq!( + descriptor.commit_ordinal_digest.as_deref(), + Some(graph.ordinal_digest().as_str()) + ); + let reader = CommitGraphReader::load( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let requested = (0..16).collect::>(); + let actual = reader.records(&store, &requested).await.unwrap(); + assert_eq!( + actual, + requested + .iter() + .map(|ordinal| graph.record(*ordinal).cloned()) + .collect::>() + ); +} + +#[tokio::test] +async fn graph_query_budget_applies_before_io_and_to_warm_pages() { + let (store, layout, write, observed) = fixture(1_000).await; + let mut reader = CommitGraphReader::load( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + reader.records(&store, &[999]).await.unwrap(); + reader.max_bytes = reader.descriptor_bytes; + observed.store(0, Ordering::Relaxed); + assert!(reader.records(&store, &[999]).await.is_err()); + assert_eq!(observed.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn missing_or_corrupt_graph_pages_are_errors_including_out_of_range_oid_misses() { + let (store, layout, write, _) = fixture(1_000).await; + let path = layout.repo_path(&write.layers[0].reference.path); + let root = &write.layers[0].reference.pages.as_ref().unwrap().oids; + let mut bytes = write.layers[0].bytes.clone(); + bytes[root.offset as usize] ^= 1; + store + .inner() + .put(&path, bytes::Bytes::from(bytes).into()) + .await + .unwrap(); + let reader = CommitGraphReader::load( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + assert!(reader.ordinals(&store, &[oid(20_000)]).await.is_err()); + store.inner().delete(&path).await.unwrap(); + assert!(reader.records(&store, &[0]).await.is_err()); +} + +#[tokio::test] +async fn full_validation_rejects_oid_directory_disagreement_and_positional_digest_tampering() { + let (store, layout, write, _) = fixture(3).await; + let graph = load_split_commit_graph( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let mut descriptor = graph.descriptor.clone(); + descriptor.commit_ordinal_digest = Some("f".repeat(64)); + assert!(SplitCommitGraph::new(descriptor, graph.layers.clone()).is_err()); + let reference = &write.layers[0].reference; + let mut roots = reference.pages.clone().unwrap(); + let mut bytes = write.layers[0].bytes.clone(); + // Re-sign a wrong ordinal to test semantic consistency rather than hashing. + let start = roots.oids.offset as usize; + bytes[start + 14 + 20 + 4..start + 14 + 20 + 8].copy_from_slice(&2u32.to_le_bytes()); + roots.oids.hash = *blake3::hash(&bytes[start..start + roots.oids.bytes as usize]).as_bytes(); + assert!(decode_layer(&bytes, reference, &roots, "wrong-directory").is_err()); +} + +#[tokio::test] +async fn paged_append_reuses_unchanged_layers_and_migrates_only_derived_eager_graphs() { + let (store, layout, initial, _) = fixture(8).await; + let base = load_split_commit_graph( + &store, + &layout, + &initial.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let old_layer = base.descriptor.layers[0].clone(); + let write = append_paged_commit_graph( + Some(base), + 2, + "a".repeat(64), + "b".repeat(64), + &[oid(8)], + inputs(8, 1), + ) + .unwrap() + .unwrap(); + let descriptor = decode_commit_graph_descriptor(&write.descriptor_bytes, "append").unwrap(); + assert_eq!((&descriptor.layers[0], write.layers.len()), (&old_layer, 1)); + let eager = append_split_commit_graph( + None, + 1, + "a".repeat(64), + "b".repeat(64), + &[oid(0)], + inputs(0, 1), + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &eager) + .await + .unwrap(); + assert!( + CommitGraphReader::load( + &store, + &layout, + &eager.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES + ) + .await + .is_err() + ); + let base = load_split_commit_graph( + &store, + &layout, + &eager.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let converted = append_paged_commit_graph( + Some(base), + 1, + "a".repeat(64), + "b".repeat(64), + &[oid(0)], + vec![], + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &converted) + .await + .unwrap(); + let reader = CommitGraphReader::load( + &store, + &layout, + &converted.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + assert_eq!( + reader.ordinals(&store, &[oid(0)]).await.unwrap(), + vec![Some(0)] + ); +} + +#[tokio::test] +async fn cancelled_graph_range_reads_leave_the_reader_reusable() { + use object_store::throttle::{ThrottleConfig, ThrottledStore}; + let (store, layout, write, _) = fixture(1_000).await; + let throttled = Arc::new(ThrottledStore::new( + store.inner().clone(), + ThrottleConfig::default(), + )); + let store = Store::new(throttled.clone()); + let reader = CommitGraphReader::load( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + reader.records(&store, &[0]).await.unwrap(); + throttled.config_mut(|config| config.wait_get_per_call = std::time::Duration::from_secs(60)); + { + let query = reader.records(&store, &[999]); + futures_util::pin_mut!(query); + assert!(futures_util::poll!(query).is_pending()); + } + throttled.config_mut(|config| config.wait_get_per_call = std::time::Duration::ZERO); + assert_eq!( + reader.records(&store, &[999]).await.unwrap()[0] + .as_ref() + .unwrap() + .oid, + oid(999) + ); +} + +#[tokio::test] +async fn decreasing_append_sizes_cannot_grow_the_layer_directory_without_bound() { + let store = Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = StoreLayout::new(store.clone(), "graph-carries".to_owned()); + let mut graph = None; + let mut count = 0; + for batch in (1..=40).rev() { + let write = append_paged_commit_graph( + graph, + 1, + "a".repeat(64), + "b".repeat(64), + &[oid(count + batch - 1)], + inputs(count, batch), + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &write) + .await + .unwrap(); + graph = Some( + load_split_commit_graph( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(), + ); + count += batch; + assert!(graph.as_ref().unwrap().descriptor.layers.len() <= MAX_PAGED_LAYERS); + } + let graph = graph.unwrap(); + assert_eq!( + (0..count) + .map(|ordinal| graph.record(ordinal).unwrap().oid) + .collect::>(), + (0..count).map(oid).collect::>() + ); +} + +proptest::proptest! { + #[test] + fn paged_graph_matches_eager_graph_for_branching_parent_orders(selectors in proptest::collection::vec(0u8..8, 1..40)) { + let additions = selectors.iter().enumerate().map(|(index, selector)| { + let mut parents = Vec::new(); + if index > 0 && selector & 1 != 0 { parents.push(oid(index as u32 - 1)) } + if index > 1 && selector & 2 != 0 { parents.push(oid(index as u32 - 2)) } + if selector & 4 != 0 { parents.reverse() } + CommitGraphInput { oid: oid(index as u32), tree_oid: oid(index as u32 + 1000), commit_time: index as i64, parents } + }).collect::>(); + let roots = additions.iter().map(|input| input.oid).collect::>(); + let eager = append_split_commit_graph(None, 1, "a".repeat(64), "b".repeat(64), &roots, additions.clone()).unwrap().unwrap(); + let paged = append_paged_commit_graph(None, 1, "a".repeat(64), "b".repeat(64), &roots, additions).unwrap().unwrap(); + let eager_layers = eager.layers.iter().map(|object| decode_commit_graph_layer(&object.bytes, "eager").unwrap()).collect::>(); + let paged_layers = paged.layers.iter().map(|object| decode_layer(&object.bytes, &object.reference, object.reference.pages.as_ref().unwrap(), "paged").unwrap()).collect::>(); + proptest::prop_assert_eq!(&paged_layers, &eager_layers); + } +} diff --git a/crates/crab-metadata/src/split_commit_graph/reader.rs b/crates/crab-metadata/src/split_commit_graph/reader.rs new file mode 100644 index 000000000..6a13b0d43 --- /dev/null +++ b/crates/crab-metadata/src/split_commit_graph/reader.rs @@ -0,0 +1,291 @@ +use super::paged::{CommitGraphLayerPages, record}; +use super::*; +use crate::PageRef; +use crate::pages::{self, Kind, Page}; +use futures_util::{StreamExt, TryStreamExt}; +use std::collections::BTreeSet; + +/// Immutable graph point lookups; record presence does not prove ref visibility. +#[derive(Debug)] +pub struct CommitGraphReader { + descriptor: CommitGraphDescriptor, + paths: Vec, + descriptor_bytes: u64, + max_bytes: u64, + cache: pages::PageCache, +} + +struct Work { + bytes: u64, + seen: BTreeSet<(usize, u64)>, +} + +impl CommitGraphReader { + /// Load a bounded paged descriptor without fetching graph layers. + pub async fn load( + store: &Store, + layout: &StoreLayout, + hash: &str, + max_bytes: u64, + ) -> Result { + let (descriptor, descriptor_bytes) = + read_split_commit_graph_descriptor(store, layout, hash, max_bytes.min(128 * 1024)) + .await?; + if descriptor.version != PAGED_VERSION { + return corrupt("graph is not the capsule paged format"); + } + let paths = descriptor + .layers + .iter() + .map(|layer| layout.repo_path(&layer.path)) + .collect(); + Ok(Self { + descriptor, + paths, + descriptor_bytes, + max_bytes, + cache: pages::PageCache::default(), + }) + } + + /// Return the captured immutable descriptor, including positional identity. + #[must_use] + pub fn descriptor(&self) -> &CommitGraphDescriptor { + &self.descriptor + } + + /// Resolve requested OIDs, retaining duplicate input occurrences. + /// + /// The caller admits request/output counts and supplies an operation-budgeted store. + pub async fn ordinals(&self, store: &Store, oids: &[[u8; 20]]) -> Result>> { + let mut work = self.work(); + let mut output = vec![None; oids.len()]; + let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + for (layer, reference) in self.descriptor.layers.iter().enumerate() { + let roots = self.roots(reference)?; + let values = self + .lookup( + store, + layer, + &roots.oids, + Kind::CommitOids, + &keys, + &mut work, + ) + .await?; + for (index, value) in values.into_iter().enumerate() { + let Some(value) = value else { continue }; + let ordinal = u32::from_le_bytes(value.as_slice().try_into().map_err(|_| { + crate::validation::corrupt_object("graph OID", "invalid ordinal value") + })?); + if ordinal < reference.base_ordinal + || ordinal - reference.base_ordinal >= reference.commit_count + || output[index].is_some() + { + return corrupt("graph OID mapping escapes its layer or repeats"); + } + output[index] = Some(ordinal); + } + } + let ordinals = output.iter().flatten().copied().collect::>(); + let records = self.raw_records(store, &ordinals, &mut work).await?; + let mut records = records.into_iter(); + for (oid, ordinal) in oids.iter().zip(&output) { + if ordinal.is_some() + && records + .next() + .flatten() + .is_none_or(|record| record.oid != *oid) + { + return corrupt("graph OID mapping disagrees with its record"); + } + } + Ok(output) + } + + /// Resolve records and verify ordered parent edges and corrected generations. + /// + /// The caller admits request/output counts and supplies an operation-budgeted store. + pub async fn records( + &self, + store: &Store, + ordinals: &[u32], + ) -> Result>> { + let mut work = self.work(); + let output = self.raw_records(store, ordinals, &mut work).await?; + let parent_ids = output + .iter() + .flatten() + .flat_map(|record| record.parents.iter().copied()) + .collect::>() + .into_iter() + .collect::>(); + let parent_records = self.raw_records(store, &parent_ids, &mut work).await?; + let parents = parent_ids + .into_iter() + .zip(parent_records) + .collect::>(); + for record in output.iter().flatten() { + let mut generation = non_negative_time(record.commit_time); + for parent in &record.parents { + let parent = parents + .get(parent) + .and_then(Option::as_ref) + .ok_or_else(|| { + crate::validation::corrupt_object( + "graph parents", + "required parent is absent", + ) + })?; + generation = generation.max(parent.corrected_generation.saturating_add(1)); + } + if generation != record.corrected_generation { + return corrupt("graph corrected generation mismatch"); + } + } + Ok(output) + } + + fn work(&self) -> Work { + Work { + bytes: self.descriptor_bytes, + seen: BTreeSet::new(), + } + } + + fn roots<'a>(&self, layer: &'a CommitGraphLayerRef) -> Result<&'a CommitGraphLayerPages> { + layer + .pages + .as_ref() + .ok_or_else(|| crate::validation::corrupt_object("graph", "paged roots are absent")) + } + + async fn raw_records( + &self, + store: &Store, + ordinals: &[u32], + work: &mut Work, + ) -> Result>> { + let mut output = vec![None; ordinals.len()]; + let mut requests = BTreeMap::)>>::new(); + for (index, ordinal) in ordinals.iter().copied().enumerate() { + if ordinal >= self.descriptor.commit_count { + continue; + } + let layer = self + .descriptor + .layers + .partition_point(|layer| layer.base_ordinal <= ordinal) + .checked_sub(1) + .ok_or_else(|| { + crate::validation::corrupt_object("graph", "ordinal has no layer") + })?; + requests + .entry(layer) + .or_default() + .push((index, ordinal.to_be_bytes().to_vec())); + } + for (layer, requests) in requests { + let keys = requests + .iter() + .map(|(_, key)| key.clone()) + .collect::>(); + let root = &self.roots(&self.descriptor.layers[layer])?.records; + let values = self + .lookup(store, layer, root, Kind::CommitRecords, &keys, work) + .await?; + for ((index, _), value) in requests.into_iter().zip(values) { + let value = value.ok_or_else(|| { + crate::validation::corrupt_object("graph", "dense record is absent") + })?; + output[index] = Some(record(&value, ordinals[index], self.paths[layer].as_ref())?); + } + } + Ok(output) + } + + async fn lookup( + &self, + store: &Store, + layer: usize, + root: &PageRef, + kind: Kind, + keys: &[Vec], + work: &mut Work, + ) -> Result>>> { + let mut output = vec![None; keys.len()]; + if keys.is_empty() { + return Ok(output); + } + let mut pending = vec![(root.clone(), (0..keys.len()).collect::>())]; + while !pending.is_empty() { + for (reference, _) in &pending { + reference.validate(kind, self.descriptor.layers[layer].bytes)?; + if work.seen.insert((layer, reference.offset)) { + work.bytes = work + .bytes + .checked_add(u64::from(reference.bytes)) + .ok_or_else(|| { + crate::validation::corrupt_object("graph", "query bytes overflow") + })?; + if work.bytes > self.max_bytes { + return corrupt("graph query exceeds byte bound"); + } + } + } + let batch = futures_util::stream::iter(std::mem::take(&mut pending)) + .map(|(reference, positions)| async move { + let page = self + .cache + .fetch( + store, + &self.paths[layer], + layer, + self.descriptor.layers[layer].bytes, + &reference, + kind, + ) + .await?; + Ok::<_, MetadataError>((page, positions)) + }) + .buffered(4) + .try_collect::>() + .await?; + for (page, positions) in batch { + match page.as_ref() { + Page::Leaf(rows) => { + for position in positions { + if let Ok(index) = + rows.binary_search_by(|(key, _)| key.cmp(&keys[position])) + { + output[position] = Some(rows[index].1.clone()) + } + } + } + Page::Branch(children) => { + let mut batches = BTreeMap::>::new(); + for position in positions { + let index = + children.partition_point(|child| child.last < keys[position]); + if children + .get(index) + .is_some_and(|child| child.first <= keys[position]) + { + batches.entry(index).or_default().push(position) + } + } + pending.extend( + batches + .into_iter() + .map(|(index, positions)| (children[index].clone(), positions)), + ); + } + } + } + } + Ok(output) + } +} +#[cfg(test)] +#[path = "paged_tests.rs"] +mod tests; diff --git a/crates/crab-remote-git/src/commit_graph.rs b/crates/crab-remote-git/src/commit_graph.rs index 6cd7b23b6..332097b23 100644 --- a/crates/crab-remote-git/src/commit_graph.rs +++ b/crates/crab-remote-git/src/commit_graph.rs @@ -352,6 +352,7 @@ mod tests { }; let descriptor = CommitGraphDescriptor { version: 1, + commit_ordinal_digest: None, generation: 7, pack_index_hash: hash(1), git_validation_digest: hash(2), @@ -362,6 +363,7 @@ mod tests { base_ordinal: 0, commit_count: layer.records.len() as u32, bytes: bytes as u64, + pages: None, }] .into_iter() .filter(|layer| layer.commit_count > 0) diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 35c663c26..a886c2fb8 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -78,3 +78,8 @@ this orchestration boundary. The low-level remote Git reader retains its narrow Git/storage/metadata dependency boundary. See the [indexed-read qualification contract](../../crab/docs/architecture/indexed-remote-reads.md) for commands, native Git oracles and cold/warm measurement semantics. +`qualify_remote graph-pages` measures sampled +ordinal/parent records and duplicate OID batches through `CommitGraphReader`. +It verifies the derived graph's captured state and Git identity before reads. +Control capture remains in the reported opening measurements; this component +workload does not claim bounded foreground opening or replace ref visibility. diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 5863e5df8..274ef17c3 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -40,7 +40,11 @@ async fn main() -> Result<()> { ) .into()), }; - if selection != "all" && selection != "build-indexes" && !WORKLOADS.contains(&selection) { + if selection != "all" + && selection != "build-indexes" + && selection != "graph-pages" + && !WORKLOADS.contains(&selection) + { return Err(io::Error::other("unknown qualification operation").into()); } let path = GitPath::new(Bytes::copy_from_slice(path.as_bytes()))?; @@ -89,6 +93,8 @@ async fn main() -> Result<()> { .await; runtime.shutdown().await; result?; + } else if selection == "graph-pages" { + qualify_graph_pages(&layout, &path, &counters).await?; } else { for &name in WORKLOADS .iter() @@ -116,6 +122,106 @@ async fn main() -> Result<()> { Ok(()) } +// This measures the graph paging component, with existing eager control capture +// reported as opening. It makes no foreground visibility or bounded-opening claim. +async fn qualify_graph_pages( + layout: &StoreLayout, + path: &GitPath, + counters: &Measurements, +) -> Result<()> { + use crab_metadata::split_commit_graph::{ + CommitGraphReader, DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + }; + let open = async || -> Result { + let root = crab_metadata::capsule_protocol::load_root(layout).await?; + let indexes = crab_metadata::capsule_protocol::load_browse_indexes(layout) + .await? + .ok_or_else(|| io::Error::other("browse indexes are absent"))?; + let view = crab_read::capsule_protocol::open_view_from_root_with_control( + layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 2 * 1024 * 1024 * 1024, + max_frontier_bytes: 2 * 1024 * 1024 * 1024, + }, + ) + .await?; + if indexes.state_digest() != view.state_digest() { + return Err(io::Error::other("browse index does not match captured authority").into()); + } + let snapshot = view.git_snapshot()?; + let reader = CommitGraphReader::load( + layout.store(), + layout, + indexes.commit_graph_hash(), + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await?; + let descriptor = reader.descriptor(); + if descriptor.generation != snapshot.manifest.generation + || descriptor.pack_index_hash != snapshot.manifest.pack_index_hash + || descriptor.git_validation_digest != snapshot.manifest.git_validation_digest + { + return Err(io::Error::other("graph does not match captured Git identity").into()); + } + Ok(reader) + }; + let reader = counters.measure("open", "cold", open()).await?; + let count = reader.descriptor().commit_count; + if count == 0 { + return Err(io::Error::other("graph is empty").into()); + } + let requested = [0, count / 2, count - 1, count / 2]; + let query = + async || -> Result { + let records = reader + .records(layout.store(), &requested) + .await? + .into_iter() + .collect::>>() + .ok_or_else(|| io::Error::other("required graph record is absent"))?; + let oids = records.iter().map(|record| record.oid).collect::>(); + if reader.ordinals(layout.store(), &oids).await? + != requested.iter().copied().map(Some).collect::>() + { + return Err(io::Error::other("graph OID/ordinal batch differs").into()); + } + let parents = records + .iter() + .flat_map(|record| record.parents.iter().copied()) + .collect::>(); + let parents = reader + .records(layout.store(), &parents) + .await? + .into_iter() + .collect::>>() + .ok_or_else(|| io::Error::other("graph parent is absent"))?; + let mut parents = parents.into_iter(); + let rows = records.iter().zip(requested).map(|(record, ordinal)| { + let parent_oids = parents.by_ref().take(record.parents.len()) + .map(|record| gix_hash::ObjectId::Sha1(record.oid).to_string()).collect::>(); + json!({"ordinal": ordinal, "oid": gix_hash::ObjectId::Sha1(record.oid).to_string(), + "tree": gix_hash::ObjectId::Sha1(record.tree_oid).to_string(), + "parents": parent_oids, "committer_seconds": record.commit_time}) + }).collect::>(); + Ok(json!({"records": rows})) + }; + let cold = counters.measure("graph-pages", "cold", query()).await?; + let warm = counters.measure("graph-pages", "warm", query()).await?; + if cold != warm { + return Err(io::Error::other("cold/warm graph queries differ").into()); + } + println!( + "{}", + json!({"kind":"result", "operation":"graph-pages", "path":path.as_bytes(), "value":cold}) + ); + let reopened = counters.measure("open", "warm", open()).await?; + if reopened.descriptor() != reader.descriptor() { + return Err(io::Error::other("graph changed during qualification").into()); + } + Ok(()) +} + async fn qualify( name: &str, layout: &StoreLayout, diff --git a/crates/crab-write/README.md b/crates/crab-write/README.md index f36031df1..bcb0e2baf 100644 --- a/crates/crab-write/README.md +++ b/crates/crab-write/README.md @@ -59,7 +59,13 @@ orchestrator publishes graph readiness. `build_path_state` then attaches that verified graph, resumes attribution from its checkpoint and persists progress every 32 commits. Completion replaces the partial record through the returned CAS token; cached readers observe both phases without a ref change. Graph -discovery drains a separate bounded operation per batch. `generation::with_generation_owner` shares renewed owner election and +construction writes paged descriptor version 2, reuses unchanged paged layers +and performs bounded-directory carries. Released manifest maintenance retains +the eager version-1 encoding. The capsule owner rebuilds an eager browse graph +even when its state digest is otherwise current, preserving reusable attribution +through the existing checkpoint boundary. Full readback validates graph/page +closure before the returned descriptor can become readiness. Graph discovery +drains a separate bounded operation per batch. `generation::with_generation_owner` shares renewed owner election and global/repository GC fences between v1 maintenance and the v2 orchestrator; all acquired leases are released on returned failure or cancellation. The orchestrator rechecks capsule activity and conditionally publishes the small diff --git a/crates/crab-write/src/generation/browse.rs b/crates/crab-write/src/generation/browse.rs index 33dd13b04..ded8b9443 100644 --- a/crates/crab-write/src/generation/browse.rs +++ b/crates/crab-write/src/generation/browse.rs @@ -1,6 +1,7 @@ //! Build immutable browse indexes from a caller-pinned Git snapshot. use crab_metadata::capsule_protocol::BrowseIndexes; +use crab_metadata::split_commit_graph::append_paged_commit_graph; use super::*; @@ -74,6 +75,7 @@ pub async fn build_graph( graph.descriptor.generation == manifest.generation && graph.descriptor.pack_index_hash == manifest.pack_index_hash && graph.descriptor.git_validation_digest == manifest.git_validation_digest + && graph.descriptor.commit_ordinal_digest.is_some() }) && old_paths.is_some() { @@ -82,7 +84,9 @@ pub async fn build_graph( let roots = commit_roots(repository, manifest, cancel).await?; let inputs = collect_commit_graph_inputs(repository, old_graph.as_ref(), &roots, cancel).await?; - let write = append_split_commit_graph( + // Capsule browse has no released eager-graph contract. Its owner rebuilds + // paged layers while the shipped manifest graph keeps its existing encoding. + let write = append_paged_commit_graph( old_graph, manifest.generation, manifest.pack_index_hash.clone(), From 28459bcfb6fc87578d959a08a7dc08f6e354eda1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 16:08:48 -0700 Subject: [PATCH 009/127] feat(metadata): add paged ancestry proofs --- .../docs/architecture/indexed-remote-reads.md | 48 ++- crab/scripts/e2e/verify_remote_browse.py | 9 +- crates/crab-metadata/README.md | 11 +- crates/crab-metadata/src/pages.rs | 45 +- .../crab-metadata/src/split_commit_graph.rs | 37 +- .../src/split_commit_graph/ancestry.rs | 405 ++++++++++++++++++ .../src/split_commit_graph/paged.rs | 96 ++++- .../src/split_commit_graph/paged_tests.rs | 201 ++++++++- .../src/split_commit_graph/reader.rs | 125 ++++++ crates/crab-remote/README.md | 5 + crates/crab-remote/examples/qualify_remote.rs | 42 +- crates/crab-write/README.md | 5 +- 12 files changed, 983 insertions(+), 46 deletions(-) create mode 100644 crates/crab-metadata/src/split_commit_graph/ancestry.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index bf7ba469d..ad87d8b8b 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -29,7 +29,7 @@ Xet files and unified local staging are subsequent workstreams. | 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | | 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Implemented; 23 focused tests, GC retention regression and 1,000/10,000-commit RustFS format qualification passed | | 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Canonical scaling/transitions passed; streaming Cell ingestion, projected native Git oracle and two process starts passed on RustFS; broader cloud/lifecycle qualification remains open | -| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Paged producer/component reader implemented; explicit visibility and bounded foreground opening remain pending | +| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Paged producer and authenticated ancestor-set component qualified; foreground conversion and bounded opening remain pending | | 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | | 6 | Indexed historical Xet catalog | Indexed checkpoints and visible deltas; bounded compaction; 10-GiB edits and byte-identical cold historical hydration; protected receive and sibling consumers | Pending | | 7 | Lifecycle and migration | Interrupted publication/builds, concurrent owners, GC retained roots/readers, stale caches and released-format migration qualification | Pending; also required in every earlier implementation | @@ -379,3 +379,49 @@ Receipts, failed negative controls, source hashes and binary hashes remain outsi the checkout. Stage 4 still requires explicit visible ancestry proof, lazy foreground conversion, removal of full positional-digest work and bounded control opening. Stages 5–8 remain incomplete. + +## Authenticated ancestor-set component + +Unreleased capsule graph version 3 replaces version 2 with a dense commit-to-set +root directory and a content-addressed node directory in each existing layer +object. Persistent binary interval trees share immutable children, collapse full +ranges and omit unary branches. Exact DAG unions preserve both merge parents and +disconnected history. Full readback recomputes every set from verified positional +records, verifies canonical node bytes and first-use layer placement, and rejects +extra or missing nodes. Removing recurrence validation makes a re-signed forged +set pass; restoring it rejects that set. This proves why authenticated pages alone +are insufficient for visibility. Released manifest graph version 1 is unchanged. + +`CommitGraphReader::ancestry_membership` reads the selected root and batched node +dependencies, preserves duplicates, checks self membership and rejects missing +nodes and malformed interval routing. Its caller must supply roots authorized by +the captured ref view. It does not authorize a root from record presence. The +foreground repository still uses the materialized graph and pinned bitmap; their +replacement and lazy path-state graph binding are the next implementation slice. + +`paged-ancestry-r1` passes 20 isolated cold/warm native Git comparisons at each of +100, 1,000 and 10,000 commits, plus separate attribution checks. HTTP observation +and whole-process CPU/RSS accompany opening and query measurements. At 10,000 +commits, the sampled record/OID/membership component reads at most 411,872 bytes; +all warm component queries read zero backend bytes. Its cold p95 is 60.388 ms on +the debug host, so these receipts do not establish a final foreground latency pass. +Maintenance writes 47,483 / 649,672 / 8,238,957 backend bytes, with 0.15 / 1.03 / +13.49 process CPU seconds; these include migration/build work, not just set union. +The prototype's encoded membership storage test stays below 3x on each doubling +from 1k to 8k for linear, eight-branch, criss-cross and disconnected shapes. This +is fixture qualification, not a worst-case complexity claim. + +`ancestry-transitions-r1` uses a separate task-created native Git repository and +RustFS prefix. The main tip excludes side descendants, a merge includes both +parents, and force-push rewind plus side-ref deletion excludes physically retained +descendants. Native Git independently verifies all membership answers. The source +fixtures and raw receipts remain outside the checkout. Full stage-4 opening, +ref-count/frontier scaling, release-image/cloud performance and stages 5–8 remain +open. + +The final rebuilt binary also passes `paged-ancestry-r2` (60 cold/warm native Git +comparisons and attribution checks) and `ancestry-transitions-r2`. Focused final +checks pass 27 graph/ancestry tests, 21 path-state tests, capsule browse publication +integration, payload-only compilation, strict metadata library Clippy, qualifier +diagnostic Clippy and architecture guardrails. The negative-control source is +restored; final source/binary hashes and receipts are retained externally. diff --git a/crab/scripts/e2e/verify_remote_browse.py b/crab/scripts/e2e/verify_remote_browse.py index a07fd2d9f..cbb0bd4a7 100644 --- a/crab/scripts/e2e/verify_remote_browse.py +++ b/crab/scripts/e2e/verify_remote_browse.py @@ -100,7 +100,7 @@ def check_object(value: dict) -> None: require(item["commit"] == expected, "last-change commit differs") elif name == "blame": lines_checked += check_line_origins(git, head, path, value["ranges"]) - elif name == "graph-pages": + elif name in {"graph-pages", "graph-ancestry"}: rows = value["records"] require(len(rows) == 4 and rows[1] == rows[3], "graph batch order/duplicates differ") for item in rows: @@ -112,6 +112,13 @@ def check_object(value: dict) -> None: require(item["tree"] == tree and item["parents"] == parents and item["committer_seconds"] == int(committer.rsplit(b" ", 2)[1]), "graph record differs from native Git") + if name == "graph-ancestry": + require(value["root"] == head, "ancestry root differs from captured HEAD") + require(len(value["membership"]) == len(rows), "missing ancestry answers") + for item, actual in zip(rows, value["membership"]): + result = subprocess.run(["git", "-C", str(source), "merge-base", "--is-ancestor", item["oid"], value["root"]], capture_output=True) + require(result.returncode in {0, 1}, "native ancestry oracle failed") + require(actual == (result.returncode == 0), "ancestry membership differs from native Git") else: raise AssertionError("unverified qualification operation: " + name) return {"status":"passed", "head":head, "queries_checked":len(results), diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 09218deaf..3d2cd7e31 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -61,9 +61,16 @@ budget before buffering, then admit each layer against its authenticated size and the remaining aggregate budget. All bodies still require hash and exact length verification; malformed provider sizes cannot bypass the encoded-byte ceiling. Decoded index structures require separate memory qualification. -Capsule browse construction writes split-graph descriptor version 2, with +Capsule browse construction writes split-graph descriptor version 3, with authenticated ordinal-record and OID-directory pages and a verified positional -digest. Its carry rule caps the layer directory at 32 entries even for decreasing +digest. Per-commit ancestor roots and content-addressed interval nodes share +unchanged subtrees and collapse full ordinal ranges. Maintenance recomputes the +sets from the complete verified DAG and checks their canonical first-use layer; +local page hashes alone cannot certify ancestry. Point membership reads preserve +duplicates, verify the root contains itself and fail on missing required nodes. +The caller must authorize roots against its captured refs before using membership +as a visibility proof. Version 2 was an unreleased derived format; its owner +rebuilds it rather than introducing a retired-format reader. Its carry rule caps the layer directory at 32 entries even for decreasing append batch sizes; internal page directories remain bounded. `CommitGraphReader` loads only the descriptor and dependency ranges, preserves duplicate query positions and validates selected parent ordering/corrected generations. It does diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 522d53847..11a4e5835 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -47,12 +47,14 @@ pub(crate) enum Kind { Names, CommitRecords, CommitOids, + CommitAncestryRoots, + CommitAncestryNodes, } impl Kind { pub(crate) fn key_bytes(self) -> usize { match self { - Self::Names => 32, + Self::Names | Self::CommitAncestryNodes => 32, Self::CommitOids => 20, _ => 4, } @@ -83,7 +85,10 @@ impl PageRef { { return Err(corruption("invalid metadata page reference")); } - if matches!(kind, Kind::Records | Kind::Nodes | Kind::CommitRecords) { + if matches!( + kind, + Kind::Records | Kind::Nodes | Kind::CommitRecords | Kind::CommitAncestryRoots + ) { let first = integer(&self.first)?; let last = integer(&self.last)?; if last.checked_sub(first).and_then(|span| span.checked_add(1)) != Some(self.count) { @@ -286,14 +291,15 @@ pub(crate) fn decode( { return corrupt_at(path, "invalid page directory routing or ancestry"); } - if matches!(kind, Kind::Records | Kind::Nodes | Kind::CommitRecords) - && children.last().is_some_and(|previous| { - integer(&previous.last) - .ok() - .and_then(|last| last.checked_add(1)) - != integer(&child.first).ok() - }) - { + if matches!( + kind, + Kind::Records | Kind::Nodes | Kind::CommitRecords | Kind::CommitAncestryRoots + ) && children.last().is_some_and(|previous| { + integer(&previous.last) + .ok() + .and_then(|last| last.checked_add(1)) + != integer(&child.first).ok() + }) { return corrupt_at(path, "positional directory has a key gap"); } total = total @@ -310,7 +316,7 @@ pub(crate) fn decode( Page::Branch(children) }; input.finish()?; - tracing::debug!(target:"crab_metadata::index_decode", index_format=if matches!(kind, Kind::CommitRecords | Kind::CommitOids) { "commit_graph" } else { "path_state" }, + tracing::debug!(target:"crab_metadata::index_decode", index_format=if matches!(kind, Kind::CommitRecords | Kind::CommitOids | Kind::CommitAncestryRoots | Kind::CommitAncestryNodes) { "commit_graph" } else { "path_state" }, decoded_bytes=bytes.len() as u64, "index page decoded"); Ok(page) } @@ -367,14 +373,15 @@ fn validate_rows(kind: Kind, rows: &Rows) -> Result<()> { { return Err(corruption("page keys are empty, invalid or unordered")); } - if matches!(kind, Kind::Records | Kind::Nodes | Kind::CommitRecords) - && rows.windows(2).any(|pair| { - integer(&pair[0].0) - .ok() - .and_then(|value| value.checked_add(1)) - != integer(&pair[1].0).ok() - }) - { + if matches!( + kind, + Kind::Records | Kind::Nodes | Kind::CommitRecords | Kind::CommitAncestryRoots + ) && rows.windows(2).any(|pair| { + integer(&pair[0].0) + .ok() + .and_then(|value| value.checked_add(1)) + != integer(&pair[1].0).ok() + }) { return Err(corruption("positional leaf has a key gap")); } Ok(()) diff --git a/crates/crab-metadata/src/split_commit_graph.rs b/crates/crab-metadata/src/split_commit_graph.rs index f9e74ab12..ed2aa1af6 100644 --- a/crates/crab-metadata/src/split_commit_graph.rs +++ b/crates/crab-metadata/src/split_commit_graph.rs @@ -10,6 +10,7 @@ use crate::validation::validate_content_hash; #[cfg(feature = "storage")] use crab_storage::{Store, StoreLayout}; +mod ancestry; mod paged; pub use paged::CommitGraphLayerPages; #[cfg(feature = "storage")] @@ -17,7 +18,7 @@ mod reader; #[cfg(feature = "storage")] pub use reader::CommitGraphReader; -const PAGED_VERSION: u32 = 2; +const PAGED_VERSION: u32 = 3; const MAX_PAGED_LAYERS: usize = 32; const LAYER_MAGIC: &[u8; 8] = b"CRABCG01"; @@ -642,6 +643,11 @@ fn update_split_commit_graph( return Ok(None); } + let ancestry = if encoding == PAGED_VERSION { + Some(ancestry::Index::build(&layers)?) + } else { + None + }; let mut layer_refs = Vec::with_capacity(layers.len()); let mut layer_objects = Vec::new(); for (index, layer) in layers.iter().enumerate() { @@ -653,7 +659,12 @@ fn update_split_commit_graph( continue; } let (bytes, pages) = if encoding == PAGED_VERSION { - let (bytes, pages) = paged::encode_layer(layer)?; + let (bytes, pages) = paged::encode_layer( + layer, + ancestry.as_ref().ok_or_else(|| { + crate::validation::corrupt_object("graph", "ancestry construction is absent") + })?, + )?; (bytes, Some(pages)) } else { (encode_commit_graph_layer(layer)?, None) @@ -823,6 +834,8 @@ pub async fn load_split_commit_graph( read_split_commit_graph_descriptor(store, router, descriptor_hash, max_bytes).await?; let descriptor_path = router.bulk_manifest_path("commit-graph", descriptor_hash); let mut layers = Vec::with_capacity(descriptor.layers.len()); + let mut ancestry_roots = Vec::new(); + let mut ancestry_nodes = BTreeMap::new(); for reference in &descriptor.layers { fetched_bytes = fetched_bytes.checked_add(reference.bytes).ok_or_else(|| { MetadataError::CorruptObject { @@ -848,11 +861,27 @@ pub async fn load_split_commit_graph( }); } layers.push(match &reference.pages { - Some(pages) => paged::decode_layer(&bytes, reference, pages, path.as_ref())?, + Some(pages) => { + let decoded = paged::decode_layer(&bytes, reference, pages, path.as_ref())?; + ancestry_roots.extend(decoded.roots); + for (hash, node) in decoded.nodes { + if ancestry_nodes + .insert(hash, (node, reference.base_ordinal, reference.commit_count)) + .is_some() + { + return corrupt("ancestry node occurs in multiple layers"); + } + } + decoded.graph + } None => decode_commit_graph_layer(&bytes, path.as_ref())?, }); } - SplitCommitGraph::new(descriptor, layers) + let graph = SplitCommitGraph::new(descriptor, layers)?; + if graph.descriptor.version == PAGED_VERSION { + ancestry::Index::build(&graph.layers)?.validate_layers(&ancestry_roots, &ancestry_nodes)?; + } + Ok(graph) } /// Load and content-verify only the small split-graph descriptor. diff --git a/crates/crab-metadata/src/split_commit_graph/ancestry.rs b/crates/crab-metadata/src/split_commit_graph/ancestry.rs new file mode 100644 index 000000000..2e90188fc --- /dev/null +++ b/crates/crab-metadata/src/split_commit_graph/ancestry.rs @@ -0,0 +1,405 @@ +//! Persistent authenticated ancestor sets; full intervals collapse shared subtrees. +use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; + +#[cfg(any(feature = "storage", test))] +use super::corrupt_at; +use super::{CommitGraphLayer, Result, corrupt}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Reference { + pub(super) hash: [u8; 32], + pub(super) start: u32, + pub(super) height: u8, +} + +impl Reference { + pub(super) fn end(self) -> u64 { + u64::from(self.start) + (1u64 << self.height) + } + + #[cfg(feature = "storage")] + pub(super) fn contains(self, ordinal: u32) -> bool { + ordinal >= self.start && u64::from(ordinal) < self.end() + } + + pub(super) fn encode(self) -> Vec { + let mut bytes = self.hash.to_vec(); + bytes.extend_from_slice(&self.start.to_le_bytes()); + bytes.push(self.height); + bytes + } + + #[cfg(any(feature = "storage", test))] + pub(super) fn decode(bytes: &[u8], path: &str) -> Result { + if bytes.len() != 37 { + return corrupt_at(path, "invalid ancestry reference length"); + } + let mut hash = [0; 32]; + hash.copy_from_slice(&bytes[..32]); + let mut start = [0; 4]; + start.copy_from_slice(&bytes[32..36]); + let reference = Self { + hash, + start: u32::from_le_bytes(start), + height: bytes[36], + }; + if reference.height > 32 + || u64::from(reference.start) & ((1u64 << reference.height) - 1) != 0 + { + return corrupt_at(path, "invalid ancestry interval"); + } + Ok(reference) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Node { + pub(super) reference: Reference, + pub(super) children: Option<[Reference; 2]>, +} + +impl Node { + fn new(start: u32, height: u8, children: Option<[Reference; 2]>) -> Self { + let mut node = Self { + reference: Reference { + hash: [0; 32], + start, + height, + }, + children, + }; + node.reference.hash = *blake3::hash(&node.encode()).as_bytes(); + node + } + + pub(super) fn encode(self) -> Vec { + let mut bytes = vec![u8::from(self.children.is_some()), self.reference.height]; + bytes.extend_from_slice(&self.reference.start.to_le_bytes()); + if let Some(children) = self.children { + for child in children { + bytes.extend_from_slice(&child.encode()); + } + } + bytes + } + + #[cfg(any(feature = "storage", test))] + pub(super) fn decode(bytes: &[u8], expected: Reference, path: &str) -> Result { + if !matches!(bytes.len(), 6 | 80) || *blake3::hash(bytes).as_bytes() != expected.hash { + return corrupt_at(path, "ancestry node identity or length mismatch"); + } + let children = match bytes[0] { + 0 if bytes.len() == 6 => None, + 1 if bytes.len() == 80 => Some([ + Reference::decode(&bytes[6..43], path)?, + Reference::decode(&bytes[43..80], path)?, + ]), + _ => return corrupt_at(path, "invalid ancestry node kind"), + }; + let mut start = [0; 4]; + start.copy_from_slice(&bytes[2..6]); + if u32::from_le_bytes(start) != expected.start || bytes[1] != expected.height { + return corrupt_at(path, "ancestry node disagrees with its reference"); + } + if let Some([left, right]) = children { + if expected.height == 0 { + return corrupt_at(path, "ancestry branch has zero height"); + } + let middle = u64::from(expected.start) + (1u64 << (expected.height - 1)); + if left.height >= expected.height + || right.height >= expected.height + || left.start < expected.start + || left.end() > middle + || u64::from(right.start) < middle + || right.end() > expected.end() + || (left.height == expected.height - 1 + && right.height == left.height + && left.hash == Self::new(left.start, left.height, None).reference.hash + && right.hash == Self::new(right.start, right.height, None).reference.hash) + { + return corrupt_at(path, "invalid or noncanonical ancestry branch"); + } + } + Ok(Self { + reference: expected, + children, + }) + } +} + +pub(super) struct Index { + pub(super) roots: Vec, + nodes: BTreeMap<[u8; 32], Node>, + first_use: BTreeMap<[u8; 32], u32>, +} + +impl Index { + pub(super) fn build(layers: &[CommitGraphLayer]) -> Result { + let mut index = Self { + roots: Vec::new(), + nodes: BTreeMap::new(), + first_use: BTreeMap::new(), + }; + for layer in layers { + for record in &layer.records { + let ordinal = u32::try_from(index.roots.len()).map_err(|_| { + crate::validation::corrupt_object("ancestry", "ordinal exceeds u32") + })?; + let mut root = None; + for parent in &record.parents { + let parent_root = + index.roots.get(*parent as usize).copied().ok_or_else(|| { + crate::validation::corrupt_object( + "ancestry", + "parent is not earlier than child", + ) + })?; + root = Some(match root { + None => parent_root, + Some(root) => index.union(root, parent_root)?, + }); + } + let leaf = index.intern(Node::new(ordinal, 0, None)); + let root = match root { + None => leaf, + Some(root) => index.union(root, leaf)?, + }; + index.roots.push(root); + } + } + // Union can create transient nodes. Only closure reachable from committed + // roots belongs in the on-disk directory; its first use fixes the layer. + for (ordinal, root) in index.roots.iter().copied().enumerate() { + let mut pending = vec![root]; + while let Some(reference) = pending.pop() { + if index.first_use.contains_key(&reference.hash) { + continue; + } + index.first_use.insert(reference.hash, ordinal as u32); + if let Some(children) = index.node(reference)?.children { + pending.extend(children); + } + } + } + Ok(index) + } + + pub(super) fn rows(&self, start: u32, count: u32) -> Result { + let end = start + .checked_add(count) + .ok_or_else(|| crate::validation::corrupt_object("ancestry", "layer range overflow"))?; + self.first_use + .iter() + .filter(|(_, ordinal)| **ordinal >= start && **ordinal < end) + .map(|(hash, _)| { + Ok(( + hash.to_vec(), + self.nodes + .get(hash) + .ok_or_else(|| { + crate::validation::corrupt_object("ancestry", "node disappeared") + })? + .encode(), + )) + }) + .collect() + } + + fn intern(&mut self, node: Node) -> Reference { + self.nodes.entry(node.reference.hash).or_insert(node); + node.reference + } + + fn node(&self, reference: Reference) -> Result { + let node = self.nodes.get(&reference.hash).copied().ok_or_else(|| { + crate::validation::corrupt_object("ancestry", "required node is absent") + })?; + if node.reference != reference { + return corrupt("ancestry reference changed"); + } + Ok(node) + } + + fn union(&mut self, left: Reference, right: Reference) -> Result { + if left == right { + return Ok(left); + } + if left.start > right.start || (left.start == right.start && left.height < right.height) { + return self.union(right, left); + } + let left_node = self.node(left)?; + if left.start == right.start && left.height == right.height { + let right_node = self.node(right)?; + let (Some([a, b]), Some([c, d])) = (left_node.children, right_node.children) else { + return Ok(if left_node.children.is_none() { + left + } else { + right + }); + }; + let first = self.union(a, c)?; + let second = self.union(b, d)?; + return self.branch(first, second); + } + if left.end() >= right.end() { + let Some([first, second]) = left_node.children else { + return Ok(left); + }; + let middle = u64::from(left.start) + (1u64 << (left.height - 1)); + if u64::from(right.start) < middle { + let first = self.union(first, right)?; + return self.branch(first, second); + } + let second = self.union(second, right)?; + return self.branch(first, second); + } + self.branch(left, right) + } + + fn branch(&mut self, left: Reference, right: Reference) -> Result { + if left.start >= right.start || left.end() > u64::from(right.start) { + return corrupt("ancestry branches overlap"); + } + let last = u32::try_from(right.end() - 1) + .map_err(|_| crate::validation::corrupt_object("ancestry", "interval exceeds u32"))?; + let height = (32 - (left.start ^ last).leading_zeros()) as u8; + let mask = u32::MAX.checked_shl(u32::from(height)).unwrap_or(0); + let start = left.start & mask; + let full = left.height + 1 == height + && right.height == left.height + && self.node(left)?.children.is_none() + && self.node(right)?.children.is_none(); + Ok(self.intern(Node::new(start, height, (!full).then_some([left, right])))) + } + + #[cfg(feature = "storage")] + pub(super) fn validate_layers( + &self, + roots: &[Reference], + rows: &BTreeMap<[u8; 32], (Vec, u32, u32)>, + ) -> Result<()> { + if self.roots != roots || self.first_use.len() != rows.len() { + return corrupt("ancestry coverage differs from the verified DAG"); + } + for hash in self.first_use.keys() { + let node = self.nodes.get(hash).ok_or_else(|| { + crate::validation::corrupt_object("ancestry", "verified node disappeared") + })?; + let first_use = self.first_use[hash]; + if rows.get(hash).is_none_or(|(bytes, start, count)| { + *bytes != node.encode() || first_use < *start || first_use - *start >= *count + }) { + return corrupt("ancestry node differs from the verified DAG"); + } + } + Ok(()) + } + + #[cfg(test)] + fn members(&self, root: Reference) -> Result> { + let mut members = BTreeSet::new(); + let mut pending = vec![root]; + while let Some(reference) = pending.pop() { + match self.node(reference)?.children { + Some(children) => pending.extend(children), + None => members.extend(u64::from(reference.start)..reference.end()), + } + } + Ok(members.into_iter().map(|value| value as u32).collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::split_commit_graph::CommitGraphRecord; + + #[test] + fn membership_storage_growth_is_bounded_for_linear_branch_and_merge_shapes() { + for shape in 0..4 { + let mut previous = None; + for count in [1_000, 2_000, 4_000, 8_000] { + let records = (0..count) + .map(|i| { + let parents = match shape { + 0 if i > 0 => vec![i - 1], + 1 if i >= 8 => vec![i - 8], + 2 if i >= 2 => vec![(i / 2 - 1) * 2, (i / 2 - 1) * 2 + 1], + _ => Vec::new(), + }; + CommitGraphRecord { + oid: [0; 20], + tree_oid: [0; 20], + commit_time: 0, + corrected_generation: 0, + parents, + } + }) + .collect(); + let index = Index::build(&[CommitGraphLayer { + base_ordinal: 0, + records, + }]) + .unwrap(); + let bytes = index + .first_use + .keys() + .map(|hash| index.nodes[hash].encode().len() + 36) + .sum::() + + index.roots.len() * 45; + if let Some(previous) = previous { + assert!( + bytes <= previous * 3, + "shape {shape}, count {count}: {bytes} > 3 * {previous}" + ); + } + previous = Some(bytes); + } + } + } + + #[test] + fn ancestry_nodes_reject_invalid_intervals_and_noncanonical_branch_shapes() { + let left = Node::new(0, 0, None).reference; + let right = Node::new(1, 0, None).reference; + let reducible = Node::new(0, 1, Some([left, right])); + let mut invalid = left.encode(); + invalid[36] = 33; + assert!(Reference::decode(&invalid, "invalid").is_err()); + assert!(Node::decode(&reducible.encode(), reducible.reference, "reducible").is_err()); + let overlapping = Node::new(0, 1, Some([left, left])); + assert!(Node::decode(&overlapping.encode(), overlapping.reference, "overlapping").is_err()); + } + + proptest::proptest! { + #[test] + fn persistent_sets_match_dags_with_arbitrary_distant_parent_edges(edges in proptest::collection::vec(proptest::collection::vec(0u16..u16::MAX, 0..5), 1..60)) { + let records = edges.iter().enumerate().map(|(i, edges)| CommitGraphRecord { + oid:[0;20], tree_oid:[0;20], commit_time:0, corrected_generation:0, + parents: if i == 0 { Vec::new() } else { edges.iter().map(|edge| u32::from(*edge) % i as u32).collect() }, + }).collect::>(); + let index = Index::build(&[CommitGraphLayer {base_ordinal:0, records:records.clone()}]).unwrap(); + for ordinal in 0..records.len() { + let mut expected=BTreeSet::new(); let mut pending=vec![ordinal as u32]; + while let Some(current)=pending.pop() { if expected.insert(current) { pending.extend(records[current as usize].parents.iter().copied()); } } + proptest::prop_assert_eq!(index.members(index.roots[ordinal]).unwrap(), expected); + } + } + + #[test] + fn persistent_sets_match_full_dag_walk(selectors in proptest::collection::vec(0u8..8, 1..80)) { + let records = selectors.iter().enumerate().map(|(i,s)| CommitGraphRecord { + oid:[0;20],tree_oid:[0;20],commit_time:0,corrected_generation:0, + parents: [(i>0 && s&1!=0).then_some(i.saturating_sub(1) as u32), (i>1 && s&2!=0).then_some(i.saturating_sub(2) as u32)].into_iter().flatten().collect(), + }).collect::>(); + let index = Index::build(&[CommitGraphLayer {base_ordinal:0, records:records.clone()}]).unwrap(); + for ordinal in 0..records.len() { + let mut expected=BTreeSet::new(); let mut pending=vec![ordinal as u32]; + while let Some(current)=pending.pop() { if expected.insert(current) { pending.extend(records[current as usize].parents.iter().copied()); } } + proptest::prop_assert_eq!(index.members(index.roots[ordinal]).unwrap(),expected); + } + } + } +} diff --git a/crates/crab-metadata/src/split_commit_graph/paged.rs b/crates/crab-metadata/src/split_commit_graph/paged.rs index 26a468343..3eac8434a 100644 --- a/crates/crab-metadata/src/split_commit_graph/paged.rs +++ b/crates/crab-metadata/src/split_commit_graph/paged.rs @@ -14,9 +14,14 @@ use crate::pages::{self, Kind, Writer}; pub struct CommitGraphLayerPages { pub records: PageRef, pub oids: PageRef, + pub ancestry_roots: PageRef, + pub ancestry_nodes: PageRef, } -pub(super) fn encode_layer(layer: &CommitGraphLayer) -> Result<(Vec, CommitGraphLayerPages)> { +pub(super) fn encode_layer( + layer: &CommitGraphLayer, + ancestry: &ancestry::Index, +) -> Result<(Vec, CommitGraphLayerPages)> { let mut writer = Writer::default(); let mut records = Vec::with_capacity(layer.records.len()); let mut oids = BTreeMap::new(); @@ -51,7 +56,40 @@ pub(super) fn encode_layer(layer: &CommitGraphLayer) -> Result<(Vec, CommitG } let records = writer.tree(Kind::CommitRecords, records)?; let oids = writer.tree(Kind::CommitOids, oids.into_iter().collect())?; - Ok((writer.bytes, CommitGraphLayerPages { records, oids })) + let end = layer + .base_ordinal + .checked_add(layer.records.len() as u32) + .ok_or_else(|| crate::validation::corrupt_object("ancestry", "ordinal range overflows"))?; + let roots = ancestry + .roots + .get(layer.base_ordinal as usize..end as usize) + .ok_or_else(|| crate::validation::corrupt_object("ancestry", "layer roots are absent"))?; + let ancestry_roots = writer.tree( + Kind::CommitAncestryRoots, + roots + .iter() + .enumerate() + .map(|(index, root)| { + ( + (layer.base_ordinal + index as u32).to_be_bytes().to_vec(), + root.encode(), + ) + }) + .collect(), + )?; + let ancestry_nodes = writer.tree( + Kind::CommitAncestryNodes, + ancestry.rows(layer.base_ordinal, layer.records.len() as u32)?, + )?; + Ok(( + writer.bytes, + CommitGraphLayerPages { + records, + oids, + ancestry_roots, + ancestry_nodes, + }, + )) } pub(super) fn validate_roots( @@ -63,12 +101,21 @@ pub(super) fn validate_roots( .records .validate(Kind::CommitRecords, reference.bytes)?; roots.oids.validate(Kind::CommitOids, reference.bytes)?; + roots + .ancestry_roots + .validate(Kind::CommitAncestryRoots, reference.bytes)?; + roots + .ancestry_nodes + .validate(Kind::CommitAncestryNodes, reference.bytes)?; let end = reference .base_ordinal .checked_add(reference.commit_count) .and_then(|end| end.checked_sub(1)); if roots.records.count != reference.commit_count || roots.oids.count != reference.commit_count + || roots.ancestry_roots.count != reference.commit_count + || roots.ancestry_roots.first != roots.records.first + || roots.ancestry_roots.last != roots.records.last || roots.records.first != reference.base_ordinal.to_be_bytes() || end.is_none_or(|end| roots.records.last != end.to_be_bytes()) { @@ -106,13 +153,20 @@ pub(super) fn record(bytes: &[u8], ordinal: u32, path: &str) -> Result, + pub(super) nodes: BTreeMap<[u8; 32], Vec>, +} + #[cfg(any(feature = "storage", test))] pub(super) fn decode_layer( bytes: &[u8], reference: &CommitGraphLayerRef, roots: &CommitGraphLayerPages, path: &str, -) -> Result { +) -> Result { validate_roots(reference, roots, path)?; let mut visited = BTreeMap::new(); let rows = pages::scan( @@ -145,6 +199,32 @@ pub(super) fn decode_layer( "graph OID directory disagrees with positional records", ); } + let ancestry_roots = pages::scan( + bytes, + &roots.ancestry_roots, + Kind::CommitAncestryRoots, + path, + &mut visited, + )? + .into_iter() + .map(|(_, value)| ancestry::Reference::decode(&value, path)) + .collect::>>()?; + let mut ancestry_nodes = BTreeMap::new(); + for (key, value) in pages::scan( + bytes, + &roots.ancestry_nodes, + Kind::CommitAncestryNodes, + path, + &mut visited, + )? { + let hash = key.as_slice().try_into().map_err(|_| { + crate::validation::corrupt_object(path, "invalid ancestry node hash key") + })?; + if *blake3::hash(&value).as_bytes() != hash { + return corrupt_at(path, "ancestry directory node hash mismatch"); + } + ancestry_nodes.insert(hash, value); + } let mut offset = 0u64; for (start, (_, page)) in visited { if start != offset { @@ -157,8 +237,12 @@ pub(super) fn decode_layer( if offset != bytes.len() as u64 { return corrupt_at(path, "graph has trailing unreferenced bytes"); } - Ok(CommitGraphLayer { - base_ordinal: reference.base_ordinal, - records, + Ok(DecodedLayer { + graph: CommitGraphLayer { + base_ordinal: reference.base_ordinal, + records, + }, + roots: ancestry_roots, + nodes: ancestry_nodes, }) } diff --git a/crates/crab-metadata/src/split_commit_graph/paged_tests.rs b/crates/crab-metadata/src/split_commit_graph/paged_tests.rs index 85322a6ad..3fad37609 100644 --- a/crates/crab-metadata/src/split_commit_graph/paged_tests.rs +++ b/crates/crab-metadata/src/split_commit_graph/paged_tests.rs @@ -387,7 +387,206 @@ proptest::proptest! { let eager = append_split_commit_graph(None, 1, "a".repeat(64), "b".repeat(64), &roots, additions.clone()).unwrap().unwrap(); let paged = append_paged_commit_graph(None, 1, "a".repeat(64), "b".repeat(64), &roots, additions).unwrap().unwrap(); let eager_layers = eager.layers.iter().map(|object| decode_commit_graph_layer(&object.bytes, "eager").unwrap()).collect::>(); - let paged_layers = paged.layers.iter().map(|object| decode_layer(&object.bytes, &object.reference, object.reference.pages.as_ref().unwrap(), "paged").unwrap()).collect::>(); + let paged_layers = paged.layers.iter().map(|object| decode_layer(&object.bytes, &object.reference, object.reference.pages.as_ref().unwrap(), "paged").unwrap().graph).collect::>(); proptest::prop_assert_eq!(&paged_layers, &eager_layers); } } + +#[tokio::test] +async fn ancestry_membership_includes_both_merge_parents_but_excludes_retained_other_history() { + let store = Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = StoreLayout::new(store.clone(), "branch-ancestry".to_owned()); + let mut additions = inputs(0, 5); + additions[2].parents = vec![oid(0)]; + additions[3].parents = vec![oid(1), oid(2)]; + additions[4].parents.clear(); + let write = append_paged_commit_graph( + None, + 1, + "a".repeat(64), + "b".repeat(64), + &[oid(3)], + additions, + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &write) + .await + .unwrap(); + let reader = CommitGraphReader::load( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let queries = [ + ( + 3, + vec![0, 1, 2, 3, 4, 2], + vec![true, true, true, true, false, true], + ), + (1, vec![0, 1, 2, 3], vec![true, true, false, false]), + ( + 4, + vec![0, 1, 2, 3, 4], + vec![false, false, false, false, true], + ), + ]; + for (root, candidates, expected) in queries { + assert_eq!( + reader + .ancestry_membership(&store, root, &candidates) + .await + .unwrap(), + expected + ); + } +} + +#[tokio::test] +async fn ancestry_lookup_reads_dependency_pages_and_charges_warm_queries() { + let (store, layout, write, observed) = fixture(10_000).await; + let mut reader = CommitGraphReader::load(&store, &layout, &write.descriptor_hash, 512 * 1024) + .await + .unwrap(); + observed.store(0, Ordering::Relaxed); + let candidates = [0, 1, 5_000, 9_999, 10_000, 5_000]; + assert_eq!( + reader + .ancestry_membership(&store, 9_999, &candidates) + .await + .unwrap(), + vec![true, true, true, true, false, true] + ); + assert!(observed.load(Ordering::Relaxed) < 512 * 1024); + observed.store(0, Ordering::Relaxed); + reader + .ancestry_membership(&store, 9_999, &candidates) + .await + .unwrap(); + assert_eq!(observed.load(Ordering::Relaxed), 0); + reader.max_bytes = reader.descriptor_bytes; + assert!( + reader + .ancestry_membership(&store, 9_999, &candidates) + .await + .is_err() + ); +} + +#[tokio::test] +async fn missing_ancestry_nodes_cannot_become_a_negative_membership_answer() { + let (store, layout, write, _) = fixture(100).await; + let reader = CommitGraphReader::load( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap(); + let roots = write.layers[0].reference.pages.as_ref().unwrap(); + let mut bytes = write.layers[0].bytes.clone(); + bytes[roots.ancestry_nodes.offset as usize] ^= 1; + store + .inner() + .put( + &layout.repo_path(&write.layers[0].reference.path), + bytes::Bytes::from(bytes).into(), + ) + .await + .unwrap(); + assert!( + reader + .ancestry_membership(&store, 99, &[200]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn full_validation_rejects_a_resigned_set_that_grants_a_non_ancestor() { + use super::super::ancestry::Reference; + use crate::pages::{self, Kind, Writer}; + let store = Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = StoreLayout::new(store.clone(), "forged-ancestry".to_owned()); + let mut additions = inputs(0, 4); + additions[2].parents = vec![oid(0)]; + let mut write = append_paged_commit_graph( + None, + 1, + "a".repeat(64), + "b".repeat(64), + &[oid(3)], + additions, + ) + .unwrap() + .unwrap(); + let object = &mut write.layers[0]; + let roots = object.reference.pages.as_ref().unwrap(); + let scan = |root: &crate::PageRef, kind| { + pages::scan( + &object.bytes, + root, + kind, + "original", + &mut Default::default(), + ) + .unwrap() + }; + let records = scan(&roots.records, Kind::CommitRecords); + let oids = scan(&roots.oids, Kind::CommitOids); + let mut ancestry_roots = scan(&roots.ancestry_roots, Kind::CommitAncestryRoots); + let old = Reference::decode(&ancestry_roots[3].1, "old").unwrap(); + let fake = vec![0, 2, 0, 0, 0, 0]; + let reference = Reference { + hash: *blake3::hash(&fake).as_bytes(), + start: 0, + height: 2, + }; + ancestry_roots[3].1 = reference.encode(); + let mut nodes = scan(&roots.ancestry_nodes, Kind::CommitAncestryNodes) + .into_iter() + .collect::>(); + nodes.remove(old.hash.as_slice()); + nodes.insert(reference.hash.to_vec(), fake); + let mut writer = Writer::default(); + let pages = CommitGraphLayerPages { + records: writer.tree(Kind::CommitRecords, records).unwrap(), + oids: writer.tree(Kind::CommitOids, oids).unwrap(), + ancestry_roots: writer + .tree(Kind::CommitAncestryRoots, ancestry_roots) + .unwrap(), + ancestry_nodes: writer + .tree(Kind::CommitAncestryNodes, nodes.into_iter().collect()) + .unwrap(), + }; + object.bytes = writer.bytes; + object.reference.hash = blake3::hash(&object.bytes).to_hex().to_string(); + object.reference.path = commit_graph_layer_path(&object.reference.hash); + object.reference.bytes = object.bytes.len() as u64; + object.reference.pages = Some(pages); + let mut descriptor = + decode_commit_graph_descriptor(&write.descriptor_bytes, "descriptor").unwrap(); + descriptor.layers[0] = object.reference.clone(); + write.descriptor_bytes = encode_commit_graph_descriptor(&descriptor).unwrap(); + write.descriptor_hash = blake3::hash(&write.descriptor_bytes).to_hex().to_string(); + upload_split_commit_graph(&store, &layout, &write) + .await + .unwrap(); + let error = load_split_commit_graph( + &store, + &layout, + &write.descriptor_hash, + DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("ancestry coverage differs from the verified DAG") + ); +} diff --git a/crates/crab-metadata/src/split_commit_graph/reader.rs b/crates/crab-metadata/src/split_commit_graph/reader.rs index 6a13b0d43..b3df89e39 100644 --- a/crates/crab-metadata/src/split_commit_graph/reader.rs +++ b/crates/crab-metadata/src/split_commit_graph/reader.rs @@ -21,6 +21,131 @@ struct Work { } impl CommitGraphReader { + /// Test membership in a captured commit's verified ancestor set, including itself. + /// + /// The caller authorizes the root against captured refs and admits request/output counts. + pub async fn ancestry_membership( + &self, + store: &Store, + root: u32, + candidates: &[u32], + ) -> Result> { + if root >= self.descriptor.commit_count { + return corrupt("ancestry root ordinal is absent"); + } + if candidates.is_empty() { + return Ok(Vec::new()); + } + let layer = self + .descriptor + .layers + .partition_point(|layer| layer.base_ordinal <= root) + .checked_sub(1) + .ok_or_else(|| crate::validation::corrupt_object("ancestry", "root has no layer"))?; + let mut work = self.work(); + let values = self + .lookup( + store, + layer, + &self.roots(&self.descriptor.layers[layer])?.ancestry_roots, + Kind::CommitAncestryRoots, + &[root.to_be_bytes().to_vec()], + &mut work, + ) + .await?; + let reference = ancestry::Reference::decode( + values[0].as_deref().ok_or_else(|| { + crate::validation::corrupt_object("ancestry", "required root is absent") + })?, + "ancestry root", + )?; + let mut requested = candidates.to_vec(); + requested.push(root); + let mut output = vec![false; requested.len()]; + let mut pending = vec![(reference, (0..requested.len()).collect::>())]; + while !pending.is_empty() { + let references = pending + .iter() + .map(|(reference, _)| *reference) + .collect::>(); + let nodes = self.ancestry_nodes(store, &references, &mut work).await?; + let mut next = BTreeMap::<[u8; 32], (ancestry::Reference, Vec)>::new(); + for ((_, positions), node) in std::mem::take(&mut pending).into_iter().zip(nodes) { + if node.children.is_none() && node.reference.end() > u64::from(root) + 1 { + return corrupt("ancestry set contains a future ordinal"); + } + for position in positions { + let candidate = requested[position]; + if candidate > root || !node.reference.contains(candidate) { + continue; + } + let Some(children) = node.children else { + output[position] = true; + continue; + }; + if let Some(child) = + children.into_iter().find(|child| child.contains(candidate)) + { + let batch = next.entry(child.hash).or_insert((child, Vec::new())); + if batch.0 != child { + return corrupt("ancestry node has conflicting routing references"); + } + batch.1.push(position); + } + } + } + pending = next.into_values().collect(); + } + if output.pop() != Some(true) { + return corrupt("ancestry root does not contain itself"); + } + Ok(output) + } + + async fn ancestry_nodes( + &self, + store: &Store, + references: &[ancestry::Reference], + work: &mut Work, + ) -> Result> { + let keys = references + .iter() + .map(|reference| reference.hash.to_vec()) + .collect::>(); + let mut output = vec![None; references.len()]; + for (layer, reference) in self.descriptor.layers.iter().enumerate() { + let values = self + .lookup( + store, + layer, + &self.roots(reference)?.ancestry_nodes, + Kind::CommitAncestryNodes, + &keys, + work, + ) + .await?; + for (index, value) in values.into_iter().enumerate() { + let Some(value) = value else { continue }; + if output[index].is_some() { + return corrupt("ancestry node occurs in multiple layers"); + } + output[index] = Some(ancestry::Node::decode( + &value, + references[index], + "ancestry node", + )?); + } + } + output + .into_iter() + .map(|node| { + node.ok_or_else(|| { + crate::validation::corrupt_object("ancestry", "required node is absent") + }) + }) + .collect() + } + /// Load a bounded paged descriptor without fetching graph layers. pub async fn load( store: &Store, diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index a886c2fb8..0d85ea124 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -83,3 +83,8 @@ ordinal/parent records and duplicate OID batches through `CommitGraphReader`. It verifies the derived graph's captured state and Git identity before reads. Control capture remains in the reported opening measurements; this component workload does not claim bounded foreground opening or replace ref visibility. + +`graph-ancestry` adds membership checks against the HEAD tip from the same captured +ref view. The native oracle uses `git merge-base --is-ancestor`, treating only +exit status 1 as a valid negative answer. Foreground graph conversion is a separate +gate; this component qualification does not bypass repository visibility. diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 274ef17c3..830b0a0a8 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -43,6 +43,7 @@ async fn main() -> Result<()> { if selection != "all" && selection != "build-indexes" && selection != "graph-pages" + && selection != "graph-ancestry" && !WORKLOADS.contains(&selection) { return Err(io::Error::other("unknown qualification operation").into()); @@ -93,8 +94,8 @@ async fn main() -> Result<()> { .await; runtime.shutdown().await; result?; - } else if selection == "graph-pages" { - qualify_graph_pages(&layout, &path, &counters).await?; + } else if matches!(selection, "graph-pages" | "graph-ancestry") { + qualify_graph_pages(&layout, &path, &counters, selection).await?; } else { for &name in WORKLOADS .iter() @@ -128,11 +129,12 @@ async fn qualify_graph_pages( layout: &StoreLayout, path: &GitPath, counters: &Measurements, + selection: &str, ) -> Result<()> { use crab_metadata::split_commit_graph::{ CommitGraphReader, DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, }; - let open = async || -> Result { + let open = async || -> Result<(CommitGraphReader, gix_hash::ObjectId)> { let root = crab_metadata::capsule_protocol::load_root(layout).await?; let indexes = crab_metadata::capsule_protocol::load_browse_indexes(layout) .await? @@ -164,9 +166,15 @@ async fn qualify_graph_pages( { return Err(io::Error::other("graph does not match captured Git identity").into()); } - Ok(reader) + let tip = view + .peeled_refs() + .get(view.head()) + .or_else(|| view.refs().get(view.head())) + .ok_or_else(|| io::Error::other("captured HEAD is absent"))?; + let tip = gix_hash::ObjectId::from_hex(tip.as_bytes())?; + Ok((reader, tip)) }; - let reader = counters.measure("open", "cold", open()).await?; + let (reader, tip) = counters.measure("open", "cold", open()).await?; let count = reader.descriptor().commit_count; if count == 0 { return Err(io::Error::other("graph is empty").into()); @@ -204,19 +212,33 @@ async fn qualify_graph_pages( "tree": gix_hash::ObjectId::Sha1(record.tree_oid).to_string(), "parents": parent_oids, "committer_seconds": record.commit_time}) }).collect::>(); - Ok(json!({"records": rows})) + let membership = if selection == "graph-ancestry" { + let gix_hash::ObjectId::Sha1(tip_bytes) = tip else { + return Err(io::Error::other("unsupported graph tip hash").into()); + }; + let tip_ordinal = reader.ordinals(layout.store(), &[tip_bytes]).await?[0] + .ok_or_else(|| io::Error::other("captured HEAD is not a graph commit"))?; + Some( + reader + .ancestry_membership(layout.store(), tip_ordinal, &requested) + .await?, + ) + } else { + None + }; + Ok(json!({"records": rows, "root": tip.to_string(), "membership": membership})) }; - let cold = counters.measure("graph-pages", "cold", query()).await?; - let warm = counters.measure("graph-pages", "warm", query()).await?; + let cold = counters.measure(selection, "cold", query()).await?; + let warm = counters.measure(selection, "warm", query()).await?; if cold != warm { return Err(io::Error::other("cold/warm graph queries differ").into()); } println!( "{}", - json!({"kind":"result", "operation":"graph-pages", "path":path.as_bytes(), "value":cold}) + json!({"kind":"result", "operation":selection, "path":path.as_bytes(), "value":cold}) ); let reopened = counters.measure("open", "warm", open()).await?; - if reopened.descriptor() != reader.descriptor() { + if reopened.0.descriptor() != reader.descriptor() || reopened.1 != tip { return Err(io::Error::other("graph changed during qualification").into()); } Ok(()) diff --git a/crates/crab-write/README.md b/crates/crab-write/README.md index bcb0e2baf..0372db111 100644 --- a/crates/crab-write/README.md +++ b/crates/crab-write/README.md @@ -59,12 +59,13 @@ orchestrator publishes graph readiness. `build_path_state` then attaches that verified graph, resumes attribution from its checkpoint and persists progress every 32 commits. Completion replaces the partial record through the returned CAS token; cached readers observe both phases without a ref change. Graph -construction writes paged descriptor version 2, reuses unchanged paged layers +construction writes paged descriptor version 3, reuses unchanged paged layers and performs bounded-directory carries. Released manifest maintenance retains the eager version-1 encoding. The capsule owner rebuilds an eager browse graph even when its state digest is otherwise current, preserving reusable attribution through the existing checkpoint boundary. Full readback validates graph/page -closure before the returned descriptor can become readiness. Graph discovery +closure, including recomputed ancestor-set recurrence, before the returned +descriptor can become readiness. Graph discovery drains a separate bounded operation per batch. `generation::with_generation_owner` shares renewed owner election and global/repository GC fences between v1 maintenance and the v2 orchestrator; all acquired leases are released on returned failure or cancellation. The From 401f6390da9e14e3cbb046f73cb333dfb04fe2bc Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 17:46:26 -0700 Subject: [PATCH 010/127] perf(remote-git)!: page foreground graph queries Use captured ref roots and admitted page reads for foreground graph queries. BREAKING CHANGE: commits_reachable_from is asynchronous; callers must await its proof. Crate API versioning remains required before publication. --- .../docs/architecture/indexed-remote-reads.md | 55 +- crab/src/cmd/gc/mod.rs | 9 +- crab/src/git/upload_pack_wire.rs | 6 +- crates/crab-http-server/src/projection.rs | 2 +- crates/crab-metadata/README.md | 23 +- .../examples/qualify_graph_build.rs | 80 ++ crates/crab-metadata/src/path_state/reader.rs | 36 +- .../src/path_state/reader_tests.rs | 131 +++- crates/crab-metadata/src/path_state/scan.rs | 21 +- .../crab-metadata/src/split_commit_graph.rs | 2 +- .../src/split_commit_graph/reader.rs | 54 ++ crates/crab-remote-git/README.md | 6 + crates/crab-remote-git/REFERENCE.md | 12 +- crates/crab-remote-git/src/commit_graph.rs | 694 ++++++------------ crates/crab-remote-git/src/operation.rs | 24 + crates/crab-remote-git/src/path_state.rs | 4 +- crates/crab-remote-git/src/repository.rs | 119 ++- .../src/repository/tests/snapshot_indexes.rs | 191 ++++- crates/crab-remote-git/src/snapshot.rs | 70 +- .../tests/remote_repository.rs | 138 ++-- crates/crab-remote/examples/qualify_remote.rs | 9 +- 21 files changed, 1027 insertions(+), 659 deletions(-) create mode 100644 crates/crab-metadata/examples/qualify_graph_build.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index ad87d8b8b..dac082889 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -29,7 +29,7 @@ Xet files and unified local staging are subsequent workstreams. | 1 | Reproducible cold/warm baseline and contracts | Native Git oracle, isolated query/open measurements, process resources, HTTP bytes/classes, index decode counts, history scaling receipts | Implemented; RustFS history-scaling baseline passed; cloud latency cell pending | | 2 | Authenticated bounded path-state pages | Equivalence/property tests; malformed hashes, lengths, ordering, routing and allocations rejected; bounded internal directories | Implemented; 23 focused tests, GC retention regression and 1,000/10,000-commit RustFS format qualification passed | | 3 | Lazy canonical and projected attribution | Fixed directory page reads only needed pages; branch/force-push/recreation/raw-path/restart E2E proof | Canonical scaling/transitions passed; streaming Cell ingestion, projected native Git oracle and two process starts passed on RustFS; broader cloud/lifecycle qualification remains open | -| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Paged producer and authenticated ancestor-set component qualified; foreground conversion and bounded opening remain pending | +| 4 | Lazy graph lookup and bounded opening | Paged OID/ordinal/parent/summary queries; reachable versus retained proof; tag/shallow/cancellation/cleanup checks | Foreground graph/path integration qualified on RustFS; bounded control opening, broader shallow/lifecycle and release/cloud qualification remain pending | | 5 | Change-point history and first-parent blame | Exact native Git origins under matching semantics; fixed file changes with growing unrelated history; comparison/result cache isolation | Pending | | 6 | Indexed historical Xet catalog | Indexed checkpoints and visible deltas; bounded compaction; 10-GiB edits and byte-identical cold historical hydration; protected receive and sibling consumers | Pending | | 7 | Lifecycle and migration | Interrupted publication/builds, concurrent owners, GC retained roots/readers, stale caches and released-format migration qualification | Pending; also required in every earlier implementation | @@ -425,3 +425,56 @@ checks pass 27 graph/ancestry tests, 21 path-state tests, capsule browse publica integration, payload-only compilation, strict metadata library Clippy, qualifier diagnostic Clippy and architecture guardrails. The negative-control source is restored; final source/binary hashes and receipts are retained externally. + +## Foreground graph integration + +Capsule foreground handles now retain the paged graph reader and captured ref +roots. OID resolution, ordered parent validation, generation lookup and membership +are asynchronous and use the operation's admitted store and cancellation. Path-state +binds the paged descriptor's positional identity and validates consumed records +through batched graph reads, without recomputing the complete ordinal digest. +The released manifest format still receives full graph validation; its membership +work charges metadata entries rather than the raw-history budget. + +The final-source repository-index run passes nine tests, including rejection of full +graph-layer body reads, admission on warm results and dependency-page cancellation. +The retained-OID public resolution check passes. The shallow-fetch caller is now +async, and the native Git tag fixture passes for both formats. +The resolve qualifier now checks full-OID visibility +after obtaining the selected ref's commit; older resolve timings therefore measure +less work and are not direct comparisons for this expanded workload. Four Python +oracle regressions and architecture guardrails pass. Real-store foreground scaling, +build-cost and consumer qualification are recorded below. Broad CI remains +required; this is not a completed stage-4 gate. Outer control opening and later +history/catalog/lifecycle/blame stages remain open. + +Release audit: the synchronous `commits_reachable_from` introduced at +`acf0408de89` is reachable from v1.0.1 and v1.2.0 through v1.2.4. Its conversion +to asynchronous I/O is a Rust source API change; callers now await the proof. +The current crate still declares 0.1.0. A breaking crate-version transition and +consumer migration remain required before publication; the passing architecture +version-alignment check is not proof of that migration. Version-1 stored graphs +retain complete validation and native protocol consumers use the async proof. + +`foreground-graph-r1` passes 20 isolated cold/warm attribution comparisons at +each of 100/1k/10k commits and a full-OID resolution comparison at each size. +Native Git verifies all results. At 10k, query reads reach at most 138,597 bytes +and decoded graph/path pages reach at most 110,104 bytes. Graph opening decodes +at most 35,764 bytes; the outer opening still downloads up to 6,094,562 bytes. +Cold attribution p95 is 45.792 / 31.149 / 54.412 ms on the shared debug host. +These pass the frozen local fixture bounds but are not release/cloud latency +qualification or direct comparisons with earlier release-binary receipts. +Final-source checks pass nine repository-index tests, the native Git tag/retention +fixture for both formats, 22 path-state tests, four Python oracle tests and +architecture guardrails. The qualifier builds with real consumer dependencies. +Strict metadata/remote-Git library Clippy and the CLI binary consumer check pass. +Build-qualifier Clippy also passes. No dependency or lockfile changes were needed. + +`graph-build-cpu-r1` records 20 samples for each of four graph shapes at +1k/2k/4k/8k commits: 320 component builds. At the two largest sizes, process CPU +growth stays at most 2.358x and complete encoded index growth at most 2.179x per +doubling. The 8k criss-cross fixture uses 1.29 median CPU seconds, 7,228,939 encoded +bytes and 62,357,504 median RSS bytes. Synthetic OIDs isolate the builder; these +are debug component receipts, not native Git integrity, release-image performance +or whole-publication measurements. Release/provider and broader lifecycle gates +remain open. Fixtures, source/binary hashes and raw receipts remain external. diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index eea3c325d..ff87d48cc 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -5880,13 +5880,18 @@ mod tests { store.as_storage(), &layout, ¤t_path_state_hash, - ¤t_graph, + crab_metadata::split_commit_graph::CommitGraphView::Materialized(¤t_graph), crab_metadata::path_state::DEFAULT_MAX_PATH_STATE_BYTES, ) .await .unwrap(); let attribution = reader - .latest(store.as_storage(), ¤t_graph, &[3; 20], &[b"file"]) + .latest( + store.as_storage(), + crab_metadata::split_commit_graph::CommitGraphView::Materialized(¤t_graph), + &[3; 20], + &[b"file"], + ) .await .unwrap(); assert_eq!(attribution[0].oid, [3; 20]); diff --git a/crab/src/git/upload_pack_wire.rs b/crab/src/git/upload_pack_wire.rs index 30285003f..9f5f7777d 100644 --- a/crab/src/git/upload_pack_wire.rs +++ b/crab/src/git/upload_pack_wire.rs @@ -1691,7 +1691,7 @@ async fn visible_objects_catalog( operation.finish(result).await.map_err(remote_error) } -fn native_shallow_visibility( +async fn native_shallow_visibility( repository: &RemoteGitRepository, request: &FetchRequest, visible_ref_names: &[String], @@ -1714,6 +1714,7 @@ fn native_shallow_visibility( let started = Instant::now(); let Some(visible) = repository .commits_reachable_from(&object_ids, &roots, cancellation) + .await .map_err(remote_error)? else { tracing::debug!( @@ -1990,7 +1991,8 @@ async fn write_preplanned_cached_fetch_response( let cancellation = &producer_cancellation; if native_shallow_pack_eligible(request) && proof.as_catalog().is_some() { let (common_haves, shallow_visible) = - native_shallow_visibility(repository, request, visible_ref_names, cancellation)?; + native_shallow_visibility(repository, request, visible_ref_names, cancellation) + .await?; if !common_haves.is_empty() && shallow_visible { let visible_tag_refs = visible_ref_names .iter() diff --git a/crates/crab-http-server/src/projection.rs b/crates/crab-http-server/src/projection.rs index 53f8738d3..a62cf9b9c 100644 --- a/crates/crab-http-server/src/projection.rs +++ b/crates/crab-http-server/src/projection.rs @@ -445,7 +445,7 @@ async fn build_epoch_contents( store, layout, path_hash, - &graph, + crab_metadata::split_commit_graph::CommitGraphView::Materialized(&graph), options.object_limits().max_path_state_bytes, ) .await diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 3d2cd7e31..c34376dbe 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -81,9 +81,26 @@ encoding: upstream v1.0.1 through v1.2.4 publish `Manifest::commit_graph_hash` f CLI push and read it in the remote helper. Lazy capsule readers reject that format; maintenance may rebuild the unreleased eager browse index into pages. Path and graph readers share the authenticated page codec and an 8-MiB/256-page -cache; each cache belongs to one immutable captured layer directory. The current -foreground graph handshake still materializes reachability and is not yet a -bounded-opening implementation. +cache; each cache belongs to one immutable captured layer directory. +`CommitGraphView` supplies the same positional queries to complete maintenance +graphs and immutable paged readers. Path-state opening binds the paged descriptor's +verified positional digest; queried records are checked through batched graph reads. +It does not hash the complete paged graph during foreground opening. The outer +capsule/ref-control capture remains a separate bounded-opening qualification gate. + +`qualify_graph_build` measures paged graph construction on the ancestry-growth +fixtures: linear, eight branches, criss-cross merges and disconnected commits. +Run each shape at 1k/2k/4k/8k commits under `/usr/bin/time -l`; repeat samples to +compare process CPU, RSS, complete encoded index bytes and ancestry-node counts. +Its synthetic OIDs measure builder cost, not Git object integrity or publication. +Keep results separate from real-store maintenance and foreground qualification. +For example, after selecting the checkout's external Cargo target: + +```sh +cargo build -p crab-metadata --example qualify_graph_build --release --locked +/usr/bin/time -l "$CARGO_TARGET_DIR/release/examples/qualify_graph_build" branches 8000 +``` + Path-state descriptor version 5 requires exact-name tree comparison semantics and authenticated pages; retired descriptors are rejected. Maintenance rebuilds completed indexes and resumable checkpoints through the diff --git a/crates/crab-metadata/examples/qualify_graph_build.rs b/crates/crab-metadata/examples/qualify_graph_build.rs new file mode 100644 index 000000000..18ee41cd2 --- /dev/null +++ b/crates/crab-metadata/examples/qualify_graph_build.rs @@ -0,0 +1,80 @@ +//! Measure paged graph construction on the frozen ancestry-growth DAG shapes. +use std::{error::Error, io, time::Instant}; + +use crab_metadata::split_commit_graph::{CommitGraphInput, append_paged_commit_graph}; +use serde_json::json; + +fn oid(index: u32) -> [u8; 20] { + let mut value = [0; 20]; + value[16..].copy_from_slice(&index.to_be_bytes()); + value +} + +fn main() -> Result<(), Box> { + let args = std::env::args().skip(1).collect::>(); + let [shape, count] = args.as_slice() else { + return Err(io::Error::other( + "usage: qualify_graph_build ", + ) + .into()); + }; + let count = count.parse::()?; + if count == 0 || count > 100_000 { + return Err(io::Error::other("commit count must be between 1 and 100000").into()); + } + if !matches!( + shape.as_str(), + "linear" | "branches" | "criss-cross" | "disconnected" + ) { + return Err(io::Error::other("unknown graph shape").into()); + } + // Synthetic OIDs isolate builder cost; these are not Git object-integrity + // receipts. Parent layouts match the ancestry storage-growth regression. + let inputs = (0..count) + .map(|index| { + let parents = match shape.as_str() { + "linear" if index > 0 => vec![index - 1], + "branches" if index >= 8 => vec![index - 8], + "criss-cross" if index >= 2 => { + vec![(index / 2 - 1) * 2, (index / 2 - 1) * 2 + 1] + } + _ => Vec::new(), + }; + CommitGraphInput { + oid: oid(index), + tree_oid: [0; 20], + commit_time: 0, + parents: parents.into_iter().map(oid).collect(), + } + }) + .collect::>(); + let roots = inputs.iter().map(|input| input.oid).collect::>(); + let pack_index_hash = "1".repeat(64); + let validation_digest = "2".repeat(64); + let started = Instant::now(); + let write = + append_paged_commit_graph(None, 1, pack_index_hash, validation_digest, &roots, inputs)? + .ok_or_else(|| io::Error::other("complete fixture graph was not produced"))?; + let elapsed = started.elapsed(); + let bytes = write.descriptor_bytes.len() as u64 + + write + .layers + .iter() + .map(|layer| layer.bytes.len() as u64) + .sum::(); + let nodes = write.layers.iter().try_fold(0_u64, |total, layer| { + let pages = layer + .reference + .pages + .as_ref() + .ok_or_else(|| io::Error::other("fixture graph lacks paged roots"))?; + Ok::<_, io::Error>(total + u64::from(pages.ancestry_nodes.count)) + })?; + println!( + "{}", + json!({"schema":1, "shape":shape, "commits":count, + "build_elapsed_us":elapsed.as_micros(), "index_bytes":bytes, + "ancestry_nodes":nodes, "descriptor_hash":write.descriptor_hash}) + ); + Ok(()) +} diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index cb9a7d2cc..e98e27e13 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -11,7 +11,7 @@ use super::{ corrupt, corruption, validate_path, }; use crate::pages::{self as paged, Kind, Page, PageRef}; -use crate::split_commit_graph::{CommitGraphLayerRef, SplitCommitGraph}; +use crate::split_commit_graph::{CommitGraphLayerRef, CommitGraphView, SplitCommitGraph}; #[path = "scan.rs"] mod scan; @@ -45,14 +45,14 @@ impl PathStateReader { store: &Store, layout: &StoreLayout, hash: &str, - graph: &SplitCommitGraph, + graph: CommitGraphView<'_>, max_bytes: u64, ) -> Result { let (descriptor, descriptor_bytes) = super::storage::load_path_state_descriptor_with_size(store, layout, hash, max_bytes) .await?; if !matches_graph(&descriptor, graph) - || descriptor.commit_ordinal_digest != graph.ordinal_digest() + || descriptor.commit_ordinal_digest != graph.ordinal_digest()? { return corrupt("path-state descriptor does not match its commit graph"); } @@ -63,7 +63,7 @@ impl PathStateReader { .collect(); Ok(Self { descriptor, - graph_layers: graph.descriptor.layers.clone(), + graph_layers: graph.descriptor().layers.clone(), paths, descriptor_bytes, max_bytes, @@ -75,13 +75,14 @@ impl PathStateReader { pub async fn latest( &self, store: &Store, - graph: &SplitCommitGraph, + graph: CommitGraphView<'_>, start: &[u8; 20], paths: &[&[u8]], ) -> Result> { // Scalars can match two rebuilt graphs with different ordinals. The // immutable layer identities pin the ordering verified during load. - if !matches_graph(&self.descriptor, graph) || self.graph_layers != graph.descriptor.layers { + if !matches_graph(&self.descriptor, graph) || self.graph_layers != graph.descriptor().layers + { return corrupt("path-state query graph identity changed"); } if paths.is_empty() { @@ -91,7 +92,9 @@ impl PathStateReader { validate_path(path, false)?; } let start_ordinal = graph - .ordinal(start) + .ordinals(store, &[*start]) + .await? + .remove(0) .ok_or_else(|| corruption("start commit is absent from graph"))?; let mut work = Work { maximum: self.max_bytes, @@ -206,7 +209,7 @@ impl PathStateReader { async fn records( &self, store: &Store, - graph: &SplitCommitGraph, + graph: CommitGraphView<'_>, ordinals: &[u32], work: &mut Work, ) -> Result> { @@ -225,6 +228,7 @@ impl PathStateReader { grouped.entry(layer).or_default().push((position, ordinal)); } let mut output = vec![None; ordinals.len()]; + let commits = graph.records(store, ordinals).await?; for (layer, requests) in grouped { let reference = &self.descriptor.layers[layer]; let keys = requests @@ -241,11 +245,11 @@ impl PathStateReader { work, ) .await?; - for ((position, ordinal), value) in requests.into_iter().zip(values) { + for ((position, _), value) in requests.into_iter().zip(values) { let value = value.ok_or_else(|| corruption("positional record is missing"))?; let record = codec::decode_record(&value, self.paths[layer].as_ref())?; - let commit = graph - .record(ordinal) + let commit = commits[position] + .as_ref() .ok_or_else(|| corruption("commit graph record is missing"))?; if record.oid != commit.oid || record.first_parent != commit.parents.first().copied() @@ -433,11 +437,11 @@ impl PathStateReader { } } -fn matches_graph(descriptor: &PathStateDescriptor, graph: &SplitCommitGraph) -> bool { - descriptor.generation == graph.descriptor.generation - && descriptor.pack_index_hash == graph.descriptor.pack_index_hash - && descriptor.git_validation_digest == graph.descriptor.git_validation_digest - && descriptor.commit_count == graph.descriptor.commit_count +fn matches_graph(descriptor: &PathStateDescriptor, graph: CommitGraphView<'_>) -> bool { + descriptor.generation == graph.descriptor().generation + && descriptor.pack_index_hash == graph.descriptor().pack_index_hash + && descriptor.git_validation_digest == graph.descriptor().git_validation_digest + && descriptor.commit_count == graph.descriptor().commit_count } #[cfg(test)] diff --git a/crates/crab-metadata/src/path_state/reader_tests.rs b/crates/crab-metadata/src/path_state/reader_tests.rs index 72c68fc42..7625b1316 100644 --- a/crates/crab-metadata/src/path_state/reader_tests.rs +++ b/crates/crab-metadata/src/path_state/reader_tests.rs @@ -56,7 +56,7 @@ async fn batched_pages_preserve_old_roots_order_and_repeated_paths() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await @@ -69,7 +69,12 @@ async fn batched_pages_preserve_old_roots_order_and_repeated_paths() { ]; for commit in [[1; 20], [2; 20]] { let actual = reader - .latest(&store, &graph, &commit, &paths) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &commit, + &paths, + ) .await .unwrap(); let expected = write.index.latest(&graph, &commit, &paths).unwrap(); @@ -89,7 +94,7 @@ async fn query_reads_only_dependency_pages_and_caches_verified_results() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await @@ -97,12 +102,22 @@ async fn query_reads_only_dependency_pages_and_caches_verified_results() { bytes.store(0, Ordering::Relaxed); let paths = [b"tree/name-0001".as_slice()]; reader - .latest(&store, &graph, &[2; 20], &paths) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &paths, + ) .await .unwrap(); let cold = bytes.swap(0, Ordering::Relaxed); reader - .latest(&store, &graph, &[2; 20], &paths) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &paths, + ) .await .unwrap(); let full = write @@ -122,7 +137,7 @@ async fn query_rejects_a_rebuilt_graph_with_matching_scalar_identity() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await @@ -130,7 +145,12 @@ async fn query_rejects_a_rebuilt_graph_with_matching_scalar_identity() { graph.descriptor.layers[0].hash = "f".repeat(64); assert!( reader - .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &[b"tree/name-0001"] + ) .await .is_err() ); @@ -155,14 +175,19 @@ async fn required_corrupt_or_missing_pages_never_become_a_lookup_miss() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await .unwrap(); assert!( reader - .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &[b"tree/name-0001"] + ) .await .is_err() ); @@ -176,7 +201,7 @@ async fn cache_identity_includes_the_page_kind() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await @@ -201,19 +226,29 @@ async fn warm_cache_cannot_bypass_the_query_byte_limit() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await .unwrap(); reader - .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &[b"tree/name-0001"], + ) .await .unwrap(); reader.max_bytes = reader.descriptor_bytes; assert!( reader - .latest(&store, &graph, &[2; 20], &[b"tree/name-0001"]) + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &[b"tree/name-0001"] + ) .await .is_err() ); @@ -229,11 +264,77 @@ async fn page_queries_remain_send_without_holding_cache_guards_across_io() { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await .unwrap(); - let result = require_send(reader.latest(&store, &graph, &[2; 20], &[b"tree/name-0001"])).await; + let result = require_send(reader.latest( + &store, + CommitGraphView::Materialized(&graph), + &[2; 20], + &[b"tree/name-0001"], + )) + .await; assert!(result.is_ok()); } + +#[tokio::test] +async fn paged_graph_binding_matches_materialized_attribution_without_layer_loading() { + use crate::split_commit_graph::{ + CommitGraphReader, append_paged_commit_graph, upload_split_commit_graph, + }; + let (store, layout, graph, write) = fixture().await; + let paged = append_paged_commit_graph( + Some(graph.clone()), + graph.descriptor.generation, + graph.descriptor.pack_index_hash.clone(), + graph.descriptor.git_validation_digest.clone(), + &[[2; 20]], + Vec::new(), + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&store, &layout, &paged) + .await + .unwrap(); + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = store.with_read_byte_observer(Arc::new(move |read| { + observed.fetch_add(read, Ordering::Relaxed); + })); + let graph_reader = CommitGraphReader::load(&store, &layout, &paged.descriptor_hash, 512 * 1024) + .await + .unwrap(); + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + CommitGraphView::Paged(&graph_reader), + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + assert_eq!( + bytes.load(Ordering::Relaxed), + paged.descriptor_bytes.len() as u64 + reader.descriptor_bytes + ); + let paths = [ + b"tree/name-0001".as_slice(), + b"tree/name-2999", + b"tree/name-0001", + ]; + let expected = write.index.latest(&graph, &[2; 20], &paths).unwrap(); + assert_eq!( + reader + .latest( + &store, + CommitGraphView::Paged(&graph_reader), + &[2; 20], + &paths + ) + .await + .unwrap(), + expected + ); +} diff --git a/crates/crab-metadata/src/path_state/scan.rs b/crates/crab-metadata/src/path_state/scan.rs index a850f99ed..e63163607 100644 --- a/crates/crab-metadata/src/path_state/scan.rs +++ b/crates/crab-metadata/src/path_state/scan.rs @@ -55,7 +55,9 @@ impl PathStateReader { store: &'a Store, graph: &'a SplitCommitGraph, ) -> Result> { - if !matches_graph(&self.descriptor, graph) || self.graph_layers != graph.descriptor.layers { + if !matches_graph(&self.descriptor, CommitGraphView::Materialized(graph)) + || self.graph_layers != graph.descriptor.layers + { return corrupt("path-state scan graph identity changed"); } let bytes = @@ -347,7 +349,7 @@ mod tests { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await @@ -388,7 +390,7 @@ mod tests { &store, &layout, &write.descriptor_hash, - &graph, + CommitGraphView::Materialized(&graph), DEFAULT_MAX_PATH_STATE_BYTES, ) .await @@ -468,10 +470,15 @@ mod tests { ) .await .unwrap(); - let reader = - PathStateReader::load(&store, &layout, &hash, &graph, DEFAULT_MAX_PATH_STATE_BYTES) - .await - .unwrap(); + let reader = PathStateReader::load( + &store, + &layout, + &hash, + CommitGraphView::Materialized(&graph), + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); let mut scan = reader.scan(&store, &graph).unwrap(); while matches!(scan.next().await, Ok(Some(_))) {} assert!(scan.next().await.is_err()); diff --git a/crates/crab-metadata/src/split_commit_graph.rs b/crates/crab-metadata/src/split_commit_graph.rs index ed2aa1af6..f58899254 100644 --- a/crates/crab-metadata/src/split_commit_graph.rs +++ b/crates/crab-metadata/src/split_commit_graph.rs @@ -16,7 +16,7 @@ pub use paged::CommitGraphLayerPages; #[cfg(feature = "storage")] mod reader; #[cfg(feature = "storage")] -pub use reader::CommitGraphReader; +pub use reader::{CommitGraphReader, CommitGraphView}; const PAGED_VERSION: u32 = 3; const MAX_PAGED_LAYERS: usize = 32; diff --git a/crates/crab-metadata/src/split_commit_graph/reader.rs b/crates/crab-metadata/src/split_commit_graph/reader.rs index b3df89e39..5190f2665 100644 --- a/crates/crab-metadata/src/split_commit_graph/reader.rs +++ b/crates/crab-metadata/src/split_commit_graph/reader.rs @@ -20,6 +20,60 @@ struct Work { seen: BTreeSet<(usize, u64)>, } +/// Read-only positional access for maintenance graphs and immutable paged readers. +#[derive(Clone, Copy)] +pub enum CommitGraphView<'a> { + Materialized(&'a SplitCommitGraph), + Paged(&'a CommitGraphReader), +} + +impl CommitGraphView<'_> { + /// Return the descriptor defining this view's positional identity. + pub fn descriptor(&self) -> &CommitGraphDescriptor { + match self { + Self::Materialized(graph) => &graph.descriptor, + Self::Paged(graph) => graph.descriptor(), + } + } + + pub(crate) fn ordinal_digest(&self) -> Result> { + match self { + Self::Materialized(graph) => Ok(graph.ordinal_digest().into()), + Self::Paged(graph) => graph + .descriptor + .commit_ordinal_digest + .as_deref() + .map(std::borrow::Cow::Borrowed) + .ok_or_else(|| { + crate::validation::corrupt_object("graph", "positional identity is absent") + }), + } + } + + /// Resolve OIDs while preserving requested positions and duplicates. + pub async fn ordinals(&self, store: &Store, oids: &[[u8; 20]]) -> Result>> { + match self { + Self::Materialized(graph) => Ok(oids.iter().map(|oid| graph.ordinal(oid)).collect()), + Self::Paged(graph) => graph.ordinals(store, oids).await, + } + } + + /// Resolve the requested records, validating the consumed paged parent edges. + pub async fn records( + &self, + store: &Store, + ordinals: &[u32], + ) -> Result>> { + match self { + Self::Materialized(graph) => Ok(ordinals + .iter() + .map(|ordinal| graph.record(*ordinal).cloned()) + .collect()), + Self::Paged(graph) => graph.records(store, ordinals).await, + } + } +} + impl CommitGraphReader { /// Test membership in a captured commit's verified ancestor set, including itself. /// diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index d50d8ef5c..dec6352d4 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -31,6 +31,12 @@ identity, integrity and admission checks; a journal ref change discards stale base indexes even when its generation is unchanged. Snapshots without indexes make no additional index requests. Publication of capsule-bound browse indexes is a separate maintenance responsibility, not part of opening or pushing. +Capsule graphs retain an immutable paged reader and captured commit roots; +membership reads exact ancestor sets rather than allocating a full reachability +bitmap. Graph/OID/parent reads share operation admission and cancellation, including +warm queries. Released manifest graph version 1 retains complete graph validation. +`commits_reachable_from` is asynchronous and uses a metadata-admitted operation +without opening the object-locator database. Without a commit graph, full commit-ID resolution checks matching ref tips before traversing other roots. Matching annotated tags still require verified peeling; non-tip commits retain bounded breadth-first reachability traversal. diff --git a/crates/crab-remote-git/REFERENCE.md b/crates/crab-remote-git/REFERENCE.md index 6ad7c85e9..bb6e70ff0 100644 --- a/crates/crab-remote-git/REFERENCE.md +++ b/crates/crab-remote-git/REFERENCE.md @@ -224,7 +224,7 @@ diff, blame, archive, storage, inflation, and response work have independent aggregate limits. History remains authoritative over verified raw commit objects. When the -manifest names an immutable split commit graph, open bounds the complete graph +released manifest names a version-1 split commit graph, open bounds the complete graph to 128 MiB, verifies every descriptor and layer Blake3 identity, validates stable ordinals, parent closure, corrected generations, and the exact manifest generation/pack/digest tuple. A snapshot uses it only while each positional @@ -234,6 +234,16 @@ commit. First-parent path cursors carry the next verified raw parent, so later pages do not replay newer commits. A matching complete graph groups bounded raw commit and tree reads for range coalescing without becoming the history authority. +Capsule browse graph version 3 uses descriptor and dependency-page reads instead +of loading the complete graph. Its captured ref roots authorize exact ancestor-set +queries; physically retained records do not grant visibility. OID, parent and +generation access is batched and asynchronous. Required page corruption fails +explicitly. Path-state binds the immutable descriptor's verified positional digest +and validates consumed records through the graph reader. Complete recurrence and +physical coverage validation remain the publication/maintenance owner's obligation. +This removes the graph bitmap and full positional hashing from capsule foreground +reads; outer capsule control capture is a separate remaining opening cost. + All snapshot constructors use this same graph validation when their supplied metadata names an index. Attribution captures the path-state descriptor and graph identity, batches path prefixes, and fetches only required authenticated diff --git a/crates/crab-remote-git/src/commit_graph.rs b/crates/crab-remote-git/src/commit_graph.rs index 332097b23..187a77b91 100644 --- a/crates/crab-remote-git/src/commit_graph.rs +++ b/crates/crab-remote-git/src/commit_graph.rs @@ -1,27 +1,39 @@ use std::collections::{HashMap, HashSet}; -use crab_metadata::split_commit_graph::{SplitCommitGraph, load_split_commit_graph}; +use crab_metadata::split_commit_graph::{ + CommitGraphReader, CommitGraphRecord, CommitGraphView, SplitCommitGraph, + load_split_commit_graph, load_split_commit_graph_descriptor, +}; use crab_storage::{Store, StoreLayout}; use gix_hash::ObjectId; use tokio_util::sync::CancellationToken; -use crate::{BudgetDimension, CorruptionStage, Error, RemoteGitRuntime, Result}; +use crate::{BudgetDimension, CorruptionStage, Error, OperationContext, RemoteGitRuntime, Result}; -/// Validated, generation-bound acceleration data for raw commit traversal. +#[derive(Debug)] +enum Graph { + Manifest(SplitCommitGraph), + Capsule(CommitGraphReader), +} + +/// Validated positional graph bound to captured ref roots, never physical retention. #[derive(Debug)] pub(crate) struct CommitGraphIndex { - graph: SplitCommitGraph, - pinned_reachable: Vec, + graph: Graph, + roots: Vec, } impl CommitGraphIndex { - pub(crate) const fn graph(&self) -> &SplitCommitGraph { - &self.graph + pub(crate) fn view(&self) -> CommitGraphView<'_> { + match &self.graph { + Graph::Manifest(graph) => CommitGraphView::Materialized(graph), + Graph::Capsule(graph) => CommitGraphView::Paged(graph), + } } #[expect( clippy::too_many_arguments, - reason = "graph loading validates manifest identity and two cancellation scopes" + reason = "graph loading binds captured state and operation admission" )] pub(crate) async fn load( store: &Store, @@ -34,512 +46,256 @@ impl CommitGraphIndex { max_bytes: u64, cancellation: &CancellationToken, runtime: &RemoteGitRuntime, - budget: &crate::budget::OperationBudget, - operation: &crate::OperationContext, + operation: &OperationContext, ) -> Result { let runtime_cancellation = runtime.background_cancellation(); let graph = tokio::select! { biased; () = cancellation.cancelled() => return Err(Error::Cancelled), () = runtime_cancellation.cancelled() => return Err(Error::Cancelled), - result = load_split_commit_graph(store, layout, content_hash, max_bytes) => result?, + result = async { + let descriptor = load_split_commit_graph_descriptor(store, layout, content_hash, max_bytes).await?; + // Release-tagged manifest graph v1 needs complete validation. Capsule + // graphs use authenticated pages; corruption cannot select the old reader. + if descriptor.version == 1 { + load_split_commit_graph(store, layout, content_hash, max_bytes).await.map(Graph::Manifest) + } else { + CommitGraphReader::load(store, layout, content_hash, max_bytes).await.map(Graph::Capsule) + } + } => result?, + }; + let mut index = Self { + graph, + roots: Vec::new(), }; - if graph.descriptor.generation != expected_generation - || graph.descriptor.pack_index_hash != expected_pack_index_hash - || graph.descriptor.git_validation_digest != expected_validation_digest + let view = index.view(); + let descriptor = view.descriptor(); + if descriptor.generation != expected_generation + || descriptor.pack_index_hash != expected_pack_index_hash + || descriptor.git_validation_digest != expected_validation_digest { - return Err(Error::Corrupt { - stage: CorruptionStage::CommitGraph, - }); + return Err(corrupt()); } - let roots = crate::repository::resolved_commit_roots(refs, Some(&graph), operation).await?; - if roots - .iter() - .any(|root| sha1_bytes(*root).is_none_or(|root| !graph.contains(&root))) - { - return Err(Error::Corrupt { - stage: CorruptionStage::CommitGraph, - }); + let roots = crate::repository::resolved_commit_roots(refs, Some(view), operation).await?; + let ordinals = index.ordinals(&roots, operation).await?; + if ordinals.iter().any(Option::is_none) { + return Err(corrupt()); } - // One byte per ordinal is bounded by the verified graph byte limit. - // Charge before allocation; reuse this proof only for this pinned ref set. - budget - .charge( - BudgetDimension::InflatedBytes, - u64::from(graph.descriptor.commit_count), - ) + index.roots = roots; + Ok(index) + } + + pub(crate) async fn ordinals( + &self, + oids: &[ObjectId], + operation: &OperationContext, + ) -> Result>> { + operation.ensure_active()?; + operation + .charge(BudgetDimension::Entries, oids.len() as u64) .await?; - let permit = tokio::select! { - biased; - () = cancellation.cancelled() => return Err(Error::Cancelled), - () = runtime_cancellation.cancelled() => return Err(Error::Cancelled), - permit = runtime.decode_permit(cancellation) => permit?, - }; - let cancellation = cancellation.clone(); - runtime - .spawn_blocking(move || { - let _permit = permit; - let pinned_reachable = - pinned_reachability(&graph, &roots, &cancellation, &runtime_cancellation)?; - Ok(Self { - graph, - pinned_reachable, - }) - }) - .await - .map_err(|source| Error::DecodeTask { source })? + let mut positions = Vec::new(); + let mut keys = Vec::new(); + for (position, oid) in oids.iter().enumerate() { + if let ObjectId::Sha1(bytes) = oid { + positions.push(position); + keys.push(*bytes); + } + } + let store = operation.metadata_store(); + let ordinals = operation + .read_metadata(self.view().ordinals(&store, &keys)) + .await?; + operation.ensure_active()?; + let mut output = vec![None; oids.len()]; + for (position, ordinal) in positions.into_iter().zip(ordinals) { + output[position] = ordinal; + } + Ok(output) } - pub(crate) fn is_reachable(&self, oid: ObjectId) -> bool { - sha1_bytes(oid) - .and_then(|oid| self.graph.ordinal(&oid)) - .and_then(|ordinal| self.pinned_reachable.get(ordinal as usize)) - .copied() - .unwrap_or(false) + async fn records( + &self, + ordinals: &[u32], + operation: &OperationContext, + ) -> Result>> { + operation.ensure_active()?; + operation + .charge(BudgetDimension::Entries, ordinals.len() as u64) + .await?; + let store = operation.metadata_store(); + let records = operation + .read_metadata(self.view().records(&store, ordinals)) + .await?; + operation.ensure_active()?; + Ok(records) } - pub(crate) fn parents_match(&self, oid: ObjectId, raw_parents: &[ObjectId]) -> bool { - let Some(ordinal) = sha1_bytes(oid).and_then(|oid| self.graph.ordinal(&oid)) else { - return false; - }; - let Some(record) = self.graph.record(ordinal) else { - return false; - }; - record.parents.len() == raw_parents.len() - && record.parents.iter().zip(raw_parents).all(|(parent, raw)| { - self.graph - .record(*parent) - .is_some_and(|record| ObjectId::Sha1(record.oid) == *raw) + pub(crate) async fn is_reachable( + &self, + oid: ObjectId, + operation: &OperationContext, + ) -> Result { + Ok(self + .reachable_from_roots(&[oid], &self.roots, operation) + .await?[0]) + } + + pub(crate) async fn parents_match( + &self, + commits: &[crate::Commit], + operation: &OperationContext, + ) -> Result { + let oids = commits.iter().map(|commit| commit.oid).collect::>(); + let ordinals = self.ordinals(&oids, operation).await?; + if ordinals.iter().any(Option::is_none) { + return Ok(false); + } + let ordinals = ordinals.into_iter().flatten().collect::>(); + let records = self.records(&ordinals, operation).await?; + let parent_ordinals = records + .iter() + .flatten() + .flat_map(|record| record.parents.iter().copied()) + .collect::>(); + let parents = self.records(&parent_ordinals, operation).await?; + let parents = parent_ordinals + .into_iter() + .zip(parents) + .collect::>(); + Ok(records.iter().zip(commits).all(|(record, commit)| { + record.as_ref().is_some_and(|record| { + record.parents.len() == commit.parents.len() + && record + .parents + .iter() + .zip(&commit.parents) + .all(|(ordinal, raw)| { + parents + .get(ordinal) + .and_then(Option::as_ref) + .is_some_and(|parent| ObjectId::Sha1(parent.oid) == *raw) + }) }) + })) } - pub(crate) fn generation(&self, oid: &ObjectId) -> Option { - let ordinal = self.graph.ordinal(&sha1_bytes(*oid)?)?; - self.graph - .record(ordinal) - .map(|entry| entry.corrected_generation) + pub(crate) async fn generations( + &self, + oids: &[ObjectId], + operation: &OperationContext, + ) -> Result>> { + let ordinals = self.ordinals(oids, operation).await?; + let ids = ordinals.iter().flatten().copied().collect::>(); + let records = self.records(&ids, operation).await?; + let mut records = records.into_iter(); + Ok(ordinals + .into_iter() + .map(|ordinal| { + ordinal.and_then(|_| { + records + .next() + .flatten() + .map(|record| record.corrected_generation) + }) + }) + .collect()) } - pub(crate) fn first_parent_chain( + pub(crate) async fn first_parent_chain( &self, start: ObjectId, maximum: usize, - ) -> Option> { - let mut ordinal = self.graph.ordinal(&sha1_bytes(start)?)?; + operation: &OperationContext, + ) -> Result>> { + let Some(mut ordinal) = self.ordinals(&[start], operation).await?[0] else { + return Ok(None); + }; let mut chain = Vec::new(); for _ in 0..maximum { - let record = self.graph.record(ordinal)?; + let record = self + .records(&[ordinal], operation) + .await? + .remove(0) + .ok_or_else(corrupt)?; chain.push(ObjectId::Sha1(record.oid)); let Some(parent) = record.parents.first().copied() else { break; }; ordinal = parent; } - Some(chain) + Ok(Some(chain)) } - pub(crate) fn reachable_from_roots( + pub(crate) async fn reachable_from_roots( &self, candidates: &[ObjectId], roots: &[ObjectId], - cancellation: &CancellationToken, + operation: &OperationContext, ) -> Result> { - if cancellation.is_cancelled() { - return Err(Error::Cancelled); - } - let mut reachable = vec![false; candidates.len()]; - let mut candidates_by_ordinal = HashMap::>::new(); - for (index, candidate) in candidates.iter().enumerate() { - if let Some(ordinal) = sha1_bytes(*candidate).and_then(|oid| self.graph.ordinal(&oid)) { - candidates_by_ordinal - .entry(ordinal) - .or_default() - .push(index); - } - } - if candidates_by_ordinal.is_empty() { - return Ok(reachable); - } - let mut pending = roots + let ordinals = self.ordinals(candidates, operation).await?; + let roots = self.ordinals(roots, operation).await?; + let mut output = vec![false; candidates.len()]; + let positions = ordinals .iter() - .filter_map(|root| sha1_bytes(*root).and_then(|oid| self.graph.ordinal(&oid))) + .enumerate() + .filter_map(|(position, ordinal)| ordinal.map(|ordinal| (position, ordinal))) .collect::>(); - let mut visited = HashSet::new(); - while let Some(ordinal) = pending.pop() { - if !visited.insert(ordinal) { - continue; - } - if visited.len().is_multiple_of(1024) && cancellation.is_cancelled() { - return Err(Error::Cancelled); - } - if let Some(indices) = candidates_by_ordinal.remove(&ordinal) { - for index in indices { - reachable[index] = true; + let requested = positions + .iter() + .map(|(_, ordinal)| *ordinal) + .collect::>(); + match &self.graph { + Graph::Capsule(graph) => { + for root in roots.into_iter().flatten() { + operation.ensure_active()?; + let store = operation.metadata_store(); + let membership = operation + .read_metadata(graph.ancestry_membership(&store, root, &requested)) + .await?; + for ((position, _), present) in positions.iter().zip(membership) { + output[*position] |= present; + } } - if candidates_by_ordinal.is_empty() { - break; + } + Graph::Manifest(graph) => { + let mut pending = roots.into_iter().flatten().collect::>(); + let mut visited = HashSet::new(); + let mut requested = positions.into_iter().fold( + HashMap::>::new(), + |mut grouped, (position, ordinal)| { + grouped.entry(ordinal).or_default().push(position); + grouped + }, + ); + while !requested.is_empty() + && let Some(ordinal) = pending.pop() + { + operation.ensure_active()?; + if visited.contains(&ordinal) { + continue; + } + // Released graph proofs do not consume raw-history reads. + // Admit metadata work before growing the visited set. + operation.charge(BudgetDimension::Entries, 1).await?; + visited.insert(ordinal); + if let Some(positions) = requested.remove(&ordinal) { + for position in positions { + output[position] = true; + } + } + let record = graph.record(ordinal).ok_or_else(corrupt)?; + pending.extend(record.parents.iter().copied()); } } - let record = self.graph.record(ordinal).ok_or(Error::Corrupt { - stage: CorruptionStage::CommitGraph, - })?; - pending.extend(record.parents.iter().copied()); } - Ok(reachable) + operation.ensure_active()?; + Ok(output) } } -fn pinned_reachability( - graph: &SplitCommitGraph, - roots: &[ObjectId], - cancellation: &CancellationToken, - runtime_cancellation: &CancellationToken, -) -> Result> { - if cancellation.is_cancelled() || runtime_cancellation.is_cancelled() { - return Err(Error::Cancelled); - } - let count = graph.descriptor.commit_count as usize; - let mut reachable = Vec::new(); - reachable - .try_reserve_exact(count) - .map_err(|source| Error::Allocation { - requested: count, - source, - })?; - reachable.resize(count, false); - for root in roots { - let ordinal = sha1_bytes(*root) - .and_then(|oid| graph.ordinal(&oid)) - .ok_or(Error::Corrupt { - stage: CorruptionStage::CommitGraph, - })?; - reachable[ordinal as usize] = true; - } - // The metadata codec proves every parent ordinal precedes its child. - // A descending pass visits each edge once, without a per-query history walk. - for ordinal in (0..graph.descriptor.commit_count).rev() { - if ordinal.is_multiple_of(1024) - && (cancellation.is_cancelled() || runtime_cancellation.is_cancelled()) - { - return Err(Error::Cancelled); - } - if !reachable[ordinal as usize] { - continue; - } - let record = graph.record(ordinal).ok_or(Error::Corrupt { - stage: CorruptionStage::CommitGraph, - })?; - for parent in &record.parents { - reachable[*parent as usize] = true; - } - } - if cancellation.is_cancelled() || runtime_cancellation.is_cancelled() { - return Err(Error::Cancelled); - } - Ok(reachable) -} - -fn sha1_bytes(oid: ObjectId) -> Option<[u8; 20]> { - match oid { - ObjectId::Sha1(bytes) => Some(bytes), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use crab_metadata::split_commit_graph::{ - CommitGraphDescriptor, CommitGraphLayer, CommitGraphLayerRef, CommitGraphRecord, - SplitCommitGraph, commit_graph_layer_path, - }; - - use super::*; - - fn oid(value: u8) -> [u8; 20] { - [value; 20] - } - - fn hash(value: u8) -> String { - format!("{value:02x}").repeat(32) - } - - #[test] - fn pinned_membership_covers_both_merge_parents_but_excludes_retained_history() { - let graph = graph(vec![ - CommitGraphRecord { - oid: oid(1), - tree_oid: oid(101), - commit_time: 10, - corrected_generation: 10, - parents: vec![], - }, - CommitGraphRecord { - oid: oid(2), - tree_oid: oid(102), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - CommitGraphRecord { - oid: oid(3), - tree_oid: oid(103), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - CommitGraphRecord { - oid: oid(4), - tree_oid: oid(104), - commit_time: 30, - corrected_generation: 30, - parents: vec![1, 2], - }, - CommitGraphRecord { - oid: oid(5), - tree_oid: oid(105), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - ]); - let cancellation = CancellationToken::new(); - let runtime_cancellation = CancellationToken::new(); - let pinned_reachable = pinned_reachability( - &graph, - &[ObjectId::Sha1(oid(4))], - &cancellation, - &runtime_cancellation, - ) - .unwrap(); - let index = CommitGraphIndex { - graph, - pinned_reachable, - }; - assert_eq!( - (1..=6) - .map(|value| index.is_reachable(ObjectId::Sha1(oid(value)))) - .collect::>(), - vec![true, true, true, true, false, false] - ); - } - - #[test] - fn pinned_membership_honors_caller_and_runtime_cancellation_even_for_empty_graph() { - for runtime_cancelled in [false, true] { - let cancellation = CancellationToken::new(); - let runtime_cancellation = CancellationToken::new(); - if runtime_cancelled { - runtime_cancellation.cancel(); - } else { - cancellation.cancel(); - } - assert!(matches!( - pinned_reachability(&graph(vec![]), &[], &cancellation, &runtime_cancellation), - Err(Error::Cancelled) - )); - } - } - - fn graph(records: Vec) -> SplitCommitGraph { - let bytes = 24 - + records - .iter() - .map(|record| 60 + record.parents.len() * 4) - .sum::(); - let layer = CommitGraphLayer { - base_ordinal: 0, - records, - }; - let descriptor = CommitGraphDescriptor { - version: 1, - commit_ordinal_digest: None, - generation: 7, - pack_index_hash: hash(1), - git_validation_digest: hash(2), - commit_count: layer.records.len() as u32, - layers: vec![CommitGraphLayerRef { - hash: hash(3), - path: commit_graph_layer_path(&hash(3)), - base_ordinal: 0, - commit_count: layer.records.len() as u32, - bytes: bytes as u64, - pages: None, - }] - .into_iter() - .filter(|layer| layer.commit_count > 0) - .collect(), - }; - SplitCommitGraph::new( - descriptor, - vec![layer] - .into_iter() - .filter(|layer| !layer.records.is_empty()) - .collect(), - ) - .unwrap() - } - - #[test] - fn positional_parents_match_raw_commit_order() { - let index = CommitGraphIndex { - pinned_reachable: Vec::new(), - graph: graph(vec![ - CommitGraphRecord { - oid: oid(1), - tree_oid: oid(101), - commit_time: 10, - corrected_generation: 10, - parents: vec![], - }, - CommitGraphRecord { - oid: oid(2), - tree_oid: oid(102), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - ]), - }; - assert!(index.parents_match(ObjectId::Sha1(oid(2)), &[ObjectId::Sha1(oid(1))])); - assert_eq!(index.generation(&ObjectId::Sha1(oid(2))), Some(20)); - } - - #[test] - fn missing_commit_cannot_validate_raw_parents() { - let index = CommitGraphIndex { - pinned_reachable: Vec::new(), - graph: graph(vec![CommitGraphRecord { - oid: oid(1), - tree_oid: oid(101), - commit_time: 10, - corrected_generation: 10, - parents: vec![], - }]), - }; - assert!(!index.parents_match(ObjectId::Sha1(oid(2)), &[])); - } - - #[test] - fn first_parent_chain_is_bounded_and_positional() { - let index = CommitGraphIndex { - pinned_reachable: Vec::new(), - graph: graph(vec![ - CommitGraphRecord { - oid: oid(1), - tree_oid: oid(101), - commit_time: 10, - corrected_generation: 10, - parents: vec![], - }, - CommitGraphRecord { - oid: oid(2), - tree_oid: oid(102), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - CommitGraphRecord { - oid: oid(3), - tree_oid: oid(103), - commit_time: 30, - corrected_generation: 30, - parents: vec![1, 0], - }, - ]), - }; - - assert_eq!( - index.first_parent_chain(ObjectId::Sha1(oid(3)), 2), - Some(vec![ObjectId::Sha1(oid(3)), ObjectId::Sha1(oid(2))]) - ); - assert_eq!( - index.first_parent_chain(ObjectId::Sha1(oid(2)), 8), - Some(vec![ObjectId::Sha1(oid(2)), ObjectId::Sha1(oid(1))]) - ); - assert_eq!( - index.first_parent_chain(ObjectId::Sha1(oid(3)), 0), - Some(Vec::new()) - ); - assert_eq!(index.first_parent_chain(ObjectId::Sha1(oid(9)), 1), None); - } - - #[test] - fn reachable_from_roots_preserves_duplicates_and_rejects_other_history() { - let index = CommitGraphIndex { - pinned_reachable: Vec::new(), - graph: graph(vec![ - CommitGraphRecord { - oid: oid(1), - tree_oid: oid(101), - commit_time: 10, - corrected_generation: 10, - parents: vec![], - }, - CommitGraphRecord { - oid: oid(2), - tree_oid: oid(102), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - CommitGraphRecord { - oid: oid(3), - tree_oid: oid(103), - commit_time: 30, - corrected_generation: 30, - parents: vec![1], - }, - CommitGraphRecord { - oid: oid(4), - tree_oid: oid(104), - commit_time: 20, - corrected_generation: 20, - parents: vec![0], - }, - ]), - }; - let candidates = [ - ObjectId::Sha1(oid(1)), - ObjectId::Sha1(oid(2)), - ObjectId::Sha1(oid(4)), - ObjectId::Sha1(oid(9)), - ObjectId::Sha1(oid(1)), - ]; - - assert_eq!( - index - .reachable_from_roots( - &candidates, - &[ObjectId::Sha1(oid(3))], - &CancellationToken::new(), - ) - .expect("reachability"), - vec![true, true, false, false, true] - ); - } - - #[test] - fn reachable_from_roots_honors_cancellation_before_work() { - let index = CommitGraphIndex { - pinned_reachable: Vec::new(), - graph: graph(vec![CommitGraphRecord { - oid: oid(1), - tree_oid: oid(101), - commit_time: 10, - corrected_generation: 10, - parents: vec![], - }]), - }; - let cancellation = CancellationToken::new(); - cancellation.cancel(); - - assert!(matches!( - index.reachable_from_roots( - &[ObjectId::Sha1(oid(1))], - &[ObjectId::Sha1(oid(1))], - &cancellation, - ), - Err(Error::Cancelled) - )); +fn corrupt() -> Error { + Error::Corrupt { + stage: CorruptionStage::CommitGraph, } } diff --git a/crates/crab-remote-git/src/operation.rs b/crates/crab-remote-git/src/operation.rs index f18c0cc19..9614bca76 100644 --- a/crates/crab-remote-git/src/operation.rs +++ b/crates/crab-remote-git/src/operation.rs @@ -209,6 +209,30 @@ impl OperationContext { &self.budget } + pub(crate) fn metadata_store(&self) -> crab_storage::Store { + self.state + .store + .clone() + .with_read_admission(self.read_admission()) + } + + pub(crate) async fn read_metadata( + &self, + read: impl std::future::Future>, + ) -> Result { + self.ensure_active()?; + let result = tokio::select! { + biased; + () = self.cancellation.cancelled() => { + self.ensure_active()?; + return Err(Error::Cancelled); + } + result = read => result?, + }; + self.ensure_active()?; + Ok(result) + } + pub(crate) async fn open_pack_transfer( state: Arc, cancellation: &CancellationToken, diff --git a/crates/crab-remote-git/src/path_state.rs b/crates/crab-remote-git/src/path_state.rs index 4debb2d44..865d27b74 100644 --- a/crates/crab-remote-git/src/path_state.rs +++ b/crates/crab-remote-git/src/path_state.rs @@ -36,7 +36,7 @@ impl PathStateIndex { store, layout, content_hash, - commit_graph.graph(), + commit_graph.view(), max_bytes, ) => result.map_err(metadata_error)?, }; @@ -80,7 +80,7 @@ impl PathStateIndex { let summaries = tokio::select! { biased; () = operation.cancellation().cancelled() => return Err(Error::Cancelled), - result = self.index.latest(&store, self.commit_graph.graph(), &commit, &paths) => { + result = self.index.latest(&store, self.commit_graph.view(), &commit, &paths) => { result.map_err(metadata_error)? }, }; diff --git a/crates/crab-remote-git/src/repository.rs b/crates/crab-remote-git/src/repository.rs index 35bbc601b..5f25f28a3 100644 --- a/crates/crab-remote-git/src/repository.rs +++ b/crates/crab-remote-git/src/repository.rs @@ -20,9 +20,9 @@ use crate::operation::{TrackedLocatorSession, finish_with_close}; use crate::reader::{ReaderLimits, RemoteGitReader, SnapshotLookupSources}; use crate::state::RepositoryState; use crate::{ - Error, HeadReference, OperationContext, OperationKind, RemoteGitRuntime, RemoteGitSnapshot, - RepositoryRef, RepositoryRefs, RepositoryStateError, ResolvedRevision, Result, Revision, - RevisionError, + BudgetDimension, CorruptionStage, Error, HeadReference, OperationContext, OperationKind, + RemoteGitRuntime, RemoteGitSnapshot, RepositoryRef, RepositoryRefs, RepositoryStateError, + ResolvedRevision, Result, Revision, RevisionError, }; const MAX_IDENTITY_COMPONENT_BYTES: usize = 1_024; @@ -1192,22 +1192,29 @@ impl RemoteGitRepository { /// /// Returns `None` when the pinned repository has no complete commit graph. /// This metadata-only proof never opens the object-locator database. - pub fn commits_reachable_from( + pub async fn commits_reachable_from( &self, candidates: &[ObjectId], roots: &[ObjectId], cancellation: &CancellationToken, ) -> Result>> { - let _task_token = self.state.runtime.operation_token(); - let runtime_cancellation = self.state.runtime.background_cancellation(); check_cancelled(cancellation)?; - check_cancelled(&runtime_cancellation)?; + check_cancelled(&self.state.runtime.background_cancellation())?; let Some(graph) = self.state.commit_graph.as_ref() else { return Ok(None); }; - let reachable = graph.reachable_from_roots(candidates, roots, cancellation)?; - check_cancelled(&runtime_cancellation)?; - Ok(Some(reachable)) + // Membership needs admitted metadata I/O, but no catalog database. + let operation = OperationContext::open_pack_transfer( + Arc::clone(&self.state), + cancellation, + self.state.options.operation, + ) + .await?; + let result = graph + .reachable_from_roots(candidates, roots, &operation) + .await + .map(Some); + operation.finish(result).await } /// Attach a durable graph validated against this handle's exact pinned state. @@ -1277,7 +1284,7 @@ impl RemoteGitRepository { let reachable = match &self.state.commit_graph { Some(graph) => { check_cancelled(operation.cancellation())?; - graph.is_reachable(*commit) + graph.is_reachable(*commit, operation).await? } None => prove_reachable(*commit, &self.state.refs, operation).await?, }; @@ -1329,18 +1336,23 @@ impl RemoteGitRepository { .and_then(parse_merkle_hash)?, None => return Err(Error::EmptyRepository), }; + let commit_graph = match &self.state.commit_graph { + Some(index) + if index + .parents_match(std::slice::from_ref(&commit), operation) + .await? => + { + Some(Arc::clone(index)) + } + _ => None, + }; Ok(RemoteGitSnapshot { generation: self.state.generation, pack_index_hash, commit_oid: commit.oid, root_tree_oid: commit.tree, repository: Arc::clone(&self.state), - commit_graph: self - .state - .commit_graph - .as_ref() - .filter(|index| index.parents_match(commit.oid, &commit.parents)) - .cloned(), + commit_graph, }) } } @@ -1387,7 +1399,6 @@ fn load_commit_graph<'a>( state.options.object.max_commit_graph_bytes, operation.cancellation(), runtime, - operation.budget(), &operation, ) .await; @@ -1753,38 +1764,54 @@ impl TagPeelState { pub(crate) async fn resolved_commit_roots( refs: &RepositoryRefs, - graph: Option<&crab_metadata::split_commit_graph::SplitCommitGraph>, + graph: Option>, operation: &OperationContext, ) -> Result> { let mut roots = Vec::new(); - for reference in &refs.entries { - if let Some(commit) = resolved_commit_root(reference, graph, operation).await? { + let known = if let Some(graph) = graph { + let targets = refs + .entries + .iter() + .map(|reference| { + let ObjectId::Sha1(oid) = reference.peeled.unwrap_or(reference.target) else { + return Err(Error::UnsupportedObjectFormat); + }; + Ok(oid) + }) + .collect::>>()?; + operation + .charge(BudgetDimension::Entries, targets.len() as u64) + .await?; + operation + .read_metadata(graph.ordinals(&operation.metadata_store(), &targets)) + .await? + } else { + vec![None; refs.entries.len()] + }; + for (reference, ordinal) in refs.entries.iter().zip(known) { + if ordinal.is_some() { + roots.push(reference.peeled.unwrap_or(reference.target)); + continue; + } + if graph.is_some() && !reference.name.starts_with("refs/tags/") { + return Err(Error::Corrupt { + stage: CorruptionStage::CommitGraph, + }); + } + if let Some(commit) = resolved_commit_root(reference, operation).await? { roots.push(commit); } } + roots.sort_unstable(); + roots.dedup(); Ok(roots) } async fn resolved_commit_root( reference: &crate::RepositoryRef, - graph: Option<&crab_metadata::split_commit_graph::SplitCommitGraph>, operation: &OperationContext, ) -> Result> { check_cancelled(operation.cancellation())?; - let target = reference.peeled.unwrap_or(reference.target); - if let Some(graph) = graph { - // A validated snapshot's known commit roots need no pack reads at - // open. Only unresolved tags need raw peeling; object requests still - // verify OIDs/kinds and parent lists before using graph acceleration. - if <[u8; 20]>::try_from(target.as_bytes()).is_ok_and(|oid| graph.contains(&oid)) { - return Ok(Some(target)); - } - if !reference.name.starts_with("refs/tags/") { - return Err(Error::Corrupt { - stage: crate::CorruptionStage::CommitGraph, - }); - } - } match peel_to_commit(reference.target, operation).await { Ok((commit, _)) => { if reference.peeled.is_some_and(|expected| expected != commit) { @@ -1825,7 +1852,7 @@ async fn prove_reachable( .filter(|reference| !is_target(reference)), ); for reference in references { - if let Some(commit) = resolved_commit_root(reference, None, operation).await? { + if let Some(commit) = resolved_commit_root(reference, operation).await? { if commit == target { operation .charge(crate::BudgetDimension::HistoryCommits, 1) @@ -1918,6 +1945,7 @@ mod tests { manifest_release: tokio::sync::Notify, block_fragment: std::sync::Mutex>, reject_fragment: std::sync::Mutex>, + reject_graph_layer_bodies: AtomicBool, request_started: Arc, request_release: tokio::sync::Notify, } @@ -1935,6 +1963,7 @@ mod tests { manifest_release: tokio::sync::Notify::new(), block_fragment: std::sync::Mutex::new(None), reject_fragment: std::sync::Mutex::new(None), + reject_graph_layer_bodies: AtomicBool::new(false), request_started: Arc::new(tokio::sync::Notify::new()), request_release: tokio::sync::Notify::new(), } @@ -1975,6 +2004,22 @@ mod tests { location: &ObjectPath, options: GetOptions, ) -> object_store::Result { + if self.reject_graph_layer_bodies.load(Ordering::SeqCst) + && location.as_ref().contains("metadata/commit-graph/layers/") + && !options.head + && options.range.is_none() + { + return Err(object_store::Error::Generic { + store: "test admission", + source: Box::new(crab_storage::StorageError::ReadRejected { + source: Box::new(Error::LimitExceeded { + limit: "whole graph layer reads", + actual: 1, + maximum: 0, + }), + }), + }); + } if self .reject_fragment .lock() diff --git a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs index 7b713ccfe..01d18faa3 100644 --- a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs +++ b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs @@ -2,13 +2,14 @@ use super::*; async fn snapshot_with_read_indexes( fixture: &OpenFixture, + paged: bool, ) -> crab_metadata::manifest_store::RepositorySnapshot { use crab_metadata::path_state::{ PathStateInput, PathStateMutation, append_path_state, upload_path_state, }; use crab_metadata::split_commit_graph::{ - CommitGraphInput, append_split_commit_graph, load_split_commit_graph, - upload_split_commit_graph, + CommitGraphInput, append_paged_commit_graph, append_split_commit_graph, + load_split_commit_graph, upload_split_commit_graph, }; crab_metadata::layout_descriptor::ensure_canonical_layout(&fixture.store, &fixture.layout) @@ -19,7 +20,12 @@ async fn snapshot_with_read_indexes( .await .unwrap(); let manifest = &mut snapshot.manifest; - let graph = append_split_commit_graph( + let append = if paged { + append_paged_commit_graph + } else { + append_split_commit_graph + }; + let graph = append( None, manifest.generation, manifest.pack_index_hash.clone(), @@ -78,7 +84,13 @@ async fn snapshot_with_read_indexes( #[tokio::test] async fn snapshot_read_indexes_supply_exact_attribution_without_manifest_or_catalog() { let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; + // Full maintenance reads are valid during construction. Foreground opening + // and attribution must succeed when only graph-page ranges are admitted. + fixture + .backend + .reject_graph_layer_bodies + .store(true, Ordering::SeqCst); fixture .store .delete(&fixture.layout.manifest_path()) @@ -156,7 +168,7 @@ async fn snapshot_read_indexes_supply_exact_attribution_without_manifest_or_cata #[tokio::test] async fn snapshot_read_indexes_skip_origin_when_absent_or_superseded() { let fixture = open_fixture(1, None).await; - let indexed = snapshot_with_read_indexes(&fixture).await; + let indexed = snapshot_with_read_indexes(&fixture, true).await; let runtime = Arc::new(RemoteGitRuntime::default()); for absent in [false, true] { let mut snapshot = indexed.clone(); @@ -197,7 +209,7 @@ async fn snapshot_read_indexes_skip_origin_when_absent_or_superseded() { #[tokio::test] async fn snapshot_read_indexes_do_not_hide_exhausted_request_or_byte_budgets() { let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; for limits in [ OperationLimits { max_storage_requests: 1, @@ -227,7 +239,7 @@ async fn snapshot_read_indexes_do_not_hide_exhausted_request_or_byte_budgets() { async fn snapshot_read_indexes_fail_attribution_when_declared_metadata_is_corrupt() { for kind in ["commit-graph", "path-state"] { let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; let hash = match kind { "commit-graph" => snapshot.manifest.commit_graph_hash.as_deref().unwrap(), _ => snapshot.manifest.path_state_hash.as_deref().unwrap(), @@ -273,7 +285,7 @@ async fn snapshot_read_indexes_fail_attribution_when_declared_metadata_is_corrup async fn snapshot_read_indexes_drain_on_cancellation_shutdown_and_deadline() { for mode in ["caller", "shutdown", "deadline"] { let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; fixture .backend .block_path_containing("manifests/commit-graph-"); @@ -333,7 +345,7 @@ async fn snapshot_read_indexes_drain_on_cancellation_shutdown_and_deadline() { async fn snapshot_read_indexes_admit_descriptor_pages_and_warm_semantic_results() { for mode in ["descriptor", "page", "warm"] { let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; let runtime = Arc::new(RemoteGitRuntime::default()); let cancel = CancellationToken::new(); let repository = RemoteGitRepository::from_snapshot( @@ -420,9 +432,12 @@ async fn snapshot_read_indexes_admit_descriptor_pages_and_warm_semantic_results( #[tokio::test(flavor = "multi_thread")] async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { - for mode in ["caller", "shutdown", "deadline"] { + for (mode, dependency) in ["caller", "shutdown", "deadline"] + .into_iter() + .flat_map(|mode| ["path-state", "commit-graph"].map(|dependency| (mode, dependency))) + { let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; let runtime = Arc::new(RemoteGitRuntime::default()); let cancel = CancellationToken::new(); let repository = RemoteGitRepository::from_snapshot( @@ -437,7 +452,11 @@ async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { .unwrap(); fixture .backend - .block_path_containing("metadata/path-state/layers/"); + .block_path_containing(if dependency == "path-state" { + "metadata/path-state/layers/" + } else { + "metadata/commit-graph/layers/" + }); let worker_cancel = cancel.clone(); let worker = tokio::spawn(async move { let limits = OperationLimits { @@ -457,6 +476,16 @@ async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { .await .unwrap(); let result = async { + if dependency == "commit-graph" { + return repository + .state + .commit_graph + .as_ref() + .unwrap() + .is_reachable(ObjectId::Sha1([0x11; 20]), &operation) + .await + .map(|_| ()); + } let index = repository.state.path_state(&operation).await?.unwrap(); index .latest( @@ -466,6 +495,7 @@ async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { &operation, ) .await + .map(|_| ()) } .await; operation.finish(result).await @@ -493,8 +523,143 @@ async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { "deadline" => matches!(result, Err(Error::Timeout { .. })), _ => matches!(result, Err(Error::Cancelled)), }, - "{mode}: {result:?}" + "{mode}/{dependency}: {result:?}" ); runtime.shutdown().await; } } + +#[tokio::test] +async fn released_manifest_graph_remains_usable_for_metadata_membership() { + let fixture = open_fixture(1, None).await; + let snapshot = snapshot_with_read_indexes(&fixture, false).await; + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let repository = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + assert_eq!( + repository + .commits_reachable_from( + &[ + ObjectId::Sha1([0x11; 20]), + ObjectId::Sha1([0x99; 20]), + ObjectId::Sha1([0x11; 20]) + ], + &[ObjectId::Sha1([0x11; 20])], + &cancel, + ) + .await + .unwrap(), + Some(vec![true, false, true]) + ); + runtime.shutdown().await; +} + +#[tokio::test] +async fn paged_graph_queries_preserve_merge_parents_and_exclude_retained_history() { + use crab_metadata::split_commit_graph::{ + CommitGraphInput, append_paged_commit_graph, upload_split_commit_graph, + }; + let fixture = open_fixture(1, None).await; + let mut snapshot = snapshot_with_read_indexes(&fixture, true).await; + let input = |value: u8, parents: &[u8]| CommitGraphInput { + oid: [value; 20], + tree_oid: [0x22; 20], + commit_time: i64::from(value), + parents: parents.iter().map(|parent| [*parent; 20]).collect(), + }; + let graph = append_paged_commit_graph( + None, + snapshot.manifest.generation, + snapshot.manifest.pack_index_hash.clone(), + snapshot.manifest.git_validation_digest.clone(), + &[[0x11; 20], [4; 20]], + vec![ + input(1, &[]), + input(2, &[1]), + input(3, &[1]), + input(0x11, &[2, 3]), + input(4, &[1]), + ], + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(&fixture.store, &fixture.layout, &graph) + .await + .unwrap(); + snapshot.manifest.commit_graph_hash = Some(graph.descriptor_hash); + snapshot.manifest.path_state_hash = None; + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let repository = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + let operation = repository + .operation(crate::OperationKind::Resolve, &cancel) + .await + .unwrap(); + let result = async { + let graph = repository.state.commit_graph.as_ref().unwrap(); + assert!(matches!( + graph.view(), + crab_metadata::split_commit_graph::CommitGraphView::Paged(_) + )); + let candidates = [1, 2, 3, 4, 0x11, 99, 2].map(|value| ObjectId::Sha1([value; 20])); + assert_eq!( + graph + .reachable_from_roots(&candidates, &[ObjectId::Sha1([0x11; 20])], &operation) + .await?, + vec![true, true, true, false, true, false, true] + ); + assert!( + !graph + .is_reachable(ObjectId::Sha1([4; 20]), &operation) + .await? + ); + assert!(matches!( + repository + .resolve(&Revision::Commit(ObjectId::Sha1([4; 20])), &operation) + .await, + Err(Error::Revision { + reason: RevisionError::NotReachable + }) + )); + assert_eq!( + graph + .reachable_from_roots(&candidates, &[ObjectId::Sha1([1; 20])], &operation) + .await?, + vec![true, false, false, false, false, false, false] + ); + assert_eq!( + graph + .first_parent_chain(ObjectId::Sha1([0x11; 20]), 2, &operation) + .await?, + Some(vec![ObjectId::Sha1([0x11; 20]), ObjectId::Sha1([2; 20])]) + ); + assert!( + graph + .first_parent_chain(ObjectId::Sha1([99; 20]), 2, &operation) + .await? + .is_none() + ); + Ok(()) + } + .await; + operation.finish(result).await.unwrap(); + runtime.shutdown().await; +} diff --git a/crates/crab-remote-git/src/snapshot.rs b/crates/crab-remote-git/src/snapshot.rs index 8e656f1bb..9ca8c17f8 100644 --- a/crates/crab-remote-git/src/snapshot.rs +++ b/crates/crab-remote-git/src/snapshot.rs @@ -111,7 +111,13 @@ impl RemoteGitSnapshot { } } HistoryTraversal::AllParents => { - queue_all_parents(&mut pending, &commit, self.commit_graph.as_deref())?; + queue_all_parents( + &mut pending, + &commit, + self.commit_graph.as_deref(), + operation, + ) + .await?; } } if traversed >= skip { @@ -206,7 +212,13 @@ impl RemoteGitSnapshot { visited.insert(oid); operation.charge(BudgetDimension::HistoryCommits, 1).await?; let commit = operation.read_commit(oid).await?; - queue_all_parents(&mut pending, &commit, self.commit_graph.as_deref())?; + queue_all_parents( + &mut pending, + &commit, + self.commit_graph.as_deref(), + operation, + ) + .await?; if traversed >= skip { operation .charge( @@ -283,7 +295,10 @@ impl RemoteGitSnapshot { let mut carry: Option<(Commit, Option)> = None; let mut next_oid = start; loop { - let Some(oids) = commit_graph.first_parent_chain(next_oid, PREFETCH) else { + let Some(oids) = commit_graph + .first_parent_chain(next_oid, PREFETCH, operation) + .await? + else { let fallback = carry.as_ref().map_or(next_oid, |(commit, _)| commit.oid); return self .raw_first_parent_path_history(fallback, path, page, operation, items) @@ -297,10 +312,7 @@ impl RemoteGitSnapshot { let commits = path_history_commits(&oids, operation).await?; // The graph only supplies the batch shape. Raw commit objects must // confirm every positional parent before the batch can affect history. - if commits - .iter() - .any(|commit| !commit_graph.parents_match(commit.oid, &commit.parents)) - { + if !commit_graph.parents_match(&commits, operation).await? { let fallback = carry.as_ref().map_or(next_oid, |(commit, _)| commit.oid); return self .raw_first_parent_path_history(fallback, path, page, operation, items) @@ -489,7 +501,13 @@ impl RemoteGitSnapshot { } } HistoryTraversal::AllParents => { - queue_all_parents(&mut pending, &commit, self.commit_graph.as_deref())?; + queue_all_parents( + &mut pending, + &commit, + self.commit_graph.as_deref(), + operation, + ) + .await?; } } let mut filled_page = false; @@ -1770,13 +1788,13 @@ async fn blame_parent_commits( const RAW_PREFETCH: usize = 16; if let Some(commit_graph) = commit_graph - && let Some(oids) = commit_graph.first_parent_chain(start, GRAPH_PREFETCH) + && let Some(oids) = commit_graph + .first_parent_chain(start, GRAPH_PREFETCH, operation) + .await? && !oids.is_empty() { let commits = path_history_commits(&oids, operation).await?; - let valid = commits - .iter() - .all(|commit| commit_graph.parents_match(commit.oid, &commit.parents)) + let valid = commit_graph.parents_match(&commits, operation).await? && commits .windows(2) .all(|pair| pair[0].parents.first() == Some(&pair[1].oid)); @@ -1949,15 +1967,16 @@ async fn commit_ancestors( } operation.charge(BudgetDimension::HistoryCommits, 1).await?; let commit = operation.read_commit(oid).await?; - queue_all_parents(&mut pending, &commit, commit_graph)?; + queue_all_parents(&mut pending, &commit, commit_graph, operation).await?; } Ok(ancestors) } -fn queue_all_parents( +async fn queue_all_parents( pending: &mut Vec, commit: &Commit, commit_graph: Option<&CommitGraphIndex>, + operation: &OperationContext, ) -> Result<()> { pending .try_reserve(commit.parents.len()) @@ -1968,12 +1987,17 @@ fn queue_all_parents( .saturating_mul(mem::size_of::()), source, })?; - let Some(commit_graph) = - commit_graph.filter(|index| index.parents_match(commit.oid, &commit.parents)) - else { + let Some(commit_graph) = commit_graph else { pending.extend(commit.parents.iter().rev().copied()); return Ok(()); }; + if !commit_graph + .parents_match(std::slice::from_ref(commit), operation) + .await? + { + pending.extend(commit.parents.iter().rev().copied()); + return Ok(()); + } let mut ordered = Vec::new(); ordered .try_reserve_exact(commit.parents.len()) @@ -1984,14 +2008,14 @@ fn queue_all_parents( .saturating_mul(mem::size_of::()), source, })?; - ordered.extend(commit.parents.iter().copied()); - ordered.sort_unstable_by(|left, right| { - commit_graph - .generation(right) - .cmp(&commit_graph.generation(left)) + let generations = commit_graph.generations(&commit.parents, operation).await?; + ordered.extend(commit.parents.iter().copied().zip(generations)); + ordered.sort_unstable_by(|(left, left_generation), (right, right_generation)| { + right_generation + .cmp(left_generation) .then_with(|| left.cmp(right)) }); - pending.extend(ordered.into_iter().rev()); + pending.extend(ordered.into_iter().rev().map(|(oid, _)| oid)); Ok(()) } diff --git a/crates/crab-remote-git/tests/remote_repository.rs b/crates/crab-remote-git/tests/remote_repository.rs index 20626e393..6054833da 100644 --- a/crates/crab-remote-git/tests/remote_repository.rs +++ b/crates/crab-remote-git/tests/remote_repository.rs @@ -4926,77 +4926,79 @@ async fn rejects_wrong_kind_and_unreachable_retained_commit() { #[tokio::test(flavor = "multi_thread")] async fn attached_graph_preserves_tag_visibility_and_rejects_retained_commits() { - let options = RepositoryOptions::new( - ObjectLimits::default(), - OperationLimits { - max_history_commits: 1, - ..OperationLimits::default() - }, - ) - .unwrap(); - let fixture = publish(DeltaKind::Ref, false, options).await; - let hash = publish_complete_graph(&fixture).await; - let cancel = CancellationToken::new(); - let attached = fixture - .repository - .with_commit_graph(&hash, &cancel) - .await - .unwrap(); - assert_eq!( - ( - fixture.repository.commit_graph_available(), - attached.commit_graph_available() - ), - (false, true) - ); - let operation = attached - .operation(OperationKind::Resolve, &cancel) - .await + for paged in [false, true] { + let options = RepositoryOptions::new( + ObjectLimits::default(), + OperationLimits { + max_history_commits: 1, + ..OperationLimits::default() + }, + ) .unwrap(); - let result = async { - for oid in [fixture.root_commit, fixture.side_commit] { - assert_eq!( + let fixture = publish(DeltaKind::Ref, false, options).await; + let hash = publish_complete_graph(&fixture, paged).await; + let cancel = CancellationToken::new(); + let attached = fixture + .repository + .with_commit_graph(&hash, &cancel) + .await + .unwrap(); + assert_eq!( + ( + fixture.repository.commit_graph_available(), + attached.commit_graph_available() + ), + (false, true) + ); + let operation = attached + .operation(OperationKind::Resolve, &cancel) + .await + .unwrap(); + let result = async { + for oid in [fixture.root_commit, fixture.side_commit] { + assert_eq!( + attached + .resolve(&Revision::Commit(oid), &operation) + .await? + .commit, + oid + ); + } + assert!(matches!( + attached + .resolve(&Revision::Commit(fixture.unreachable), &operation) + .await, + Err(Error::Revision { + reason: crab_remote_git::RevisionError::NotReachable + }) + )); + assert!( attached - .resolve(&Revision::Commit(oid), &operation) - .await? - .commit, - oid + .resolve(&Revision::Reference("v2".to_owned()), &operation) + .await + .is_ok() ); + assert!(matches!( + attached + .resolve(&Revision::Reference("blob".to_owned()), &operation) + .await, + Err(Error::Revision { + reason: crab_remote_git::RevisionError::NotCommit + }) + )); + Ok(()) } - assert!(matches!( - attached - .resolve(&Revision::Commit(fixture.unreachable), &operation) - .await, - Err(Error::Revision { - reason: crab_remote_git::RevisionError::NotReachable - }) - )); - assert!( - attached - .resolve(&Revision::Reference("v2".to_owned()), &operation) - .await - .is_ok() - ); - assert!(matches!( - attached - .resolve(&Revision::Reference("blob".to_owned()), &operation) - .await, - Err(Error::Revision { - reason: crab_remote_git::RevisionError::NotCommit - }) - )); - Ok(()) + .await; + let result = operation.finish(result).await; + fixture.runtime.shutdown().await; + result.unwrap(); } - .await; - let result = operation.finish(result).await; - fixture.runtime.shutdown().await; - result.unwrap(); } #[tokio::test(flavor = "multi_thread")] async fn declared_graph_uses_verified_commit_roots_including_nested_tags() { let fixture = publish(DeltaKind::Ref, false, RepositoryOptions::default()).await; - let hash = publish_complete_graph(&fixture).await; + let hash = publish_complete_graph(&fixture, false).await; let (mut manifest, etag) = read_manifest(&fixture.store, &fixture.layout) .await .unwrap(); @@ -5024,7 +5026,7 @@ async fn declared_graph_uses_verified_commit_roots_including_nested_tags() { #[tokio::test(flavor = "multi_thread")] async fn graph_attachment_rejects_other_ref_state_and_cancelled_work() { let fixture = publish(DeltaKind::Ref, false, RepositoryOptions::default()).await; - let hash = publish_complete_graph(&fixture).await; + let hash = publish_complete_graph(&fixture, false).await; let cancel = CancellationToken::new(); cancel.cancel(); assert!(matches!( @@ -5062,9 +5064,10 @@ async fn graph_attachment_rejects_other_ref_state_and_cancelled_work() { )); } -async fn publish_complete_graph(fixture: &PublishedFixture) -> String { +async fn publish_complete_graph(fixture: &PublishedFixture, paged: bool) -> String { use crab_metadata::split_commit_graph::{ - CommitGraphInput, append_split_commit_graph, upload_split_commit_graph, + CommitGraphInput, append_paged_commit_graph, append_split_commit_graph, + upload_split_commit_graph, }; let (manifest, _) = read_manifest(&fixture.store, &fixture.layout) .await @@ -5115,7 +5118,12 @@ async fn publish_complete_graph(fixture: &PublishedFixture) -> String { }) .collect::>(); let roots = inputs.iter().map(|input| input.oid).collect::>(); - let graph = append_split_commit_graph( + let append = if paged { + append_paged_commit_graph + } else { + append_split_commit_graph + }; + let graph = append( None, manifest.generation, manifest.pack_index_hash, diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 830b0a0a8..aaf158a66 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -329,7 +329,14 @@ async fn query( let result = async { let snapshot = repository.snapshot(revision, &operation).await?; match name { - "resolve" => Ok(json!({"commit":snapshot.commit_oid().to_string()})), + "resolve" => { + // Ref resolution alone bypasses ancestor membership. Resolve the + // same immutable OID through the foreground visibility boundary. + let resolved = repository + .resolve(&Revision::Commit(snapshot.commit_oid()), &operation) + .await?; + Ok(json!({"commit":resolved.commit.to_string()})) + } "commit" => { let commit = snapshot.commit(&operation).await?; Ok(json!({"oid":commit.oid.to_string(), "tree":commit.tree.to_string(), From c9850b67d4ad24db566daae9a1b354fb5e9ea669 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 18:21:05 -0700 Subject: [PATCH 011/127] perf(read): keep packs cold during metadata capture Use complete metadata capture for origin-backed readers before the first checkpoint. Preserve visibility ordering and range-install thin delta chains instead of requiring resident pack bodies. --- .../docs/architecture/indexed-remote-reads.md | 18 ++ crab/src/cmd/metadb.rs | 2 +- crab/src/read/mod.rs | 2 +- crates/crab-http-server/src/projection.rs | 2 +- crates/crab-http-server/src/server.rs | 2 +- crates/crab-read/README.md | 13 +- crates/crab-read/src/capsule_protocol.rs | 122 +++++++++-- crates/crab-remote/README.md | 4 + crates/crab-remote/examples/support/mod.rs | 2 +- crates/crab-remote/src/browse_indexes.rs | 4 +- crates/crab-remote/src/checkpoint.rs | 7 +- .../tests/checkpoint/external_delta.rs | 192 ++++++++++-------- crates/crab-s3-gateway/src/repository.rs | 2 +- 13 files changed, 254 insertions(+), 118 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index dac082889..a618ceae5 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -478,3 +478,21 @@ bytes and 62,357,504 median RSS bytes. Synthetic OIDs isolate the builder; these are debug component receipts, not native Git integrity, release-image performance or whole-publication measurements. Release/provider and broader lifecycle gates remain open. Fixtures, source/binary hashes and raw receipts remain external. + +## Metadata-first opening: implementation in progress + +The existing complete-metadata checkpoint capture is now named +`open_view_from_root_with_metadata`. HTTP browsing, S3 read views, CLI remote +reads, index builders and the foreground qualifier use it before and after +checkpoints. It verifies full visibility, pointer catalogs and transaction/ref +ordering, rather than substituting the narrower ordinary-fetch footer view. +Complete body views remain available for embedded-pack and candidate installers. +Origin-backed installation now handles uncheckpointed metadata source ranges; +resident complete views retain their existing body installation path. + +The initial two focused tests pass snapshot/catalog equivalence, complete +activation/visibility equivalence and absence of whole-run body GETs. Expanded +native delta-chain installation, missing-dependency error preservation and +consumer/real-store qualification remain in progress. No bounded-opening claim +follows from this payload change: historical control directories, inventories +and ref/frontier capture still require indexing and scaling proof. diff --git a/crab/src/cmd/metadb.rs b/crab/src/cmd/metadb.rs index b5071db4e..2efb6f16d 100644 --- a/crab/src/cmd/metadb.rs +++ b/crab/src/cmd/metadb.rs @@ -634,7 +634,7 @@ async fn capsule_generation_owner_sample( started, )); } - let view = crab_read::capsule_protocol::open_view_from_root_for_checkpoint( + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crab/src/read/mod.rs b/crab/src/read/mod.rs index 70a124706..9126f54bc 100644 --- a/crab/src/read/mod.rs +++ b/crab/src/read/mod.rs @@ -569,7 +569,7 @@ impl Inner { ); let root = crab_metadata::capsule_protocol::load_root(&layout).await?; Ok( - crab_read::capsule_protocol::open_view_from_root_with_control( + crab_read::capsule_protocol::open_view_from_root_with_metadata( &layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-http-server/src/projection.rs b/crates/crab-http-server/src/projection.rs index a62cf9b9c..66b12371b 100644 --- a/crates/crab-http-server/src/projection.rs +++ b/crates/crab-http-server/src/projection.rs @@ -790,7 +790,7 @@ pub(crate) async fn open_view( .map_err(metadata_error)?; // Origin-backed readers need complete visibility controls, while Git pack // bodies remain in their authenticated sources for bounded range reads. - let view = crab_read::capsule_protocol::open_view_from_root_for_checkpoint( + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-http-server/src/server.rs b/crates/crab-http-server/src/server.rs index f384544f9..cf09de03c 100644 --- a/crates/crab-http-server/src/server.rs +++ b/crates/crab-http-server/src/server.rs @@ -792,7 +792,7 @@ impl Repository { .map_err(|source| crate::Error::Settings { source: Box::new(source), })?; - let view = crab_read::capsule_protocol::open_view_from_root_with_control( + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( &self.layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index a6ab59615..60eeed93e 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -220,9 +220,18 @@ and hydration consume this same view. Checkpoints use the layered source directory exclusively. Ordinary fetch uses `open_view_from_root_with_layered_control` to keep catalog and visibility bodies -cold; `open_view_from_root_with_control` loads those bodies for consumers that -need full authorization or pointer catalogs. These entry points differ in read +cold; `open_view_from_root_with_control` loads those bodies before the first +checkpoint for consumers that need embedded pack bytes. These entry points differ in read requirements, not storage-format compatibility. +`open_view_from_root_with_metadata` captures the complete visibility and pointer +catalog controls while leaving Git pack bodies cold, including before the first +checkpoint. Browsing and background index builders use this path with the +origin-backed Git reader. Metadata capture still verifies transaction ordering, +activations and exact resulting refs; it is not footer-only fetch admission. +Pack installation from metadata views also requires the origin layout and consumes +authenticated source ranges. The embedded-pack reader and candidate installers +retain complete body views. Control directories and ref capture remain separate +indexing work; this change does not make their historical decoding constant-cost. For an ordinary fetch whose admitted capsule frontier totals at most 128 MiB, the reader verifies each complete run once and reuses its resident pack and index bytes. Larger frontiers retain bounded suffix and range reads; stable diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 64ec3a4dc..94663cf95 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -2001,7 +2001,7 @@ fn order_native_pack_payloads(payloads: Vec) -> Result, snapshot: crab_metadata::capsule_protocol::RootSnapshot, limits: CapsuleReadLimits, @@ -5557,12 +5564,13 @@ async fn load_run_control( router: &StoreLayout, pointer: &CapsulePointer, ) -> Result<(CapsuleRunControl, Vec)> { - let path = router.capsule_path(pointer.hash()); - let (control, capsules) = - crab_metadata::capsule_protocol::load_capsule_run_control(router, pointer) - .await - .map_err(|error| corrupt_path(path.to_string(), error.to_string()))?; - Ok((control, capsules)) + match crab_metadata::capsule_protocol::load_capsule_run_control(router, pointer).await { + Ok(value) => Ok(value), + Err(crab_metadata::error::MetadataError::CorruptObject { path, reason }) => { + Err(ReadError::CorruptObject { path, reason }) + } + Err(error) => Err(error.into()), + } } fn corrupt(path: &object_store::path::Path, reason: impl Into) -> ReadError { @@ -6230,6 +6238,92 @@ mod tests { ); } + #[tokio::test] + async fn metadata_run_control_preserves_missing_dependency_error() { + let inner = Arc::new(InMemory::new()); + seed_one_capsule(inner.clone(), None).await; + let store = Store::new(inner); + let router = StoreLayout::new(store.clone(), "repositories/test".to_owned()); + let root = load_root(&router).await.unwrap(); + let pointer = &root.record().root().capsule_frontier()[0]; + store + .delete(&router.capsule_path(pointer.hash())) + .await + .unwrap(); + assert!(matches!( + load_run_control(&router, pointer).await, + Err(ReadError::Metadata( + crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::NotFound { .. } + } + )) + )); + } + + #[tokio::test] + async fn metadata_view_preserves_complete_state_without_frontier_body_reads() { + let inner = Arc::new(InMemory::new()); + seed_one_capsule(inner.clone(), None).await; + let observer = Arc::new(RecordingObserver::default()); + let store = Store::new(inner).with_storage_observer(observer.clone()); + let router = StoreLayout::new(store, "repositories/test".to_owned()); + let root = load_root(&router).await.unwrap(); + let complete = open_view_from_root_with_control(&router, root.clone(), TEST_LIMITS) + .await + .unwrap(); + observer.observations.lock().unwrap().clear(); + + let metadata = open_view_from_root_with_metadata(&router, root, TEST_LIMITS) + .await + .unwrap(); + + assert_eq!( + metadata.git_snapshot().unwrap(), + complete.git_snapshot().unwrap() + ); + assert_eq!( + metadata.pointer_catalog().unwrap(), + complete.pointer_catalog().unwrap() + ); + assert!(metadata.capsules().is_empty()); + let observations = observer.observations.lock().unwrap(); + assert!( + !observations + .iter() + .any(|read| read.operation == StorageOperation::Get) + ); + assert!( + observations + .iter() + .any(|read| read.operation == StorageOperation::Range) + ); + } + + #[tokio::test] + async fn metadata_view_preserves_multi_ref_activation_and_visibility() { + for committed in [false, true] { + let inner = Arc::new(InMemory::new()); + seed_prepared_multi_ref(inner.clone(), committed, committed).await; + let store = Store::new(inner); + let router = StoreLayout::new(store, "repositories/test".to_owned()); + let root = load_root(&router).await.unwrap(); + let complete = open_view_from_root_with_control(&router, root.clone(), TEST_LIMITS) + .await + .unwrap(); + let metadata = open_view_from_root_with_metadata(&router, root, TEST_LIMITS) + .await + .unwrap(); + assert_eq!( + metadata.git_snapshot().unwrap(), + complete.git_snapshot().unwrap() + ); + assert_eq!( + metadata.git_visibility_index().unwrap(), + complete.git_visibility_index().unwrap() + ); + } + } + #[tokio::test] async fn bounded_layered_control_reads_the_complete_frontier_once() { let inner = Arc::new(InMemory::new()); diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 0d85ea124..e415713bd 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -78,6 +78,10 @@ this orchestration boundary. The low-level remote Git reader retains its narrow Git/storage/metadata dependency boundary. See the [indexed-read qualification contract](../../crab/docs/architecture/indexed-remote-reads.md) for commands, native Git oracles and cold/warm measurement semantics. +Foreground qualification and browse-index builders capture complete metadata +through `open_view_from_root_with_metadata`; pack payloads stay in authenticated +origin ranges before and after checkpoints. Catalog/control decoding and ref +capture remain measured opening work, not a completed bounded-opening gate. `qualify_remote graph-pages` measures sampled ordinal/parent records and duplicate OID batches through `CommitGraphReader`. It verifies the derived graph's captured state and Git identity before reads. diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index adfd35103..cf8770e82 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -197,7 +197,7 @@ pub async fn open_repository( ) -> Result { let root = crab_metadata::capsule_protocol::load_root(&layout).await?; let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&layout).await?; - let view = crab_read::capsule_protocol::open_view_from_root_with_control( + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( &layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-remote/src/browse_indexes.rs b/crates/crab-remote/src/browse_indexes.rs index fe58da963..8f6d1ff5f 100644 --- a/crates/crab-remote/src/browse_indexes.rs +++ b/crates/crab-remote/src/browse_indexes.rs @@ -4,7 +4,7 @@ use std::{sync::Arc, time::Duration}; use crab_metadata::capsule_protocol::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES, load_root}; use crab_read::capsule_protocol::{ - CapsuleReadLimits, open_view_from_root_with_control, read_activity_from_root, + CapsuleReadLimits, open_view_from_root_with_metadata, read_activity_from_root, }; use crab_remote_git::{RemoteGitRuntime, RepositoryIdentity, RepositoryOptions}; use crab_storage::{ETag, StorageError, Store, StoreLayout}; @@ -51,7 +51,7 @@ pub async fn ensure( max_capsule_bytes: max_bytes, max_frontier_bytes: max_bytes, }; - let view = open_view_from_root_with_control(layout, root, limits).await?; + let view = open_view_from_root_with_metadata(layout, root, limits).await?; let snapshot = view.git_snapshot()?; if snapshot.manifest.refs.is_empty() { return Ok(()); diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index a0c8c1253..8591808fb 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -148,9 +148,8 @@ pub async fn publish_capsule_checkpoint( max_capsule_bytes: maximum_bytes, max_frontier_bytes: maximum_bytes, }; - let view = - crab_read::capsule_protocol::open_view_from_root_for_checkpoint(layout, root, limits) - .await?; + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata(layout, root, limits) + .await?; publish_capsule_checkpoint_from_view(layout, &view, threshold, maximum_bytes, cancel).await } @@ -167,7 +166,7 @@ pub async fn maintain_capsule_repository( ) -> Result { check_cancelled(cancel)?; let root = crab_write::capsule_protocol::open_root(layout).await?; - let view = crab_read::capsule_protocol::open_view_from_root_for_checkpoint( + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-remote/tests/checkpoint/external_delta.rs b/crates/crab-remote/tests/checkpoint/external_delta.rs index 3cd085191..c50bc3681 100644 --- a/crates/crab-remote/tests/checkpoint/external_delta.rs +++ b/crates/crab-remote/tests/checkpoint/external_delta.rs @@ -89,98 +89,110 @@ async fn publish_dependent_blob( #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn native_capsule_install_repairs_uncheckpointed_delta_chains_and_reuses_content_names() { - let (layout, _) = empty_fixture().await; - let base = vec![b'a'; 32 * 1024]; - let mut first = base.clone(); - first[1024] = b'b'; - let mut second = first.clone(); - second[2048] = b'c'; - publish_blobs(&layout, &[("z-base", &base)]).await; - publish_dependent_blob(&layout, "m-first", &base, &first).await; - publish_dependent_blob(&layout, "a-second", &first, &second).await; - let view = view(&layout).await; - assert!(view.layered_checkpoint().is_none()); - let directory = tempfile::tempdir().unwrap(); - crab_git::initialize_bare_git_dir(directory.path()).unwrap(); - let original_packs = view - .capsules() - .iter() - .flat_map(|capsule| capsule.git_packs()) - .filter(|pack| !pack.external_delta_bases().is_empty()) - .count(); - assert_eq!(original_packs, 2); - let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); - let cancel = CancellationToken::new(); - let reader = view - .git_repository_from_store( - layout.clone(), - crab_remote_git::RepositoryIdentity::new("memory", "delta-reader", 1).unwrap(), - Arc::clone(&runtime), - crab_remote_git::RepositoryOptions::default(), - LIMIT, - &cancel, - ) - .await - .unwrap(); - let operation = reader - .operation(crab_remote_git::OperationKind::Repository, &cancel) - .await - .unwrap(); - let oids = ["a-second", "m-first", "z-base"].map(|name| { - gix_hash::ObjectId::from_hex(view.refs()[&format!("refs/tags/{name}")].as_bytes()).unwrap() - }); - let objects = operation.read_objects(&oids).await; - let objects = operation.finish(objects).await; - runtime.shutdown().await; - for (actual, expected) in objects.unwrap().iter().zip([&second, &first, &base]) { - assert_eq!(actual.data.as_ref(), expected); - } - let mut prior = None; - for _ in 0..2 { - let installed = crab_read::capsule_protocol::install_git_packs_from_store( - &view, - &layout, - directory.path(), - LIMIT, - None, - &CancellationToken::new(), - ) - .await - .unwrap(); - let paths = installed - .paths + for metadata_only in [false, true] { + let (layout, _) = empty_fixture().await; + let base = vec![b'a'; 32 * 1024]; + let mut first = base.clone(); + first[1024] = b'b'; + let mut second = first.clone(); + second[2048] = b'c'; + publish_blobs(&layout, &[("z-base", &base)]).await; + publish_dependent_blob(&layout, "m-first", &base, &first).await; + publish_dependent_blob(&layout, "a-second", &first, &second).await; + let view = if metadata_only { + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + crab_read::capsule_protocol::open_view_from_root_with_metadata(&layout, root, LIMITS) + .await + .unwrap() + } else { + view(&layout).await + }; + assert!(view.layered_checkpoint().is_none()); + let directory = tempfile::tempdir().unwrap(); + crab_git::initialize_bare_git_dir(directory.path()).unwrap(); + let original_packs = view + .capsule_run_sources() .iter() - .cloned() - .collect::>(); - assert_eq!(paths.len(), 3); - if let Some(prior) = prior.replace(paths.clone()) { - assert_eq!(paths, prior); - } - for path in paths { - let body = std::fs::read(&path).unwrap(); - assert_eq!( - path.file_name().unwrap().to_str().unwrap(), - format!("pack-{}.pack", blake3::hash(&body).to_hex()) - ); + .flat_map(|source| source.members()) + .filter(|pack| !pack.external_delta_bases().is_empty()) + .count(); + assert_eq!(original_packs, 2); + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let reader = view + .git_repository_from_store( + layout.clone(), + crab_remote_git::RepositoryIdentity::new("memory", "delta-reader", 1).unwrap(), + Arc::clone(&runtime), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap(); + let operation = reader + .operation(crab_remote_git::OperationKind::Repository, &cancel) + .await + .unwrap(); + let oids = ["a-second", "m-first", "z-base"].map(|name| { + gix_hash::ObjectId::from_hex(view.refs()[&format!("refs/tags/{name}")].as_bytes()) + .unwrap() + }); + let objects = operation.read_objects(&oids).await; + let objects = operation.finish(objects).await; + runtime.shutdown().await; + for (actual, expected) in objects.unwrap().iter().zip([&second, &first, &base]) { + assert_eq!(actual.data.as_ref(), expected); } - for (name, expected) in [ - ("z-base", &base), - ("m-first", &first), - ("a-second", &second), - ] { - let oid = &view.refs()[&format!("refs/tags/{name}")]; - let output = std::process::Command::new("git") - .arg("--git-dir") - .arg(directory.path()) - .args(["cat-file", "blob", oid]) - .output() - .unwrap(); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!(output.stdout, *expected); + let mut prior = None; + for _ in 0..2 { + let installed = crab_read::capsule_protocol::install_git_packs_from_store( + &view, + &layout, + directory.path(), + LIMIT, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + let paths = installed + .paths + .iter() + .cloned() + .collect::>(); + assert_eq!(paths.len(), 3); + if let Some(prior) = prior.replace(paths.clone()) { + assert_eq!(paths, prior); + } + for path in paths { + let body = std::fs::read(&path).unwrap(); + assert_eq!( + path.file_name().unwrap().to_str().unwrap(), + format!("pack-{}.pack", blake3::hash(&body).to_hex()) + ); + } + for (name, expected) in [ + ("z-base", &base), + ("m-first", &first), + ("a-second", &second), + ] { + let oid = &view.refs()[&format!("refs/tags/{name}")]; + let output = std::process::Command::new("git") + .arg("--git-dir") + .arg(directory.path()) + .args(["cat-file", "blob", oid]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, *expected); + } } } } diff --git a/crates/crab-s3-gateway/src/repository.rs b/crates/crab-s3-gateway/src/repository.rs index 97641b7f6..5e5e9e20f 100644 --- a/crates/crab-s3-gateway/src/repository.rs +++ b/crates/crab-s3-gateway/src/repository.rs @@ -340,7 +340,7 @@ impl ReadViewCache { } match crab_metadata::capsule_protocol::load_root(&repository.layout).await { Ok(root) => { - let capsule = crab_read::capsule_protocol::open_view_from_root_with_control( + let capsule = crab_read::capsule_protocol::open_view_from_root_with_metadata( &repository.layout, root, crab_read::capsule_protocol::CapsuleReadLimits { From b61cd1ff1674243a0d8b95999fe67c4e2bf1ec4e Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 18:55:23 -0700 Subject: [PATCH 012/127] perf(remote-git): reuse captured tip proofs Cold full-OID resolution reread ancestry pages to prove that a captured ref tip reaches itself. Reuse the root validated during opening while preserving entry admission and cancellation. --- .../docs/architecture/indexed-remote-reads.md | 34 ++++++-- crates/crab-remote-git/README.md | 4 +- crates/crab-remote-git/src/commit_graph.rs | 8 ++ .../src/repository/tests/snapshot_indexes.rs | 77 ++++++++++++++++++- 4 files changed, 115 insertions(+), 8 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index a618ceae5..e2ef8cde1 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -490,9 +490,31 @@ Complete body views remain available for embedded-pack and candidate installers. Origin-backed installation now handles uncheckpointed metadata source ranges; resident complete views retain their existing body installation path. -The initial two focused tests pass snapshot/catalog equivalence, complete -activation/visibility equivalence and absence of whole-run body GETs. Expanded -native delta-chain installation, missing-dependency error preservation and -consumer/real-store qualification remain in progress. No bounded-opening claim -follows from this payload change: historical control directories, inventories -and ref/frontier capture still require indexing and scaling proof. +Final-source metadata checks pass five tests, including snapshot/catalog and +activation/visibility equivalence, absence of whole-run body GETs and preservation +of typed missing-dependency errors. Native delta-chain installation passes both +complete and metadata-only views, including repeated installation and native Git +byte comparisons. Architecture guardrails, strict read/remote-Git library Clippy, +CLI and S3 gateway consumer checks pass. The HTTP consumer check remains running. + +`metadata-queries-r1` passes 20 cold/warm attribution comparisons at both 100 and +1,000 commits. At 10,000 commits opening reads 2,182,079 bytes, compared with +6,094,562 in `foreground-graph-r1` on the same fixed RustFS fixture. Full-OID +resolution returns the correct native Git result but reads 1,057,066 cold-query +bytes, exceeding the unchanged 1-MiB qualification bound. An independent replay +reproduces this failure: its pack-index range is 844,460 bytes, its commit entry +148 bytes, and graph pages account for another 212,458 bytes. Cold attribution +independently passes at 985,949 backend bytes and 110,104 decoded graph/path bytes; +20-sample largest-history qualification remains pending while resolution is fixed. +The terminal failure and diagnostic receipts remain outside the checkout. + +The pre-fix captured-tip regression fails with two entry charges against a +one-entry budget. Its implementation now reuses the root verified during opening, +charging one entry and checking cancellation without rereading ancestor pages. +The focused suite passes ten tests, including the new root proof, retained-history +denial, merge parents and dependency-page cancellation. The qualifier rebuild +passes; the real-store repeat remains pending. Other commit IDs +keep exact ancestry checks; arbitrary batch roots retain page verification and +in-flight cancellation tests. Pack-index and historical-control work still scales, so +this local optimization cannot establish the bounded-opening gate: directories, +inventories and ref/frontier capture require indexing and further scaling proof. diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index dec6352d4..719d238f2 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -33,7 +33,9 @@ make no additional index requests. Publication of capsule-bound browse indexes is a separate maintenance responsibility, not part of opening or pushing. Capsule graphs retain an immutable paged reader and captured commit roots; membership reads exact ancestor sets rather than allocating a full reachability -bitmap. Graph/OID/parent reads share operation admission and cancellation, including +bitmap. A full commit ID matching a verified captured root reuses that visibility +proof without rereading ancestry pages; other IDs still require exact membership. +Graph/OID/parent reads share operation admission and cancellation, including warm queries. Released manifest graph version 1 retains complete graph validation. `commits_reachable_from` is asynchronous and uses a metadata-admitted operation without opening the object-locator database. diff --git a/crates/crab-remote-git/src/commit_graph.rs b/crates/crab-remote-git/src/commit_graph.rs index 187a77b91..34d18d7cf 100644 --- a/crates/crab-remote-git/src/commit_graph.rs +++ b/crates/crab-remote-git/src/commit_graph.rs @@ -136,6 +136,14 @@ impl CommitGraphIndex { oid: ObjectId, operation: &OperationContext, ) -> Result { + operation.ensure_active()?; + // Opening verifies and sorts these captured commit roots. A root is + // already visible; rereading its ancestor set adds no authority proof. + if self.roots.binary_search(&oid).is_ok() { + operation.charge(BudgetDimension::Entries, 1).await?; + operation.ensure_active()?; + return Ok(true); + } Ok(self .reachable_from_roots(&[oid], &self.roots, operation) .await?[0]) diff --git a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs index 01d18faa3..c08f3250a 100644 --- a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs +++ b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs @@ -477,12 +477,18 @@ async fn snapshot_read_indexes_cancel_inflight_dependency_pages() { .unwrap(); let result = async { if dependency == "commit-graph" { + // Caller-supplied batch roots still require ancestry pages; + // a verified captured tip no longer needs an in-flight read. return repository .state .commit_graph .as_ref() .unwrap() - .is_reachable(ObjectId::Sha1([0x11; 20]), &operation) + .reachable_from_roots( + &[ObjectId::Sha1([0x11; 20])], + &[ObjectId::Sha1([0x11; 20])], + &operation, + ) .await .map(|_| ()); } @@ -563,6 +569,75 @@ async fn released_manifest_graph_remains_usable_for_metadata_membership() { runtime.shutdown().await; } +#[tokio::test] +async fn captured_tip_membership_avoids_ancestry_reads_and_retains_admission() { + let fixture = open_fixture(1, None).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; + let requests = Arc::new(AtomicUsize::new(0)); + let observed = requests.clone(); + let store = fixture + .store + .clone() + .with_read_request_observer(Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + })); + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let repository = RemoteGitRepository::from_snapshot( + StoreLayout::new(store, "org/repo".into()), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + requests.store(0, Ordering::SeqCst); + let graph = repository.state.commit_graph.as_ref().unwrap(); + let operation = repository + .operation_with_limits( + crate::OperationKind::Resolve, + &cancel, + OperationLimits { + max_entries: 1, + ..Default::default() + }, + ) + .await + .unwrap(); + assert!( + graph + .is_reachable(ObjectId::Sha1([0x11; 20]), &operation) + .await + .unwrap() + ); + assert_eq!(requests.load(Ordering::SeqCst), 0); + assert!(matches!( + graph + .is_reachable(ObjectId::Sha1([0x11; 20]), &operation) + .await, + Err(Error::LimitExceeded { + limit: "entries", + .. + }) + )); + operation.finish(Ok(())).await.unwrap(); + let operation = repository + .operation(crate::OperationKind::Resolve, &cancel) + .await + .unwrap(); + cancel.cancel(); + assert!(matches!( + graph + .is_reachable(ObjectId::Sha1([0x11; 20]), &operation) + .await, + Err(Error::Cancelled) + )); + let _ = operation.finish(Ok(())).await; + runtime.shutdown().await; +} + #[tokio::test] async fn paged_graph_queries_preserve_merge_parents_and_exclude_retained_history() { use crab_metadata::split_commit_graph::{ From 786af179b7884ed412022770e567c9b09985353f Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 19:05:54 -0700 Subject: [PATCH 013/127] docs: record metadata opening qualification --- .../docs/architecture/indexed-remote-reads.md | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e2ef8cde1..8f5ce4ec6 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -479,7 +479,7 @@ are debug component receipts, not native Git integrity, release-image performanc or whole-publication measurements. Release/provider and broader lifecycle gates remain open. Fixtures, source/binary hashes and raw receipts remain external. -## Metadata-first opening: implementation in progress +## Metadata-first opening qualification The existing complete-metadata checkpoint capture is now named `open_view_from_root_with_metadata`. HTTP browsing, S3 read views, CLI remote @@ -495,7 +495,10 @@ activation/visibility equivalence, absence of whole-run body GETs and preservati of typed missing-dependency errors. Native delta-chain installation passes both complete and metadata-only views, including repeated installation and native Git byte comparisons. Architecture guardrails, strict read/remote-Git library Clippy, -CLI and S3 gateway consumer checks pass. The HTTP consumer check remains running. +CLI, S3 gateway and HTTP consumer checks pass; the frontend build ran before the +HTTP check. The initial PR image qualification failed because ECR rejected all +five attempts to pull its pinned AWS CLI image with a registry data-limit error. +This remains a CI qualification gap rather than a passing image result. `metadata-queries-r1` passes 20 cold/warm attribution comparisons at both 100 and 1,000 commits. At 10,000 commits opening reads 2,182,079 bytes, compared with @@ -513,8 +516,18 @@ one-entry budget. Its implementation now reuses the root verified during opening charging one entry and checking cancellation without rereading ancestor pages. The focused suite passes ten tests, including the new root proof, retained-history denial, merge parents and dependency-page cancellation. The qualifier rebuild -passes; the real-store repeat remains pending. Other commit IDs -keep exact ancestry checks; arbitrary batch roots retain page verification and +passes. Other commit IDs keep exact ancestry checks; arbitrary batch roots retain page verification and in-flight cancellation tests. Pack-index and historical-control work still scales, so this local optimization cannot establish the bounded-opening gate: directories, inventories and ref/frontier capture require indexing and further scaling proof. + +`metadata-queries-r2` passes all 63 native Git comparisons with the rebuilt binary: +20 cold/warm attribution runs at each of 100/1k/10k commits and one full-OID +resolution run at each size. At 10k, cold resolution reads 844,608 bytes and no +graph pages; cold attribution reads at most 985,949 bytes and decodes 110,104 +graph/path bytes. All warm queries read zero backend bytes. The unchanged fixture +byte bounds pass. Cold attribution p95 is 259.152 / 187.268 / 460.314 ms on this +shared debug host, within its frozen relative bound; these timings do not establish +release/provider performance. Full pack-index reads still scale with repository +objects, and control opening still scales with history. Source/binary hashes, +the earlier failure, raw receipts and checked summaries remain external. From b5011fff24a779ac13401b86f9c21f4ff4ace28f Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 20:23:02 -0700 Subject: [PATCH 014/127] feat(metadata): add paged physical object index Authenticate OID and physical-offset directories against captured pack inventory. Reuse bounded traversal for graph and path readers; production object-index publication and reader wiring remain pending. --- .../docs/architecture/indexed-remote-reads.md | 38 + crates/crab-metadata/src/error.rs | 5 + .../src/git_object_locator/mod.rs | 2 + .../src/git_object_locator/paged.rs | 685 ++++++++++++++++++ .../src/git_object_locator/paged/tests.rs | 392 ++++++++++ crates/crab-metadata/src/pages.rs | 21 +- crates/crab-metadata/src/pages/cache.rs | 91 ++- crates/crab-metadata/src/path_state/reader.rs | 121 +--- crates/crab-metadata/src/path_state/scan.rs | 1 + .../src/split_commit_graph/reader.rs | 122 +--- crates/crab-remote/examples/support/mod.rs | 15 +- 11 files changed, 1300 insertions(+), 193 deletions(-) create mode 100644 crates/crab-metadata/src/git_object_locator/paged.rs create mode 100644 crates/crab-metadata/src/git_object_locator/paged/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 8f5ce4ec6..2ed692250 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -531,3 +531,41 @@ shared debug host, within its frozen relative bound; these timings do not establ release/provider performance. Full pack-index reads still scale with repository objects, and control opening still scales with history. Source/binary hashes, the earlier failure, raw receipts and checked summaries remain external. + +## Paged physical object lookup: implementation in progress + +The measured 10k cold-resolution query reads an 844,460-byte pack-index range to +locate its 148-byte commit entry. A new component indexes OID-to-placement, +pack-offset-to-OID and pack-inventory bindings through authenticated pages. The +reverse direction is required for offset deltas; paging only OID lookup would +leave reconstruction with a hidden full-index read. Physical aliases retain all +source offsets while primary lookup prefers a self-contained encoding. Logical +facts must agree across aliases; catalog ordinals and ref visibility remain the +existing owners' contracts. + +Graph, path-state and object lookup now share one admitted page-traversal engine. +The component run passes nine tests, including bounded 10k lookup, duplicate +positions, corruption, snapshot/inventory binding, warm encoded-work admission, +input-order canonicalization and a re-signed inconsistent-directory test. Removing +OID/offset agreement makes that regression fail; source is restored. The first +negative-control invocation selected zero tests and is not qualification; the +corrected invocation runs the complete test name and rejects the forged result. +Shared consumers pass 21 path-state and 27 graph tests, strict metadata Clippy and +payload-only compilation. The per-ref publication contract permits generation +zero after a push; the new descriptor now accepts that captured generation. +Its added regression passes alongside the other eight component tests. + +Traversal now admits four concurrent page reads for every consumer; path-state +previously admitted sixteen and graph lookup already admitted four. Correctness +tests cover the shared mechanics, but cold/warm latency must be requalified before +claiming this concurrency change is performance-neutral. Object-page decode +counts are recorded separately by the existing qualification observer. + +This component is not yet wired to foreground lookup or production readiness. +Publication must construct it from verified complete physical sources, verify +complete readback, and bind its descriptor to the captured state. Runtime OID and +physical-offset consumers must use it with operation admission and cancellation; +GC must trace its descriptor/body from retained readiness roots. Native Git and +real-store qualifications, build/storage amplification and source API release +qualification remain mandatory before declaring this slice complete. Outer +control/inventory/ref opening and stages 5–8 remain open. diff --git a/crates/crab-metadata/src/error.rs b/crates/crab-metadata/src/error.rs index 1d6bb6549..7943a3e45 100644 --- a/crates/crab-metadata/src/error.rs +++ b/crates/crab-metadata/src/error.rs @@ -21,6 +21,11 @@ pub enum MetadataError { #[source] source: serde_json::Error, }, + #[error("Git object index record encoding is invalid")] + GitObjectIndexRecord { + #[source] + source: serde_json::Error, + }, /// A file lookup could not acquire process-wide execution capacity. #[cfg(feature = "file-index-reader")] #[error("file lookup admission closed")] diff --git a/crates/crab-metadata/src/git_object_locator/mod.rs b/crates/crab-metadata/src/git_object_locator/mod.rs index 852d21414..e9e3c79cc 100644 --- a/crates/crab-metadata/src/git_object_locator/mod.rs +++ b/crates/crab-metadata/src/git_object_locator/mod.rs @@ -1,9 +1,11 @@ //! Generation-bound Git object catalog for object-store range reads. mod format; +mod paged; mod reader; mod writer; +pub use paged::{PagedGitObjectIndex, PagedGitObjectIndexWrite}; pub use reader::{GitObjectLocatorSession, GitObjectLookup}; pub use writer::{GitObjectLocatorWriter, LocatorSweepStats, LocatorWriteStats}; diff --git a/crates/crab-metadata/src/git_object_locator/paged.rs b/crates/crab-metadata/src/git_object_locator/paged.rs new file mode 100644 index 000000000..ba62ee041 --- /dev/null +++ b/crates/crab-metadata/src/git_object_locator/paged.rs @@ -0,0 +1,685 @@ +//! Immutable physical placement indexes; callers retain ref visibility and origin verification. +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use bytes::Bytes; +use crab_storage::{Store, StoreLayout}; +use crab_xet::hash::MerkleHash; +use serde::{Deserialize, Serialize}; + +use super::{GitObjectLocation, GitObjectLocator, GitPackInventoryEntry, format}; +use crate::error::Result; +use crate::manifests::Manifest; +use crate::pages::{Kind, PageCache, PageRef, PageWork, Rows, Writer}; +use crate::validation::{corrupt_object, validate_content_hash}; + +const MAX_DESCRIPTOR_BYTES: u64 = 16 * 1024; + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Descriptor { + version: u32, + state_digest: String, + generation: u64, + pack_index_hash: String, + git_validation_digest: String, + body_hash: String, + body_bytes: u64, + objects: PageRef, + offsets: PageRef, + packs: PageRef, +} + +/// Canonical construction of OID locations, physical offsets and complete pack coverage. +pub struct PagedGitObjectIndexWrite { + descriptor: Descriptor, + body: Bytes, +} + +/// Immutable physical lookup data bound to a captured Git inventory and state. +#[derive(Debug)] +pub struct PagedGitObjectIndex { + descriptor: Descriptor, + descriptor_bytes: u64, + path: object_store::path::Path, + maximum: u64, + cache: PageCache, +} + +impl PagedGitObjectIndexWrite { + /// Build from every verified packed entry in the captured capsule inventory. + /// + /// Entries cover each pack without holes; this index does not assign catalog ordinals. + pub fn build( + state_digest: &str, + manifest: &Manifest, + inventory: &[GitPackInventoryEntry], + entries: &[([u8; 20], GitObjectLocator)], + ) -> Result { + for hash in [ + state_digest, + &manifest.pack_index_hash, + &manifest.git_validation_digest, + ] { + validate_content_hash(hash, "object index binding", "Git object index")?; + } + let mut packs = BTreeMap::new(); + for pack in inventory { + if packs.insert(pack.pack_id, *pack).is_some() { + return bad("duplicate pack inventory"); + } + } + let mut manifest_packs = packs + .values() + .map(|pack| crate::manifests::PackManifestEntry { + pack_id: pack.pack_id.to_string(), + size: pack.pack_size, + content_hash: pack.pack_id.to_string(), + ref_tips: Vec::new(), + object_count: pack.object_count, + }) + .collect::>(); + manifest_packs.sort_unstable_by(|left, right| left.pack_id.cmp(&right.pack_id)); + if crate::manifests::compact_pack_index(manifest.generation, &manifest_packs)?.0 + != manifest.pack_index_hash + { + return bad("physical inventory does not match captured pack identity"); + } + let mut offsets = BTreeMap::new(); + let mut objects = BTreeMap::<[u8; 20], GitObjectLocator>::new(); + for (oid, input) in entries { + let mut locator = *input; + locator.ordinal = 0; + let pack = packs + .get(&locator.pack_id) + .ok_or_else(|| corruption("location is outside inventory"))?; + if !format::validate_location_for_pack(locator.location, pack.pack_size) { + return bad("invalid physical packed-entry range"); + } + if offsets + .insert( + (locator.pack_id, locator.location.pack_offset), + (*oid, locator), + ) + .is_some() + { + return bad("duplicate physical offset"); + } + // Prefer self-contained encodings, then a stable physical identity. + // Delta-base facts belong to an encoding and must not cross pack aliases. + match objects.entry(*oid) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(locator); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + let previous = entry.get_mut(); + if previous + .metadata + .kind + .zip(locator.metadata.kind) + .is_some_and(|(left, right)| left != right) + || previous + .metadata + .logical_size + .zip(locator.metadata.logical_size) + .is_some_and(|(left, right)| left != right) + { + return bad("object aliases disagree about logical facts"); + } + let kind = previous.metadata.kind.or(locator.metadata.kind); + let logical_size = previous + .metadata + .logical_size + .or(locator.metadata.logical_size); + if placement_rank(locator) < placement_rank(*previous) { + *previous = locator; + } + previous.metadata.kind = kind; + previous.metadata.logical_size = logical_size; + } + } + } + for (pack_id, pack) in &packs { + let mut next = 12_u64; + let mut count = 0_u64; + for ((_, offset), (_, locator)) in offsets.range((*pack_id, 0)..=(*pack_id, u64::MAX)) { + if *offset != next { + return bad("pack entries overlap or leave a gap"); + } + next = next + .checked_add(locator.location.entry_len) + .ok_or_else(|| corruption("pack range overflows"))?; + count += 1; + } + if count != pack.object_count || next.checked_add(20) != Some(pack.pack_size) { + return bad("physical entries do not cover the declared pack"); + } + } + let object_rows = objects + .into_iter() + .map(|(oid, locator)| (oid.to_vec(), encode_locator(locator))) + .collect(); + let offset_rows = offsets + .into_iter() + .map(|((pack_id, offset), (oid, locator))| { + let mut value = oid.to_vec(); + value.extend_from_slice(&encode_location(locator.location, locator.metadata)); + (offset_key(pack_id, offset), value) + }) + .collect(); + let pack_rows = packs + .into_iter() + .map(|(pack_id, pack)| { + let mut value = pack.object_count.to_be_bytes().to_vec(); + value.extend_from_slice(&pack.pack_size.to_be_bytes()); + (<[u8; 32]>::from(pack_id).to_vec(), value) + }) + .collect(); + let (body, objects, offsets, packs) = encode_trees(object_rows, offset_rows, pack_rows)?; + let descriptor = Descriptor { + version: 1, + state_digest: state_digest.to_owned(), + generation: manifest.generation, + pack_index_hash: manifest.pack_index_hash.clone(), + git_validation_digest: manifest.git_validation_digest.clone(), + body_hash: blake3::hash(&body).to_hex().to_string(), + body_bytes: body.len() as u64, + objects, + offsets, + packs, + }; + descriptor.validate()?; + Ok(Self { + descriptor, + body: body.into(), + }) + } + + /// Upload immutable page bytes before their descriptor and return its content identity. + pub async fn upload(&self, store: &Store, layout: &StoreLayout) -> Result { + crate::derived_index::upload( + store, + &body_path(layout, &self.descriptor.body_hash), + &self.descriptor.body_hash, + &self.body, + ) + .await?; + let bytes = serde_json::to_vec(&self.descriptor) + .map_err(|source| crate::error::MetadataError::GitObjectIndexRecord { source })?; + let hash = blake3::hash(&bytes).to_hex().to_string(); + crate::derived_index::upload( + store, + &layout.bulk_manifest_path("git-object-index", &hash), + &hash, + &bytes, + ) + .await?; + Ok(hash) + } +} + +impl PagedGitObjectIndex { + /// Capture an authenticated descriptor without loading its physical inventory or locations. + /// + /// Callers accept readiness only after complete verification; placement never authorizes refs. + pub async fn load( + store: &Store, + layout: &StoreLayout, + hash: &str, + state_digest: &str, + manifest: &Manifest, + maximum: u64, + ) -> Result { + validate_content_hash(hash, "object index descriptor", "Git object index")?; + let expected = hash_bytes(hash)?; + let bytes = store + .verify_bounded( + &layout.bulk_manifest_path("git-object-index", hash), + &expected, + maximum.min(MAX_DESCRIPTOR_BYTES), + ) + .await?; + let descriptor: Descriptor = serde_json::from_slice(&bytes) + .map_err(|source| crate::error::MetadataError::GitObjectIndexRecord { source })?; + descriptor.validate()?; + if descriptor.state_digest != state_digest + || descriptor.generation != manifest.generation + || descriptor.pack_index_hash != manifest.pack_index_hash + || descriptor.git_validation_digest != manifest.git_validation_digest + { + return bad("object index does not match captured state"); + } + let path = body_path(layout, &descriptor.body_hash); + Ok(Self { + descriptor, + descriptor_bytes: bytes.len() as u64, + path, + maximum, + cache: PageCache::default(), + }) + } + + /// Resolve OIDs against the captured physical inventory, preserving duplicates and misses. + /// + /// The caller admits request/output counts and supplies an admitted store for the captured origin. + pub async fn lookup( + &self, + store: &Store, + oids: &[[u8; 20]], + inventory: &HashMap, + ) -> Result>> { + let mut work = self.work(); + let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + let values = self + .lookup_rows( + store, + &self.descriptor.objects, + Kind::ObjectLocators, + &keys, + &mut work, + ) + .await?; + let output = values + .into_iter() + .map(|value| value.map(|value| decode_locator(&value)).transpose()) + .collect::>>()?; + let locators = output.iter().flatten().copied().collect::>(); + let offset_keys = locators + .iter() + .map(|locator| offset_key(locator.pack_id, locator.location.pack_offset)) + .collect::>(); + let physical = self + .lookup_rows( + store, + &self.descriptor.offsets, + Kind::PackOffsets, + &offset_keys, + &mut work, + ) + .await?; + let mut physical = physical.into_iter(); + for (oid, locator) in oids.iter().zip(&output) { + let Some(locator) = locator else { + continue; + }; + let row = physical + .next() + .flatten() + .ok_or_else(|| corruption("object location has no physical offset"))?; + if row.len() != 64 || array::<20>(&row, 0)? != *oid { + return bad("object and physical offset identities differ"); + } + let (location, metadata) = decode_location(locator.location.pack_offset, &row[20..])?; + if location != locator.location + || metadata.delta_base_oid != locator.metadata.delta_base_oid + || metadata + .kind + .zip(locator.metadata.kind) + .is_some_and(|(left, right)| left != right) + || metadata + .logical_size + .zip(locator.metadata.logical_size) + .is_some_and(|(left, right)| left != right) + { + return bad("object and physical offset facts differ"); + } + } + self.validate_inventory(store, &locators, inventory, &mut work) + .await?; + Ok(output) + } + + /// Resolve physical delta-base offsets without reading an entire reverse pack index. + /// + /// The caller admits request/output counts and supplies an admitted store for the captured origin. + pub async fn oids_at_offsets( + &self, + store: &Store, + offsets: &[(MerkleHash, u64)], + inventory: &HashMap, + ) -> Result>> { + let mut work = self.work(); + let keys = offsets + .iter() + .map(|(pack, offset)| offset_key(*pack, *offset)) + .collect::>(); + let values = self + .lookup_rows( + store, + &self.descriptor.offsets, + Kind::PackOffsets, + &keys, + &mut work, + ) + .await?; + let mut locators = Vec::new(); + let mut output = Vec::with_capacity(values.len()); + for ((pack_id, offset), value) in offsets.iter().zip(values) { + let Some(value) = value else { + output.push(None); + continue; + }; + if value.len() != 64 { + return bad("invalid offset row length"); + } + let oid = array(&value, 0)?; + let (location, metadata) = decode_location(*offset, &value[20..])?; + locators.push(GitObjectLocator { + ordinal: 0, + pack_id: *pack_id, + location, + metadata, + }); + output.push(Some(oid)); + } + self.validate_inventory(store, &locators, inventory, &mut work) + .await?; + let oid_keys = output + .iter() + .flatten() + .map(|oid| oid.to_vec()) + .collect::>(); + let primary = self + .lookup_rows( + store, + &self.descriptor.objects, + Kind::ObjectLocators, + &oid_keys, + &mut work, + ) + .await?; + if primary.iter().any(Option::is_none) { + return bad("physical offset has no object identity"); + } + let primary = primary + .into_iter() + .flatten() + .map(|value| decode_locator(&value)) + .collect::>>()?; + self.validate_inventory(store, &primary, inventory, &mut work) + .await?; + Ok(output) + } + + /// Verify complete coverage and canonical bytes before publishing readiness. + pub async fn verify_complete( + &self, + store: &Store, + manifest: &Manifest, + maximum: u64, + ) -> Result<()> { + let bytes = store + .verify_bounded( + &self.path, + &hash_bytes(&self.descriptor.body_hash)?, + maximum, + ) + .await?; + if bytes.len() as u64 != self.descriptor.body_bytes { + return bad("object index body length mismatch"); + } + let mut audit = BTreeMap::new(); + crate::pages::scan( + &bytes, + &self.descriptor.objects, + Kind::ObjectLocators, + self.path.as_ref(), + &mut audit, + )?; + let packs = crate::pages::scan( + &bytes, + &self.descriptor.packs, + Kind::PackInventory, + self.path.as_ref(), + &mut audit, + )?; + let inventory = packs + .into_iter() + .map(|(key, value)| { + if value.len() != 16 { + return bad("invalid inventory row length"); + } + Ok(GitPackInventoryEntry { + pack_id: MerkleHash::from(array(&key, 0)?), + object_count: u64::from_be_bytes(array(&value, 0)?), + pack_size: u64::from_be_bytes(array(&value, 8)?), + }) + }) + .collect::>>()?; + let offsets = crate::pages::scan( + &bytes, + &self.descriptor.offsets, + Kind::PackOffsets, + self.path.as_ref(), + &mut audit, + )?; + let entries = offsets + .into_iter() + .map(|(key, value)| { + if value.len() != 64 { + return bad("invalid offset row length"); + } + let offset = u64::from_be_bytes(array(&key, 32)?); + let (location, metadata) = decode_location(offset, &value[20..])?; + Ok(( + array(&value, 0)?, + GitObjectLocator { + ordinal: 0, + pack_id: MerkleHash::from(array(&key, 0)?), + location, + metadata, + }, + )) + }) + .collect::>>()?; + let rebuilt = PagedGitObjectIndexWrite::build( + &self.descriptor.state_digest, + manifest, + &inventory, + &entries, + )?; + if rebuilt.descriptor != self.descriptor || rebuilt.body.as_ref() != bytes.as_ref() { + return bad("object index coverage or canonical placement mismatch"); + } + Ok(()) + } + + fn work(&self) -> PageWork { + PageWork { + maximum: self.maximum, + bytes: self.descriptor_bytes, + seen: BTreeSet::new(), + } + } + + async fn lookup_rows( + &self, + store: &Store, + root: &PageRef, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + self.cache + .lookup( + store, + &self.path, + 0, + self.descriptor.body_bytes, + root, + kind, + keys, + work, + ) + .await + } + + async fn validate_inventory( + &self, + store: &Store, + locators: &[GitObjectLocator], + inventory: &HashMap, + work: &mut PageWork, + ) -> Result<()> { + let pack_ids = locators + .iter() + .map(|locator| locator.pack_id) + .collect::>() + .into_iter() + .collect::>(); + let keys = pack_ids + .iter() + .map(|pack| <[u8; 32]>::from(*pack).to_vec()) + .collect::>(); + let values = self + .lookup_rows( + store, + &self.descriptor.packs, + Kind::PackInventory, + &keys, + work, + ) + .await?; + for (pack_id, value) in pack_ids.into_iter().zip(values) { + let value = value.ok_or_else(|| corruption("required pack binding is absent"))?; + if value.len() != 16 { + return bad("invalid inventory row length"); + } + let expected = inventory + .get(&pack_id) + .ok_or_else(|| corruption("pack is outside captured inventory"))?; + if u64::from_be_bytes(array(&value, 0)?) != expected.object_count + || u64::from_be_bytes(array(&value, 8)?) != expected.pack_size + { + return bad("pack binding differs from captured inventory"); + } + } + for locator in locators { + if !inventory.get(&locator.pack_id).is_some_and(|pack| { + format::validate_location_for_pack(locator.location, pack.pack_size) + }) { + return bad("location escapes captured pack"); + } + } + Ok(()) + } +} + +impl Descriptor { + fn validate(&self) -> Result<()> { + if self.version != 1 { + return bad("unsupported object index descriptor"); + } + for hash in [ + &self.state_digest, + &self.pack_index_hash, + &self.git_validation_digest, + &self.body_hash, + ] { + validate_content_hash(hash, "object index identity", "Git object index")?; + } + for (root, kind) in [ + (&self.objects, Kind::ObjectLocators), + (&self.offsets, Kind::PackOffsets), + (&self.packs, Kind::PackInventory), + ] { + root.validate(kind, self.body_bytes)?; + } + if self.objects.count > self.offsets.count || self.packs.count > self.offsets.count { + return bad("object index counts are inconsistent"); + } + Ok(()) + } +} + +fn encode_trees( + mut objects: Rows, + mut offsets: Rows, + mut packs: Rows, +) -> Result<(Vec, PageRef, PageRef, PageRef)> { + for rows in [&mut objects, &mut offsets, &mut packs] { + rows.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + } + let mut builder = Writer::default(); + let objects = builder.tree(Kind::ObjectLocators, objects)?; + let offsets = builder.tree(Kind::PackOffsets, offsets)?; + let packs = builder.tree(Kind::PackInventory, packs)?; + Ok((builder.bytes, objects, offsets, packs)) +} +fn placement_rank(locator: GitObjectLocator) -> (bool, MerkleHash, u64) { + ( + locator.metadata.delta_base_oid.is_some(), + locator.pack_id, + locator.location.pack_offset, + ) +} +fn offset_key(pack: MerkleHash, offset: u64) -> Vec { + let mut key = <[u8; 32]>::from(pack).to_vec(); + key.extend_from_slice(&offset.to_be_bytes()); + key +} +fn encode_location(location: GitObjectLocation, metadata: super::GitObjectMetadata) -> Vec { + let mut value = location.entry_len.to_be_bytes().to_vec(); + value.extend_from_slice(&location.crc32.to_be_bytes()); + value.extend_from_slice(&format::encode_object_metadata(metadata)); + value +} +fn encode_locator(locator: GitObjectLocator) -> Vec { + let mut value = <[u8; 32]>::from(locator.pack_id).to_vec(); + value.extend_from_slice(&locator.location.pack_offset.to_be_bytes()); + value.extend_from_slice(&encode_location(locator.location, locator.metadata)); + value +} +fn decode_location( + offset: u64, + bytes: &[u8], +) -> Result<(GitObjectLocation, super::GitObjectMetadata)> { + if bytes.len() != 44 { + return bad("invalid location row length"); + } + let metadata = format::decode_object_metadata(&bytes[12..]) + .ok_or_else(|| corruption("invalid object metadata"))?; + Ok(( + GitObjectLocation { + pack_offset: offset, + entry_len: u64::from_be_bytes(array(bytes, 0)?), + crc32: u32::from_be_bytes(array(bytes, 8)?), + }, + metadata, + )) +} +fn decode_locator(bytes: &[u8]) -> Result { + if bytes.len() != 84 { + return bad("invalid locator row length"); + } + let (location, metadata) = + decode_location(u64::from_be_bytes(array(bytes, 32)?), &bytes[40..])?; + Ok(GitObjectLocator { + ordinal: 0, + pack_id: MerkleHash::from(array(bytes, 0)?), + location, + metadata, + }) +} +fn array(bytes: &[u8], offset: usize) -> Result<[u8; N]> { + bytes + .get(offset..offset + N) + .and_then(|value| value.try_into().ok()) + .ok_or_else(|| corruption("truncated object index value")) +} +fn hash_bytes(hash: &str) -> Result<[u8; 32]> { + blake3::Hash::from_hex(hash) + .map(|hash| *hash.as_bytes()) + .map_err(|_| corruption("invalid object index hash")) +} +fn body_path(layout: &StoreLayout, hash: &str) -> object_store::path::Path { + layout.repo_path(&format!("metadata/git-object-index/layers/{hash}.bin")) +} +fn corruption(reason: &str) -> crate::error::MetadataError { + corrupt_object("Git object index", reason) +} +fn bad(reason: &str) -> Result { + Err(corruption(reason)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/git_object_locator/paged/tests.rs b/crates/crab-metadata/src/git_object_locator/paged/tests.rs new file mode 100644 index 000000000..5f8e288d6 --- /dev/null +++ b/crates/crab-metadata/src/git_object_locator/paged/tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::git_object_locator::GitObjectMetadata; +use object_store::{ObjectStoreExt, memory::InMemory}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +fn oid(index: u32) -> [u8; 20] { + let mut oid = [1; 20]; + oid[..4].copy_from_slice(&index.to_be_bytes()); + oid +} +type Inputs = ( + Manifest, + Vec, + Vec<([u8; 20], GitObjectLocator)>, +); + +fn inputs(count: u32) -> Inputs { + let mut manifest = Manifest::default_for_repo("refs/heads/main"); + manifest.generation = 1; + manifest.git_validation_digest = "b".repeat(64); + let pack_id = MerkleHash::from([3; 32]); + let inventory = vec![GitPackInventoryEntry { + pack_id, + object_count: u64::from(count), + pack_size: 32 + u64::from(count) * 10, + }]; + manifest.pack_index_hash = crate::manifests::compact_pack_index( + 1, + &[crate::manifests::PackManifestEntry { + pack_id: pack_id.to_string(), + size: inventory[0].pack_size, + content_hash: pack_id.to_string(), + ref_tips: Vec::new(), + object_count: u64::from(count), + }], + ) + .unwrap() + .0; + let entries = (0..count) + .map(|index| { + ( + oid(index), + GitObjectLocator { + ordinal: 0, + pack_id, + location: GitObjectLocation { + pack_offset: 12 + u64::from(index) * 10, + entry_len: 10, + crc32: index, + }, + metadata: GitObjectMetadata::default(), + }, + ) + }) + .collect(); + (manifest, inventory, entries) +} + +#[tokio::test] +async fn point_lookup_and_delta_offsets_are_bounded_and_preserve_duplicate_positions() { + let (manifest, inventory, entries) = inputs(10_000); + let write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = + Store::new(Arc::new(InMemory::new())).with_read_byte_observer(Arc::new(move |read| { + observed.fetch_add(read, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store.clone(), "paged-object-index".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + bytes.store(0, Ordering::Relaxed); + let reader = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 256 * 1024, + ) + .await + .unwrap(); + assert_eq!(bytes.swap(0, Ordering::Relaxed), reader.descriptor_bytes); + let inventory = inventory + .into_iter() + .map(|pack| (pack.pack_id, pack)) + .collect(); + let requested = [oid(9999), oid(0), oid(9999), oid(20_000)]; + let expected = vec![ + Some(entries[9999].1), + Some(entries[0].1), + Some(entries[9999].1), + None, + ]; + assert_eq!( + reader.lookup(&store, &requested, &inventory).await.unwrap(), + expected + ); + let cold = bytes.swap(0, Ordering::Relaxed); + assert!(cold <= 128 * 1024 && cold < write.body.len() as u64); + assert_eq!( + reader.lookup(&store, &requested, &inventory).await.unwrap(), + expected + ); + assert_eq!(bytes.load(Ordering::Relaxed), 0); + let offsets = [entries[9999].1.location.pack_offset, 12, 12, 13] + .map(|offset| (entries[0].1.pack_id, offset)); + assert_eq!( + reader + .oids_at_offsets(&store, &offsets, &inventory) + .await + .unwrap(), + vec![Some(oid(9999)), Some(oid(0)), Some(oid(0)), None] + ); + reader + .verify_complete(&store, &manifest, write.body.len() as u64) + .await + .unwrap(); +} + +#[test] +fn construction_rejects_incomplete_or_overlapping_physical_coverage() { + let (manifest, inventory, entries) = inputs(3); + for case in [0, 1, 2] { + let mut entries = entries.clone(); + match case { + 0 => { + entries.remove(1); + } + 1 => { + entries[1].1.location.pack_offset = 12; + } + _ => { + entries[2].1.location.entry_len = u64::MAX; + } + } + assert!( + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries) + .is_err() + ); + } +} + +#[tokio::test] +async fn corrupt_required_pages_never_become_an_absent_oid() { + let (manifest, inventory, entries) = inputs(1000); + let write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let backend = Arc::new(InMemory::new()); + let store = Store::new(backend.clone()); + let layout = StoreLayout::new(store.clone(), "corrupt-object-index".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + let reader = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 256 * 1024, + ) + .await + .unwrap(); + let mut body = write.body.to_vec(); + body[reader.descriptor.objects.offset as usize] ^= 1; + backend + .put(&reader.path, Bytes::from(body).into()) + .await + .unwrap(); + let inventory = inventory + .into_iter() + .map(|pack| (pack.pack_id, pack)) + .collect(); + assert!( + reader + .lookup(&store, &[oid(20_000)], &inventory) + .await + .is_err() + ); +} + +#[tokio::test] +async fn warm_pages_cannot_bypass_inventory_binding_or_encoded_work_limits() { + let (manifest, inventory, entries) = inputs(1000); + let write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "bound-object-index".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + let mut reader = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 256 * 1024, + ) + .await + .unwrap(); + let mut inventory: HashMap<_, _> = inventory + .into_iter() + .map(|pack| (pack.pack_id, pack)) + .collect(); + reader + .lookup(&store, &[oid(999)], &inventory) + .await + .unwrap(); + inventory.get_mut(&entries[0].1.pack_id).unwrap().pack_size += 1; + assert!( + reader + .lookup(&store, &[oid(999)], &inventory) + .await + .is_err() + ); + inventory.get_mut(&entries[0].1.pack_id).unwrap().pack_size -= 1; + reader.maximum = reader.descriptor_bytes; + assert!( + reader + .lookup(&store, &[oid(999)], &inventory) + .await + .is_err() + ); +} + +#[tokio::test] +async fn descriptors_cannot_cross_captured_git_states() { + let (manifest, inventory, entries) = inputs(10); + let write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "state-object-index".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + assert!( + PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"d".repeat(64), + &manifest, + 256 * 1024 + ) + .await + .is_err() + ); +} + +#[test] +fn object_aliases_prefer_full_encodings_and_keep_every_physical_offset() { + let (mut manifest, mut inventory, mut entries) = inputs(1); + entries[0].1.metadata.delta_base_oid = Some(oid(0)); + let mut full = entries[0].1; + full.pack_id = MerkleHash::from([4; 32]); + full.metadata.delta_base_oid = None; + full.metadata.kind = Some(crate::git_object_locator::GitObjectKind::Blob); + entries.push((oid(0), full)); + inventory.push(GitPackInventoryEntry { + pack_id: full.pack_id, + object_count: 1, + pack_size: 42, + }); + let mut packs = inventory + .iter() + .map(|pack| crate::manifests::PackManifestEntry { + pack_id: pack.pack_id.to_string(), + size: pack.pack_size, + content_hash: pack.pack_id.to_string(), + ref_tips: Vec::new(), + object_count: pack.object_count, + }) + .collect::>(); + packs.sort_unstable_by(|left, right| left.pack_id.cmp(&right.pack_id)); + manifest.pack_index_hash = crate::manifests::compact_pack_index(1, &packs).unwrap().0; + let write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let mut audit = BTreeMap::new(); + let rows = crate::pages::scan( + &write.body, + &write.descriptor.objects, + Kind::ObjectLocators, + "test index", + &mut audit, + ) + .unwrap(); + let selected = decode_locator(&rows[0].1).unwrap(); + assert_eq!((selected, write.descriptor.offsets.count), (full, 2)); +} + +proptest::proptest! { + #[test] + fn physical_input_order_does_not_change_authenticated_index_bytes(count in 1u32..1000, rotation in 0usize..1000) { + let (manifest, inventory, mut entries) = inputs(count); + let first = PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let rotation = rotation % entries.len(); + entries.rotate_left(rotation); + entries.reverse(); + let second = PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + proptest::prop_assert_eq!((first.descriptor, first.body), (second.descriptor, second.body)); + } +} + +#[tokio::test] +async fn resigned_oid_locations_must_agree_with_the_physical_offset_directory() { + let (manifest, inventory, entries) = inputs(10); + let mut write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let mut audit = BTreeMap::new(); + let mut objects = crate::pages::scan( + &write.body, + &write.descriptor.objects, + Kind::ObjectLocators, + "test index", + &mut audit, + ) + .unwrap(); + let offsets = crate::pages::scan( + &write.body, + &write.descriptor.offsets, + Kind::PackOffsets, + "test index", + &mut audit, + ) + .unwrap(); + let packs = crate::pages::scan( + &write.body, + &write.descriptor.packs, + Kind::PackInventory, + "test index", + &mut audit, + ) + .unwrap(); + let mut forged = entries[0].1; + forged.location.pack_offset = entries[1].1.location.pack_offset; + objects[0].1 = encode_locator(forged); + let (body, objects, offsets, packs) = encode_trees(objects, offsets, packs).unwrap(); + write.descriptor.body_hash = blake3::hash(&body).to_hex().to_string(); + write.descriptor.body_bytes = body.len() as u64; + write.descriptor.objects = objects; + write.descriptor.offsets = offsets; + write.descriptor.packs = packs; + write.body = body.into(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "forged-object-index".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + let reader = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 256 * 1024, + ) + .await + .unwrap(); + let inventory = inventory + .into_iter() + .map(|pack| (pack.pack_id, pack)) + .collect(); + assert_eq!( + ( + reader.lookup(&store, &[oid(0)], &inventory).await.is_err(), + reader + .verify_complete(&store, &manifest, write.body.len() as u64) + .await + .is_err() + ), + (true, true) + ); +} + +#[test] +fn generation_zero_ref_head_snapshots_can_build_physical_indexes() { + let (mut manifest, inventory, entries) = inputs(1); + manifest.generation = 0; + let pack = inventory[0]; + manifest.pack_index_hash = crate::manifests::compact_pack_index( + 0, + &[crate::manifests::PackManifestEntry { + pack_id: pack.pack_id.to_string(), + size: pack.pack_size, + content_hash: pack.pack_id.to_string(), + ref_tips: Vec::new(), + object_count: pack.object_count, + }], + ) + .unwrap() + .0; + assert!( + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).is_ok() + ); +} diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 11a4e5835..b7be2fd69 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; #[cfg(feature = "storage")] mod cache; #[cfg(feature = "storage")] -pub(crate) use cache::PageCache; +pub(crate) use cache::{PageCache, PageWork}; use crate::error::{MetadataError, Result}; #[cfg(any(feature = "storage", test))] @@ -49,6 +49,12 @@ pub(crate) enum Kind { CommitOids, CommitAncestryRoots, CommitAncestryNodes, + #[cfg(feature = "remote-index")] + ObjectLocators, + #[cfg(feature = "remote-index")] + PackOffsets, + #[cfg(feature = "remote-index")] + PackInventory, } impl Kind { @@ -56,6 +62,12 @@ impl Kind { match self { Self::Names | Self::CommitAncestryNodes => 32, Self::CommitOids => 20, + #[cfg(feature = "remote-index")] + Self::PackInventory => 32, + #[cfg(feature = "remote-index")] + Self::ObjectLocators => 20, + #[cfg(feature = "remote-index")] + Self::PackOffsets => 40, _ => 4, } } @@ -316,7 +328,12 @@ pub(crate) fn decode( Page::Branch(children) }; input.finish()?; - tracing::debug!(target:"crab_metadata::index_decode", index_format=if matches!(kind, Kind::CommitRecords | Kind::CommitOids | Kind::CommitAncestryRoots | Kind::CommitAncestryNodes) { "commit_graph" } else { "path_state" }, + tracing::debug!(target:"crab_metadata::index_decode", index_format=match kind { + Kind::CommitRecords | Kind::CommitOids | Kind::CommitAncestryRoots | Kind::CommitAncestryNodes => "commit_graph", + #[cfg(feature = "remote-index")] + Kind::ObjectLocators | Kind::PackOffsets | Kind::PackInventory => "git_object_index", + _ => "path_state", + }, decoded_bytes=bytes.len() as u64, "index page decoded"); Ok(page) } diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs index 761a48a20..caa9fb1d2 100644 --- a/crates/crab-metadata/src/pages/cache.rs +++ b/crates/crab-metadata/src/pages/cache.rs @@ -1,7 +1,8 @@ //! Each cache belongs to one captured immutable layer directory. use super::*; use crab_storage::Store; -use std::collections::{BTreeMap, VecDeque}; +use futures_util::{StreamExt, TryStreamExt}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::{Arc, Mutex, MutexGuard}; const CACHE_BYTES: usize = 8 * 1024 * 1024; @@ -17,7 +18,95 @@ struct State { bytes: usize, } +pub(crate) struct PageWork { + pub(crate) maximum: u64, + pub(crate) bytes: u64, + pub(crate) seen: BTreeSet<(usize, u64)>, +} + impl PageCache { + #[expect( + clippy::too_many_arguments, + reason = "page lookup binds routing, origin and caller-owned aggregate admission" + )] + pub(crate) async fn lookup( + &self, + store: &Store, + path: &object_store::path::Path, + layer: usize, + source_bytes: u64, + root: &PageRef, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + if keys.iter().any(|key| key.len() != kind.key_bytes()) { + return Err(corruption("invalid lookup key")); + } + let mut results = vec![None; keys.len()]; + if keys.is_empty() { + return Ok(results); + } + let mut pending = vec![(root.clone(), (0..keys.len()).collect::>())]; + while !pending.is_empty() { + for (reference, _) in &pending { + reference.validate(kind, source_bytes)?; + if work.seen.insert((layer, reference.offset)) { + work.bytes = work + .bytes + .checked_add(u64::from(reference.bytes)) + .ok_or_else(|| corruption("query byte count overflows"))?; + if work.bytes > work.maximum { + return Err(corruption("metadata query exceeds its byte limit")); + } + } + } + // Charge the entire wave before starting I/O, including warm pages. + let pages = futures_util::stream::iter(std::mem::take(&mut pending)) + .map(|(reference, positions)| async move { + let page = self + .fetch(store, path, layer, source_bytes, &reference, kind) + .await?; + Ok::<_, MetadataError>((page, positions)) + }) + .buffer_unordered(4) + .try_collect::>() + .await?; + for (page, positions) in pages { + match page.as_ref() { + Page::Leaf(rows) => { + for position in positions { + if let Ok(index) = + rows.binary_search_by(|row| row.0.cmp(&keys[position])) + { + results[position] = Some(rows[index].1.clone()); + } + } + } + Page::Branch(children) => { + let mut batches = BTreeMap::>::new(); + for position in positions { + let index = + children.partition_point(|child| child.last < keys[position]); + if children + .get(index) + .is_some_and(|child| child.first <= keys[position]) + { + batches.entry(index).or_default().push(position); + } + } + pending.extend( + batches + .into_iter() + .map(|(index, positions)| (children[index].clone(), positions)), + ); + } + } + } + } + Ok(results) + } + // Guards protect local cache work only, never origin I/O. fn state(&self) -> Result> { self.0 diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index e98e27e13..03c778711 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -1,6 +1,5 @@ //! Lazy lookups bound to an authenticated descriptor; full scans remain maintenance operations. use crab_storage::{Store, StoreLayout}; -use futures_util::{StreamExt, TryStreamExt}; use std::{ collections::{BTreeMap, BTreeSet}, sync::Arc, @@ -10,7 +9,7 @@ use super::{ PathStateDescriptor, PathStateNodeRef, PathStateRecord, PathStateSummary, Result, codec, corrupt, corruption, validate_path, }; -use crate::pages::{self as paged, Kind, Page, PageRef}; +use crate::pages::{self as paged, Kind, Page, PageRef, PageWork}; use crate::split_commit_graph::{CommitGraphLayerRef, CommitGraphView, SplitCommitGraph}; #[path = "scan.rs"] @@ -33,12 +32,6 @@ struct ChildBatch { requests: Vec<(usize, u32)>, } -struct Work { - maximum: u64, - bytes: u64, - seen: BTreeSet<(usize, u64)>, -} - impl PathStateReader { /// Load only a descriptor and bind its positional roots to the supplied graph. pub async fn load( @@ -96,7 +89,7 @@ impl PathStateReader { .await? .remove(0) .ok_or_else(|| corruption("start commit is absent from graph"))?; - let mut work = Work { + let mut work = PageWork { maximum: self.max_bytes, bytes: self.descriptor_bytes, seen: BTreeSet::new(), @@ -138,7 +131,17 @@ impl PathStateReader { .map(|(_, name, _)| blake3::hash(name).as_bytes().to_vec()) .collect::>(); let values = self - .lookup(store, layer, names, Kind::Names, &keys, &mut work) + .cache + .lookup( + store, + &self.paths[layer], + layer, + self.descriptor.layers[layer].bytes, + names, + Kind::Names, + &keys, + &mut work, + ) .await?; for ((position, component, root), (key, value)) in requests.into_iter().zip(keys.into_iter().zip(values)) @@ -170,7 +173,17 @@ impl PathStateReader { .map(|(_, ordinal)| ordinal.to_be_bytes().to_vec()) .collect::>(); let values = self - .lookup(store, layer, &batch.root, Kind::Children, &keys, &mut work) + .cache + .lookup( + store, + &self.paths[layer], + layer, + self.descriptor.layers[layer].bytes, + &batch.root, + Kind::Children, + &keys, + &mut work, + ) .await?; for ((position, _), value) in batch.requests.into_iter().zip(values) { let value = value.ok_or_else(|| corruption("visible child is absent"))?; @@ -211,7 +224,7 @@ impl PathStateReader { store: &Store, graph: CommitGraphView<'_>, ordinals: &[u32], - work: &mut Work, + work: &mut PageWork, ) -> Result> { let mut grouped = BTreeMap::>::new(); for (position, &ordinal) in ordinals.iter().enumerate() { @@ -236,9 +249,12 @@ impl PathStateReader { .map(|(_, ordinal)| (ordinal - reference.base_ordinal).to_be_bytes().to_vec()) .collect::>(); let values = self + .cache .lookup( store, + &self.paths[layer], layer, + self.descriptor.layers[layer].bytes, &reference.pages.records, Kind::Records, &keys, @@ -274,7 +290,7 @@ impl PathStateReader { store: &Store, nodes: &[PathStateNodeRef], positions: &[usize], - work: &mut Work, + work: &mut PageWork, ) -> Result, Option)>> { let mut grouped = BTreeMap::>::new(); for (output, &position) in positions.iter().enumerate() { @@ -293,9 +309,12 @@ impl PathStateReader { .map(|(_, node)| node.index.to_be_bytes().to_vec()) .collect::>(); let values = self + .cache .lookup( store, + &self.paths[layer], layer, + self.descriptor.layers[layer].bytes, &reference.pages.nodes, Kind::Nodes, &keys, @@ -341,82 +360,6 @@ impl PathStateReader { Ok(()) } - async fn lookup( - &self, - store: &Store, - layer: usize, - root: &PageRef, - kind: Kind, - keys: &[Vec], - work: &mut Work, - ) -> Result>>> { - if keys.iter().any(|key| key.len() != kind.key_bytes()) { - return corrupt("invalid lookup key"); - } - let mut pending = vec![(root.clone(), (0..keys.len()).collect::>())]; - let mut results = vec![None; keys.len()]; - while !pending.is_empty() { - let wave = std::mem::take(&mut pending); - for (reference, _) in &wave { - self.charge(layer, reference, kind, work)?; - } - let pages = futures_util::stream::iter(wave.into_iter().map( - |(reference, positions)| async move { - let page = self.page(store, layer, &reference, kind).await?; - Ok::<_, crate::error::MetadataError>((page, positions)) - }, - )) - .buffer_unordered(16) - .try_collect::>() - .await?; - for (page, positions) in pages { - match page.as_ref() { - Page::Leaf(rows) => { - for position in positions { - if let Ok(index) = - rows.binary_search_by(|row| row.0.cmp(&keys[position])) - { - results[position] = Some(rows[index].1.clone()); - } - } - } - Page::Branch(children) => { - let mut batches = BTreeMap::>::new(); - for position in positions { - let key = &keys[position]; - let index = children.partition_point(|child| child.last < *key); - if children.get(index).is_some_and(|child| child.first <= *key) { - batches.entry(index).or_default().push(position); - } - } - pending.extend( - batches - .into_iter() - .map(|(index, positions)| (children[index].clone(), positions)), - ); - } - } - } - } - Ok(results) - } - - fn charge(&self, layer: usize, reference: &PageRef, kind: Kind, work: &mut Work) -> Result<()> { - let source_bytes = self.descriptor.layers[layer].bytes; - reference.validate(kind, source_bytes)?; - let key = (layer, reference.offset); - if work.seen.insert(key) { - work.bytes = work - .bytes - .checked_add(u64::from(reference.bytes)) - .ok_or_else(|| corruption("query byte count overflows"))?; - if work.bytes > work.maximum { - return corrupt("path-state query exceeds its byte limit"); - } - } - Ok(()) - } - async fn page( &self, store: &Store, diff --git a/crates/crab-metadata/src/path_state/scan.rs b/crates/crab-metadata/src/path_state/scan.rs index e63163607..1fc8f6c3c 100644 --- a/crates/crab-metadata/src/path_state/scan.rs +++ b/crates/crab-metadata/src/path_state/scan.rs @@ -1,4 +1,5 @@ use super::*; +use futures_util::StreamExt; /// One verified record, node or edge emitted in dependency order. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/crab-metadata/src/split_commit_graph/reader.rs b/crates/crab-metadata/src/split_commit_graph/reader.rs index 5190f2665..e9b2fa9d5 100644 --- a/crates/crab-metadata/src/split_commit_graph/reader.rs +++ b/crates/crab-metadata/src/split_commit_graph/reader.rs @@ -1,8 +1,7 @@ use super::paged::{CommitGraphLayerPages, record}; use super::*; -use crate::PageRef; -use crate::pages::{self, Kind, Page}; -use futures_util::{StreamExt, TryStreamExt}; +use crate::pages::PageWork; +use crate::pages::{self, Kind}; use std::collections::BTreeSet; /// Immutable graph point lookups; record presence does not prove ref visibility. @@ -15,11 +14,6 @@ pub struct CommitGraphReader { cache: pages::PageCache, } -struct Work { - bytes: u64, - seen: BTreeSet<(usize, u64)>, -} - /// Read-only positional access for maintenance graphs and immutable paged readers. #[derive(Clone, Copy)] pub enum CommitGraphView<'a> { @@ -98,9 +92,12 @@ impl CommitGraphReader { .ok_or_else(|| crate::validation::corrupt_object("ancestry", "root has no layer"))?; let mut work = self.work(); let values = self + .cache .lookup( store, + &self.paths[layer], layer, + self.descriptor.layers[layer].bytes, &self.roots(&self.descriptor.layers[layer])?.ancestry_roots, Kind::CommitAncestryRoots, &[root.to_be_bytes().to_vec()], @@ -160,7 +157,7 @@ impl CommitGraphReader { &self, store: &Store, references: &[ancestry::Reference], - work: &mut Work, + work: &mut PageWork, ) -> Result> { let keys = references .iter() @@ -169,9 +166,12 @@ impl CommitGraphReader { let mut output = vec![None; references.len()]; for (layer, reference) in self.descriptor.layers.iter().enumerate() { let values = self + .cache .lookup( store, + &self.paths[layer], layer, + self.descriptor.layers[layer].bytes, &self.roots(reference)?.ancestry_nodes, Kind::CommitAncestryNodes, &keys, @@ -243,9 +243,12 @@ impl CommitGraphReader { for (layer, reference) in self.descriptor.layers.iter().enumerate() { let roots = self.roots(reference)?; let values = self + .cache .lookup( store, + &self.paths[layer], layer, + self.descriptor.layers[layer].bytes, &roots.oids, Kind::CommitOids, &keys, @@ -325,8 +328,9 @@ impl CommitGraphReader { Ok(output) } - fn work(&self) -> Work { - Work { + fn work(&self) -> PageWork { + PageWork { + maximum: self.max_bytes, bytes: self.descriptor_bytes, seen: BTreeSet::new(), } @@ -343,7 +347,7 @@ impl CommitGraphReader { &self, store: &Store, ordinals: &[u32], - work: &mut Work, + work: &mut PageWork, ) -> Result>> { let mut output = vec![None; ordinals.len()]; let mut requests = BTreeMap::)>>::new(); @@ -371,7 +375,17 @@ impl CommitGraphReader { .collect::>(); let root = &self.roots(&self.descriptor.layers[layer])?.records; let values = self - .lookup(store, layer, root, Kind::CommitRecords, &keys, work) + .cache + .lookup( + store, + &self.paths[layer], + layer, + self.descriptor.layers[layer].bytes, + root, + Kind::CommitRecords, + &keys, + work, + ) .await?; for ((index, _), value) in requests.into_iter().zip(values) { let value = value.ok_or_else(|| { @@ -382,88 +396,6 @@ impl CommitGraphReader { } Ok(output) } - - async fn lookup( - &self, - store: &Store, - layer: usize, - root: &PageRef, - kind: Kind, - keys: &[Vec], - work: &mut Work, - ) -> Result>>> { - let mut output = vec![None; keys.len()]; - if keys.is_empty() { - return Ok(output); - } - let mut pending = vec![(root.clone(), (0..keys.len()).collect::>())]; - while !pending.is_empty() { - for (reference, _) in &pending { - reference.validate(kind, self.descriptor.layers[layer].bytes)?; - if work.seen.insert((layer, reference.offset)) { - work.bytes = work - .bytes - .checked_add(u64::from(reference.bytes)) - .ok_or_else(|| { - crate::validation::corrupt_object("graph", "query bytes overflow") - })?; - if work.bytes > self.max_bytes { - return corrupt("graph query exceeds byte bound"); - } - } - } - let batch = futures_util::stream::iter(std::mem::take(&mut pending)) - .map(|(reference, positions)| async move { - let page = self - .cache - .fetch( - store, - &self.paths[layer], - layer, - self.descriptor.layers[layer].bytes, - &reference, - kind, - ) - .await?; - Ok::<_, MetadataError>((page, positions)) - }) - .buffered(4) - .try_collect::>() - .await?; - for (page, positions) in batch { - match page.as_ref() { - Page::Leaf(rows) => { - for position in positions { - if let Ok(index) = - rows.binary_search_by(|(key, _)| key.cmp(&keys[position])) - { - output[position] = Some(rows[index].1.clone()) - } - } - } - Page::Branch(children) => { - let mut batches = BTreeMap::>::new(); - for position in positions { - let index = - children.partition_point(|child| child.last < keys[position]); - if children - .get(index) - .is_some_and(|child| child.first <= keys[position]) - { - batches.entry(index).or_default().push(position) - } - } - pending.extend( - batches - .into_iter() - .map(|(index, positions)| (children[index].clone(), positions)), - ); - } - } - } - } - Ok(output) - } } #[cfg(test)] #[path = "paged_tests.rs"] diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index cf8770e82..6365baba1 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -25,8 +25,8 @@ pub struct Measurements { read_bytes: AtomicU64, written_bytes: AtomicU64, active: AtomicU64, - decoded_blocks: [AtomicU64; 2], - decoded_bytes: [AtomicU64; 2], + decoded_blocks: [AtomicU64; 3], + decoded_bytes: [AtomicU64; 3], } #[derive(Clone, Copy)] @@ -35,8 +35,8 @@ struct Snapshot { outcomes: [u64; 9], read_bytes: u64, written_bytes: u64, - decoded_blocks: [u64; 2], - decoded_bytes: [u64; 2], + decoded_blocks: [u64; 3], + decoded_bytes: [u64; 3], } impl StorageObserver for Measurements { @@ -70,6 +70,7 @@ impl tracing::field::Visit for DecodeEvent { self.format = match value { "commit_graph" => Some(0), "path_state" => Some(1), + "git_object_index" => Some(2), _ => None, }; } @@ -159,7 +160,7 @@ impl Measurements { ) }) .collect::>(); - let decoded = ["commit_graph", "path_state"] + let decoded = ["commit_graph", "path_state", "git_object_index"] .into_iter() .enumerate() .map(|(index, name)| { @@ -232,13 +233,15 @@ mod tests { let counters = Arc::new(Measurements::default()); let subscriber = tracing_subscriber::registry().with(DecodeLayer(counters.clone())); tracing::subscriber::with_default(subscriber, || { + tracing::debug!(target:"crab_metadata::index_decode", index_format="commit_graph", decoded_bytes=41_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="path_state", decoded_bytes=123_u64); + tracing::debug!(target:"crab_metadata::index_decode", index_format="git_object_index", decoded_bytes=67_u64); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), - ([0, 1], [0, 123]) + ([1, 1, 1], [41, 123, 67]) ); } From d5482754882b589f219491da4f7be095bbb805f1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 21:00:02 -0700 Subject: [PATCH 015/127] fix(cli): preserve physical index error sources --- crab/src/core/error.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index bc93c2920..23a14ad9d 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -2046,6 +2046,7 @@ impl From for CrabError { | crab_metadata::error::MetadataError::CapsuleContract { .. } | crab_metadata::error::MetadataError::PathStateSize { .. } | crab_metadata::error::MetadataError::BrowseIndexRecord { .. } + | crab_metadata::error::MetadataError::GitObjectIndexRecord { .. } | crab_metadata::error::MetadataError::PlanAlreadyAttempted { .. } | crab_metadata::error::MetadataError::RefJournalCommitUncertain { .. } | crab_metadata::error::MetadataError::ManifestCommitUncertain { .. }) => { @@ -4018,6 +4019,19 @@ mod tests { assert!(source.is_panic()); } + #[test] + fn physical_index_json_source_survives_cli_conversion() { + use std::error::Error; + + let source = serde_json::from_slice::(b"{").unwrap_err(); + let error = + CrabError::from(crab_metadata::error::MetadataError::GitObjectIndexRecord { source }); + assert!( + std::iter::successors(error.source(), |source| (*source).source()) + .any(|source| source.downcast_ref::().is_some()) + ); + } + #[test] fn availability_preserves_product_diagnostics() { use std::error::Error; From 30ce5b513a2c6a6858d8d3ca6636387904ba9b8c Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 21:41:06 -0700 Subject: [PATCH 016/127] perf(remote-git): wire paged object lookup Publish verified physical pages before readiness; retain their binding through path completion and GC. Resolve OIDs and offset-delta bases through captured-state pages with operation admission and cancellation. Unshipped browse record version 3 replaces version 2; maintenance rebuilds retired records under CAS. RustFS/native-Git proof passes at 100, 1000 and 10000 commits. Storage amplification and provider/lifecycle qualification remain open. --- .../docs/architecture/indexed-remote-reads.md | 104 ++++--- crab/src/cmd/gc/mod.rs | 139 +++++++++- crates/crab-metadata/README.md | 12 +- .../src/capsule_protocol/browse.rs | 47 +++- .../src/git_object_locator/paged.rs | 34 +++ .../src/git_object_locator/paged/tests.rs | 33 +++ crates/crab-read/src/capsule_protocol.rs | 254 +++++++++++++----- crates/crab-remote-git/README.md | 6 + crates/crab-remote-git/src/reader.rs | 77 +++++- .../src/reader/object_index_tests.rs | 224 +++++++++++++++ crates/crab-remote-git/src/repository.rs | 1 + crates/crab-remote/README.md | 6 +- crates/crab-remote/examples/qualify_remote.rs | 21 +- crates/crab-remote/src/browse_indexes.rs | 67 ++++- crates/crab-write/src/generation/browse.rs | 13 +- 15 files changed, 904 insertions(+), 134 deletions(-) create mode 100644 crates/crab-remote-git/src/reader/object_index_tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 2ed692250..ca718e532 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -532,40 +532,70 @@ release/provider performance. Full pack-index reads still scale with repository objects, and control opening still scales with history. Source/binary hashes, the earlier failure, raw receipts and checked summaries remain external. -## Paged physical object lookup: implementation in progress - -The measured 10k cold-resolution query reads an 844,460-byte pack-index range to -locate its 148-byte commit entry. A new component indexes OID-to-placement, -pack-offset-to-OID and pack-inventory bindings through authenticated pages. The -reverse direction is required for offset deltas; paging only OID lookup would -leave reconstruction with a hidden full-index read. Physical aliases retain all -source offsets while primary lookup prefers a self-contained encoding. Logical -facts must agree across aliases; catalog ordinals and ref visibility remain the -existing owners' contracts. - -Graph, path-state and object lookup now share one admitted page-traversal engine. -The component run passes nine tests, including bounded 10k lookup, duplicate -positions, corruption, snapshot/inventory binding, warm encoded-work admission, -input-order canonicalization and a re-signed inconsistent-directory test. Removing -OID/offset agreement makes that regression fail; source is restored. The first -negative-control invocation selected zero tests and is not qualification; the -corrected invocation runs the complete test name and rejects the forged result. -Shared consumers pass 21 path-state and 27 graph tests, strict metadata Clippy and -payload-only compilation. The per-ref publication contract permits generation -zero after a push; the new descriptor now accepts that captured generation. -Its added regression passes alongside the other eight component tests. - -Traversal now admits four concurrent page reads for every consumer; path-state -previously admitted sixteen and graph lookup already admitted four. Correctness -tests cover the shared mechanics, but cold/warm latency must be requalified before -claiming this concurrency change is performance-neutral. Object-page decode -counts are recorded separately by the existing qualification observer. - -This component is not yet wired to foreground lookup or production readiness. -Publication must construct it from verified complete physical sources, verify -complete readback, and bind its descriptor to the captured state. Runtime OID and -physical-offset consumers must use it with operation admission and cancellation; -GC must trace its descriptor/body from retained readiness roots. Native Git and -real-store qualifications, build/storage amplification and source API release -qualification remain mandatory before declaring this slice complete. Outer -control/inventory/ref opening and stages 5–8 remain open. +## Paged physical object lookup: runtime qualification + +The measured 10k cold-resolution query previously read an 844,460-byte pack-index +range to locate its 148-byte commit. Physical lookup now uses authenticated +OID-to-placement, pack-offset-to-OID and pack-inventory pages. Offset lookup is +required for delta reconstruction; paging only OIDs would retain a hidden full +reverse-index read. Every physical alias remains represented. Primary placement +prefers a self-contained encoding; logical facts agree across aliases while +encoding-specific delta-base facts remain attached to their own placement. + +The capsule owner constructs complete physical coverage from authenticated source +sidecars, admits encoded output before upload, and verifies full readback before +publishing readiness. Path completion retains that descriptor hash. Maintenance +can repair missing or corrupt derived bytes; provider/authentication failures +propagate. Foreground readers bind the descriptor to captured state, transport and +storage prefixes, use operation-admitted stores, and return definitive misses +without probing whole indexes. Catalog ordinals and visible-ref authority remain +with their existing owners. GC retains the descriptor and its immutable page +body; stale descriptors can be collected while a shared current body survives. + +Browse-readiness version 3 replaces the unshipped version 2 record. No release tag +contains the introducing browse-record commit. Normal readers reject retired +records; the elected maintenance owner rebuilds them under the observed CAS token. +The released manifest protocol and its object-catalog lookup remain unchanged. + +Focused proof includes 10 metadata component tests, five runtime regressions, +three browse-record tests, two publication-fence tests, the CLI error-source test +and the GC sweep regression. The offset-delta runtime fixture contains no IDX/REV +sidecars and reconstructs exact verified bytes. Removing OID/offset agreement +fails the forged-directory regression; the initial zero-test negative-control run +is excluded. Existing shared consumers pass 21 path-state and 27 graph tests; the +qualification observer passes two tests. Strict Clippy passes for the four touched +orchestration/read crates with `--no-deps`. The broader invocation fails six +pre-existing coordination lint expectations unchanged from the PR base. CLI test +compilation, HTTP/auth-server/SDK checks and both qualification-tool builds pass. +A strict CLI-wide invocation reports 605 lint errors; JSON diagnostics contain no +primary error span on modified CLI lines compared with the stacked base. This is +not a clean CLI-wide lint result. The missing CLI conversion for the new error +variant was found by wider build and CI proof, then corrected with source-chain +coverage. + +The local RustFS run completes 121 native-Git query processes and three maintenance +passes. Each fixed resolve/attribution cell contains 20 cold and 20 warm samples; +the 100-commit fixture additionally checks all ten supported qualification queries. +Every measured operation finishes with zero active backend operations. Source and +binary hashes, native oracles, request traces and checked receipts remain external. + +| History | Cold resolve bytes | Cold attribution bytes | Resolve p95, ms | Attribution p95, ms | Physical body bytes | +| --- | ---: | ---: | ---: | ---: | ---: | +| 100 | 46,175 | 103,428 | 10.205 | 104.636 | 91,522 | +| 1,000 | 68,117 | 483,280 | 12.756 | 110.685 | 676,936 | +| 10,000 | 87,518 | 392,549 | 72.424 | 145.501 | 6,530,830 | + +Warm fixed queries read zero backend bytes. At 10k, cold resolution falls from +844,608 to 87,518 bytes and attribution from 985,949 to 392,549 bytes. The fixed +queries stay below the frozen 512 KiB decoded / 1 MiB transferred limits. These are +shared-host debug timings on local RustFS, not representative S3/release latency. +The traversal change from sixteen to four concurrent path-page reads is included +in these measured fixed-directory cells; other workloads remain unqualified. + +Maintenance takes 2.345 / 5.080 / 22.581 seconds and peaks at 34.5 / 53.7 / 245.6 MiB +process RSS in these fixtures. The 10k physical body is 7.73 times the old pack-index +range. That storage amplification remains an optimization gate: dictionary/row +compaction and batch-request behavior must be measured before calling the design +fully efficient. Outer control/ref/inventory opening, the complete stage-4 edge +matrix, old-reader lifecycle, representative provider and source-API release gates +remain open. Stages 5–8 and the separate unified-staging workstream are not complete. diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index ff87d48cc..00304d8d4 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3250,9 +3250,11 @@ async fn sweep_capsule_objects( let history_prefix = layout.repo_path("v2/history/"); let path_state_prefix = layout.repo_path("metadata/path-state/"); let commit_graph_prefix = layout.repo_path("metadata/commit-graph/"); + let object_index_prefix = layout.repo_path("metadata/git-object-index/"); let manifests_prefix = layout.repo_path("manifests/"); let path_state_descriptor_prefix = layout.repo_path("manifests/path-state-"); let commit_graph_descriptor_prefix = layout.repo_path("manifests/commit-graph-"); + let object_index_descriptor_prefix = layout.repo_path("manifests/git-object-index-"); let ( capsules, checkpoints, @@ -3260,6 +3262,7 @@ async fn sweep_capsule_objects( history, path_state_objects, commit_graph_objects, + object_index_objects, manifests, ) = tokio::try_join!( store.list_prefix(&capsule_prefix), @@ -3268,15 +3271,18 @@ async fn sweep_capsule_objects( store.list_prefix(&history_prefix), store.list_prefix(&path_state_prefix), store.list_prefix(&commit_graph_prefix), + store.list_prefix(&object_index_prefix), store.list_prefix(&manifests_prefix), )?; let derived_index_objects = path_state_objects .into_iter() .chain(commit_graph_objects) + .chain(object_index_objects) .chain(manifests.into_iter().filter(|object| { let key = object.location.as_ref(); key.starts_with(path_state_descriptor_prefix.as_ref()) || key.starts_with(commit_graph_descriptor_prefix.as_ref()) + || key.starts_with(object_index_descriptor_prefix.as_ref()) })) .collect::>(); let derived_index_keys = derived_index_objects @@ -3402,6 +3408,26 @@ async fn mark_derived_index_objects( .map(|layer| layout.repo_path(&layer.path).to_string()), ); + if let Some(hash) = indexes.object_index_hash() { + let index = crab_metadata::git_object_locator::PagedGitObjectIndex::load( + store, + layout, + hash, + state_digest, + manifest, + crab_metadata::split_commit_graph::DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, + ) + .await?; + // Every authenticated physical page lives in this immutable body. + // Retain it with readiness so pruning cannot strand future delta reads. + reachable.insert( + layout + .bulk_manifest_path("git-object-index", hash) + .to_string(), + ); + reachable.insert(index.body_path().to_string()); + } + if let Some(path_state_hash) = indexes.path_state_hash() { let path_state_path = layout.bulk_manifest_path("path-state", path_state_hash); let path_state = crab_metadata::path_state::load_path_state_descriptor( @@ -5681,6 +5707,10 @@ mod tests { #[tokio::test] async fn capsule_protocol_gc_retains_current_indexes_and_collects_stale_index_generations() { use crab_metadata::capsule_protocol::BrowseIndexes; + use crab_metadata::git_object_locator::{ + GitObjectLocation, GitObjectLocator, GitPackInventoryEntry, PagedGitObjectIndex, + PagedGitObjectIndexWrite, + }; use crab_metadata::path_state::publish_path_state_checkpoint; use object_store::memory::InMemory; use std::sync::Arc; @@ -5698,9 +5728,61 @@ mod tests { .unwrap(); let mut manifest = crab_metadata::manifests::Manifest::default_for_repo("refs/heads/main"); manifest.generation = root.record().root().generation(); - manifest.pack_index_hash = "1".repeat(64); + let pack_id = crab_xet::hash::MerkleHash::from([8; 32]); + let inventory = GitPackInventoryEntry { + pack_id, + object_count: 1, + pack_size: 42, + }; + let pack_index_hash = |generation| { + crab_metadata::manifests::compact_pack_index( + generation, + &[crab_metadata::manifests::PackManifestEntry { + pack_id: pack_id.to_string(), + content_hash: pack_id.to_string(), + size: 42, + object_count: 1, + ref_tips: Vec::new(), + }], + ) + .unwrap() + .0 + }; + manifest.pack_index_hash = pack_index_hash(manifest.generation); manifest.seal_git_validation(); let state_digest = "a".repeat(64); + let physical_entries = [( + [7; 20], + GitObjectLocator { + ordinal: 0, + pack_id, + location: GitObjectLocation { + pack_offset: 12, + entry_len: 10, + crc32: 0, + }, + metadata: Default::default(), + }, + )]; + let physical = PagedGitObjectIndexWrite::build( + &state_digest, + &manifest, + &[inventory], + &physical_entries, + ) + .unwrap(); + let physical_hash = physical.upload(store.as_storage(), &layout).await.unwrap(); + let physical_reader = PagedGitObjectIndex::load( + store.as_storage(), + &layout, + &physical_hash, + &state_digest, + &manifest, + 128 * 1024, + ) + .await + .unwrap(); + let physical_body = physical_reader.body_path().clone(); let (current_graph_hash, current_path_state_hash, current_graph, current_count) = write_browse_indexes( store.as_storage(), @@ -5726,6 +5808,8 @@ mod tests { for path_state in [None, Some(current_path_state_hash.clone())] { let indexes = BrowseIndexes::new(state_digest.clone(), current_graph_hash.clone(), path_state) + .unwrap() + .with_object_index(physical_hash.clone()) .unwrap(); let index_path = layout.capsule_browse_indexes_path(); let bytes = indexes.encode().unwrap(); @@ -5760,6 +5844,34 @@ mod tests { } let stale_git_validation_digest = "f".repeat(64); + let mut stale_manifest = manifest.clone(); + stale_manifest.generation += 1; + stale_manifest.pack_index_hash = pack_index_hash(stale_manifest.generation); + stale_manifest.git_validation_digest = stale_git_validation_digest.clone(); + let stale_physical = PagedGitObjectIndexWrite::build( + &"f".repeat(64), + &stale_manifest, + &[inventory], + &physical_entries, + ) + .unwrap(); + let stale_physical_hash = stale_physical + .upload(store.as_storage(), &layout) + .await + .unwrap(); + let stale_physical_reader = PagedGitObjectIndex::load( + store.as_storage(), + &layout, + &stale_physical_hash, + &"f".repeat(64), + &stale_manifest, + 128 * 1024, + ) + .await + .unwrap(); + // Rebinding unchanged physical entries shares the immutable body; an + // obsolete descriptor must not make that current body's pages collectible. + assert_eq!(stale_physical_reader.body_path(), &physical_body); let (stale_graph_hash, stale_path_state_hash, stale_graph, stale_count) = write_browse_indexes( store.as_storage(), @@ -5816,6 +5928,8 @@ mod tests { &crab_metadata::path_state::path_state_checkpoint_path(&stale_git_validation_digest), ); let current_indexes = [ + layout.bulk_manifest_path("git-object-index", &physical_hash), + physical_body, current_graph_descriptor.clone(), current_graph_layer.clone(), current_path_state_descriptor.clone(), @@ -5823,6 +5937,7 @@ mod tests { current_checkpoint.clone(), ]; let stale_indexes = [ + layout.bulk_manifest_path("git-object-index", &stale_physical_hash), stale_graph_descriptor.clone(), stale_graph_layer.clone(), stale_path_state_descriptor.clone(), @@ -5895,6 +6010,26 @@ mod tests { .await .unwrap(); assert_eq!(attribution[0].oid, [3; 20]); + // Open afresh after sweeping: a cached page would conceal accidental deletion. + let physical_reader = PagedGitObjectIndex::load( + store.as_storage(), + &layout, + &physical_hash, + &state_digest, + &manifest, + 128 * 1024, + ) + .await + .unwrap(); + let locations = physical_reader + .lookup( + store.as_storage(), + &[[7; 20]], + &std::collections::HashMap::from([(pack_id, inventory)]), + ) + .await + .unwrap(); + assert_eq!(locations, vec![Some(physical_entries[0].1)]); for path in stale_indexes { let head = store.head(&path).await; assert!( @@ -5903,7 +6038,7 @@ mod tests { ); } assert_eq!(outcome.packs_deleted, 0); - assert_eq!(outcome.derived_index_objects_deleted, 5); + assert_eq!(outcome.derived_index_objects_deleted, 6); } #[tokio::test] diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index c34376dbe..a4ccf3a40 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -47,15 +47,17 @@ loading multi-ref activation records. Restore can leave retired heads in storage their checkpoint positions and dependencies must not enter the restored catalog. Current-epoch activation and checkpoint-chain errors still fail closed. `v2/browse-indexes` is a bounded, mutable derived record, not ref authority. -Its version 2 record binds a durable immutable commit graph and optional complete -path attribution to the exact capsule state, including visible per-ref -transactions. Graph publication does not wait for path construction. The prior -version was never reachable from a release tag; maintenance rebuilds this derived +Its version 3 record binds a durable immutable commit graph, optional complete +physical object lookup and optional complete path attribution to the exact +capsule state, including visible per-ref transactions. Physical lookup readiness +requires complete verified pack coverage before publication; path completion +retains that binding. Graph publication does not wait for path construction. The prior +versions were never reachable from a release tag; maintenance rebuilds this derived record in place, without a legacy reader. Readers ignore stale records; absent indexes remain a readiness state. Rebuilding an immutable index verifies the generated content identity and repairs corrupt stored bytes only with a conditional update against the observed version, followed by readback. -This applies to graph/path-state descriptors and layers, not Git/Xet data. +This applies to graph/path-state/physical-index descriptors and bodies, not Git/Xet data. Graph/path-state loaders admit descriptor bodies against the caller's byte budget before buffering, then admit each layer against its authenticated size and the remaining aggregate budget. All bodies still require hash and exact diff --git a/crates/crab-metadata/src/capsule_protocol/browse.rs b/crates/crab-metadata/src/capsule_protocol/browse.rs index a05336c29..475986a01 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse.rs @@ -16,6 +16,8 @@ pub struct BrowseIndexes { commit_graph_hash: String, #[serde(skip_serializing_if = "Option::is_none")] path_state_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + object_index_hash: Option, } impl BrowseIndexes { @@ -26,10 +28,11 @@ impl BrowseIndexes { path_state_hash: Option, ) -> Result { let value = Self { - version: 2, + version: 3, state_digest, commit_graph_hash, path_state_hash, + object_index_hash: None, }; value.validate()?; Ok(value) @@ -53,6 +56,19 @@ impl BrowseIndexes { self.path_state_hash.as_deref() } + /// Return the complete physical object-index descriptor for this captured state. + #[must_use] + pub fn object_index_hash(&self) -> Option<&str> { + self.object_index_hash.as_deref() + } + + /// Attach a durable, completely verified physical index before publishing readiness. + pub fn with_object_index(mut self, hash: String) -> Result { + self.object_index_hash = Some(hash); + self.validate()?; + Ok(self) + } + /// Encode a bounded record after every named index is durable. pub fn encode(&self) -> Result { self.validate()?; @@ -76,7 +92,7 @@ impl BrowseIndexes { } fn validate(&self) -> Result<()> { - if self.version != 2 { + if self.version != 3 { return Err(corrupt_object( "browse indexes", "unsupported record version", @@ -91,6 +107,9 @@ impl BrowseIndexes { if let Some(hash) = &self.path_state_hash { validate_content_hash(hash, "path state", "browse indexes")?; } + if let Some(hash) = &self.object_index_hash { + validate_content_hash(hash, "object index", "browse indexes")?; + } Ok(()) } } @@ -121,7 +140,7 @@ mod tests { #[test] fn graph_ready_record_round_trips_without_complete_path_attribution() { let encoded = serde_json::json!({ - "version": 2, + "version": 3, "state_digest": "1".repeat(64), "commit_graph_hash": "2".repeat(64), }); @@ -134,8 +153,10 @@ mod tests { #[test] fn browse_indexes_round_trip_exact_state_and_complete_descriptors() { - let record = - BrowseIndexes::new("1".repeat(64), "2".repeat(64), Some("3".repeat(64))).unwrap(); + let record = BrowseIndexes::new("1".repeat(64), "2".repeat(64), Some("3".repeat(64))) + .unwrap() + .with_object_index("4".repeat(64)) + .unwrap(); assert_eq!( BrowseIndexes::decode(&record.encode().unwrap()).unwrap(), record @@ -156,12 +177,16 @@ mod tests { malformed[field] = serde_json::json!("../invalid"); assert!(BrowseIndexes::decode(&serde_json::to_vec(&malformed).unwrap()).is_err()); } - let mut malformed_path = valid.clone(); - malformed_path["path_state_hash"] = serde_json::json!("../invalid"); - assert!(BrowseIndexes::decode(&serde_json::to_vec(&malformed_path).unwrap()).is_err()); - let mut unsupported = valid; - unsupported["version"] = serde_json::json!(1); - assert!(BrowseIndexes::decode(&serde_json::to_vec(&unsupported).unwrap()).is_err()); + for field in ["path_state_hash", "object_index_hash"] { + let mut malformed = valid.clone(); + malformed[field] = serde_json::json!("../invalid"); + assert!(BrowseIndexes::decode(&serde_json::to_vec(&malformed).unwrap()).is_err()); + } + for version in [1, 2] { + let mut unsupported = valid.clone(); + unsupported["version"] = serde_json::json!(version); + assert!(BrowseIndexes::decode(&serde_json::to_vec(&unsupported).unwrap()).is_err()); + } assert!(BrowseIndexes::decode(&vec![b' '; MAX_BROWSE_INDEXES_BYTES as usize + 1]).is_err()); } } diff --git a/crates/crab-metadata/src/git_object_locator/paged.rs b/crates/crab-metadata/src/git_object_locator/paged.rs index ba62ee041..b61fa76aa 100644 --- a/crates/crab-metadata/src/git_object_locator/paged.rs +++ b/crates/crab-metadata/src/git_object_locator/paged.rs @@ -1,5 +1,6 @@ //! Immutable physical placement indexes; callers retain ref visibility and origin verification. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; use bytes::Bytes; use crab_storage::{Store, StoreLayout}; @@ -41,11 +42,19 @@ pub struct PagedGitObjectIndex { descriptor: Descriptor, descriptor_bytes: u64, path: object_store::path::Path, + origin: Arc, + global_prefix: String, maximum: u64, cache: PageCache, } impl PagedGitObjectIndexWrite { + /// Return encoded physical-page bytes for maintenance admission before upload. + #[must_use] + pub fn body_bytes(&self) -> u64 { + self.descriptor.body_bytes + } + /// Build from every verified packed entry in the captured capsule inventory. /// /// Entries cover each pack without holes; this index does not assign catalog ordinals. @@ -253,11 +262,36 @@ impl PagedGitObjectIndex { descriptor, descriptor_bytes: bytes.len() as u64, path, + origin: Arc::clone(layout.store().inner()), + global_prefix: layout.global_prefix().to_owned(), maximum, cache: PageCache::default(), }) } + /// Check captured state and in-process origin before attaching lookup to a repository. + #[must_use] + pub fn matches_binding( + &self, + layout: &StoreLayout, + state_digest: &str, + manifest: &Manifest, + ) -> bool { + Arc::ptr_eq(&self.origin, layout.store().inner()) + && self.global_prefix == layout.global_prefix() + && self.path == body_path(layout, &self.descriptor.body_hash) + && self.descriptor.state_digest == state_digest + && self.descriptor.generation == manifest.generation + && self.descriptor.pack_index_hash == manifest.pack_index_hash + && self.descriptor.git_validation_digest == manifest.git_validation_digest + } + + /// Return the immutable body containing every page for retained-root GC. + #[must_use] + pub fn body_path(&self) -> &object_store::path::Path { + &self.path + } + /// Resolve OIDs against the captured physical inventory, preserving duplicates and misses. /// /// The caller admits request/output counts and supplies an admitted store for the captured origin. diff --git a/crates/crab-metadata/src/git_object_locator/paged/tests.rs b/crates/crab-metadata/src/git_object_locator/paged/tests.rs index 5f8e288d6..dec24e9fe 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/tests.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/tests.rs @@ -390,3 +390,36 @@ fn generation_zero_ref_head_snapshots_can_build_physical_indexes() { PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).is_ok() ); } + +#[tokio::test] +async fn physical_index_attachment_binds_origin_prefixes_and_captured_state() { + let (manifest, inventory, entries) = inputs(10); + let state = "c".repeat(64); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "bound-origin".to_owned()); + let write = PagedGitObjectIndexWrite::build(&state, &manifest, &inventory, &entries).unwrap(); + let hash = write.upload(&store, &layout).await.unwrap(); + let reader = PagedGitObjectIndex::load(&store, &layout, &hash, &state, &manifest, 128 * 1024) + .await + .unwrap(); + let foreign = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "bound-origin".to_owned(), + ); + let prefix = StoreLayout::new(store.clone(), "other-repository".to_owned()); + let global = StoreLayout::with_global_prefix( + store, + "bound-origin".to_owned(), + "other-global".to_owned(), + ); + assert_eq!( + [ + reader.matches_binding(&layout, &state, &manifest), + reader.matches_binding(&foreign, &state, &manifest), + reader.matches_binding(&prefix, &state, &manifest), + reader.matches_binding(&global, &state, &manifest), + reader.matches_binding(&layout, &"d".repeat(64), &manifest) + ], + [true, false, false, false, false] + ); +} diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 94663cf95..4d44449b2 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -607,6 +607,83 @@ impl CapsuleRepositoryView { self } + /// Construct complete physical lookup pages from the captured source inventory. + /// + /// The caller holds generation/GC ownership and validates full readback before readiness. + pub async fn build_object_index( + &self, + layout: &StoreLayout, + max_bytes: u64, + cancel: &CancellationToken, + ) -> Result { + let sources = self + .layered_checkpoint() + .into_iter() + .flat_map(|checkpoint| checkpoint.sources()) + .chain(self.capsule_run_sources.iter()); + let mut seen = BTreeMap::new(); + let mut members = Vec::new(); + let mut inventory = Vec::new(); + for source in sources { + let source_path = match source.kind() { + PackSourceKind::CapsuleRun => layout.capsule_path(source.object_hash()), + PackSourceKind::PackLayer => layout.capsule_pack_layer_path(source.object_hash()), + }; + for member in source.members() { + let pack_id = crab_xet::hash::MerkleHash::from_hex(member.pack().blake3()) + .map_err(|error| corrupt_path("capsule Git pack", error.to_string()))?; + if let Some(previous) = seen.insert(pack_id, member.clone()) { + if !previous.has_same_content(member) { + return Err(corrupt_path( + "capsule Git pack", + "sources disagree about one pack identity", + )); + } + continue; + } + inventory.push(crab_metadata::git_object_locator::GitPackInventoryEntry { + pack_id, + object_count: member.object_count(), + pack_size: member.pack().length(), + }); + members.push(LayeredMemberRead { + source_path: source_path.clone(), + source_size: source.object_size(), + source_bytes: None, + pack_id, + member: member.clone(), + }); + } + } + let entries = physical_locators_for_members(&members, layout, max_bytes, cancel).await?; + let snapshot = self.git_snapshot()?; + let worker_cancel = cancel.clone(); + let write = tokio::task::spawn_blocking(move || { + if worker_cancel.is_cancelled() { + return Err(ReadError::Cancelled); + } + crab_metadata::git_object_locator::PagedGitObjectIndexWrite::build( + &snapshot.manifest_etag, + &snapshot.manifest, + &inventory, + &entries, + ) + .map_err(Into::into) + }) + .await + .map_err(|error| ReadError::Io(std::io::Error::other(error)))??; + if cancel.is_cancelled() { + return Err(ReadError::Cancelled); + } + if max_bytes > 0 && write.body_bytes() > max_bytes { + return Err(ReadError::CapsuleReadLimit { + resource: "physical Git index output", + maximum: max_bytes, + }); + } + Ok(write) + } + /// Capture the exact Git state for derived indexes without reading or writing v1 metadata. /// /// The synthetic ETag binds visible per-ref positions, not only the root. @@ -813,6 +890,22 @@ impl CapsuleRepositoryView { if cancellation.is_cancelled() { return Err(ReadError::Cancelled); } + let snapshot = self.git_snapshot()?; + let object_index = match self + .browse_indexes + .as_ref() + .and_then(|indexes| indexes.object_index_hash()) + { + Some(hash) => Some(tokio::select! { + biased; + () = cancellation.cancelled() => return Err(ReadError::Cancelled), + result = crab_metadata::git_object_locator::PagedGitObjectIndex::load( + layout.store(), &layout, hash, &snapshot.manifest_etag, &snapshot.manifest, + options.object_limits().max_pack_index_bytes, + ) => result?, + }), + None => None, + }; let checkpoint = self.layered_checkpoint(); if checkpoint.is_none() && max_input_bytes > 0 && self.git_pack_bytes()? > max_input_bytes { return Err(ReadError::CapsuleReadLimit { @@ -966,70 +1059,14 @@ impl CapsuleRepositoryView { all_members.push(member_read); } } - if checkpoint.is_some_and(|value| !value.is_control_only()) && !all_members.is_empty() { - // A complete layered checkpoint authenticates a kind-bearing - // locator range for every member. Load those compact sidecars as - // one coalesced window per source so filtered planning can use - // OID/kind locators instead of issuing one object read per OID. - let payload_members = all_members - .iter() - .cloned() - .map(|member| LayeredPayloadMemberRead { - member, - complete_local: false, - }) - .collect::>(); - let selected_members = (0..payload_members.len()).collect::>(); - let windows = plan_layered_payload_windows_for_members( - &payload_members, - &selected_members, - LayeredPayloadWindowMode::Sidecars, - )?; - let window_bytes = windows.iter().try_fold(0_u64, |total, window| { - total - .checked_add(window.range.end.saturating_sub(window.range.start)) - .ok_or_else(|| corrupt_path("capsule Git locator", "sidecar bytes overflowed")) - })?; - if max_input_bytes > 0 && window_bytes > max_input_bytes { - return Err(ReadError::CapsuleReadLimit { - resource: "layered Git locator sidecars", - maximum: max_input_bytes, - }); - } - let (fetched_windows, _) = - fetch_layered_payload_windows(layout.store(), &windows).await?; - for (member_index, member_read) in all_members.iter().enumerate() { - let window_index = payload_window_for_member(&windows, member_index)?; - let body = fetched_windows - .get(&window_index) - .ok_or_else(|| corrupt_path("capsule Git locator", "sidecar window missing"))?; - let window = windows - .get(window_index) - .ok_or_else(|| corrupt_path("capsule Git locator", "sidecar window missing"))?; - let index = - layered_range_bytes(body, window.range.start, member_read.member.index())?; - let reverse_index = layered_range_bytes( - body, - window.range.start, - member_read.member.reverse_index(), - )?; - let locator = - layered_range_bytes(body, window.range.start, member_read.member.locator())?; - let pack_id = member_read.pack_id; - let index_path = workspace.path().join(format!("{pack_id}-layered.idx")); - let reverse_path = workspace.path().join(format!("{pack_id}-layered.rev")); - std::fs::write(&index_path, &index)?; - std::fs::write(&reverse_path, &reverse_index)?; - inline_locators.extend(inline_locators_for_pack( - member_read.member.object_count(), - member_read.member.git_checksum(), - pack_id, - &index_path, - &reverse_path, - &locator, - member_read.member.pack().length(), - )?); - } + if object_index.is_none() + && checkpoint.is_some_and(|value| !value.is_control_only()) + && !all_members.is_empty() + { + inline_locators.extend( + physical_locators_for_members(&all_members, &layout, max_input_bytes, cancellation) + .await?, + ); } // Complete bounded frontiers reuse authenticated resident bytes; // checkpoint and larger-frontier sources stay lazy. Captured runs may @@ -1153,11 +1190,13 @@ impl CapsuleRepositoryView { ); } } - let snapshot = self.git_snapshot()?; let mut lookup_sources = crab_remote_git::SnapshotLookupSources::default() .with_inline_locators(inline_locators) .with_pack_sources(pack_sources) .with_preferred_pack_indexes(preferred_pack_indexes); + if let Some(index) = object_index { + lookup_sources = lookup_sources.with_object_index(index); + } if !preferred_object_admission_by_oid.is_empty() { lookup_sources = lookup_sources.with_preferred_object_admission(preferred_object_admission_by_oid); @@ -3700,6 +3739,91 @@ fn layered_range_bytes( Ok(body.slice(start_usize..end_usize)) } +async fn physical_locators_for_members( + members: &[LayeredMemberRead], + layout: &StoreLayout, + max_bytes: u64, + cancel: &CancellationToken, +) -> Result< + Vec<( + [u8; 20], + crab_metadata::git_object_locator::GitObjectLocator, + )>, +> { + let payload_members = members + .iter() + .cloned() + .map(|member| LayeredPayloadMemberRead { + member, + complete_local: false, + }) + .collect::>(); + let selected = (0..members.len()).collect::>(); + let windows = plan_layered_payload_windows_for_members( + &payload_members, + &selected, + LayeredPayloadWindowMode::Sidecars, + )?; + let bytes = windows.iter().try_fold(0_u64, |total, window| { + total + .checked_add(window.range.end - window.range.start) + .ok_or_else(|| corrupt_path("capsule Git locator", "sidecar bytes overflowed")) + })?; + if max_bytes > 0 && bytes > max_bytes { + return Err(ReadError::CapsuleReadLimit { + resource: "layered Git locator sidecars", + maximum: max_bytes, + }); + } + let (fetched, _) = tokio::select! { + biased; + () = cancel.cancelled() => return Err(ReadError::Cancelled), + result = fetch_layered_payload_windows(layout.store(), &windows) => result?, + }; + let members = members.to_vec(); + let cancel = cancel.clone(); + // The worker owns its fetched bytes and scratch files through parsing. + // Cancellation cannot release caller scratch paths while gix still maps them. + tokio::task::spawn_blocking(move || { + let workspace = tempfile::tempdir()?; + let mut entries = Vec::new(); + for (position, member) in members.iter().enumerate() { + if cancel.is_cancelled() { + return Err(ReadError::Cancelled); + } + let window_index = payload_window_for_member(&windows, position)?; + let window = &windows[window_index]; + let body = fetched + .get(&window_index) + .ok_or_else(|| corrupt_path("capsule Git locator", "sidecar window missing"))?; + let index = layered_range_bytes(body, window.range.start, member.member.index())?; + let reverse = + layered_range_bytes(body, window.range.start, member.member.reverse_index())?; + let locator = layered_range_bytes(body, window.range.start, member.member.locator())?; + let index_path = workspace + .path() + .join(format!("{}-layered.idx", member.pack_id)); + let reverse_path = workspace + .path() + .join(format!("{}-layered.rev", member.pack_id)); + std::fs::write(&index_path, &index)?; + std::fs::write(&reverse_path, &reverse)?; + entries.extend(inline_locators_for_pack( + member.member.object_count(), + member.member.git_checksum(), + member.pack_id, + &index_path, + &reverse_path, + &locator, + member.member.pack().length(), + )?); + } + Ok(entries) + }) + .await + .map_err(|error| ReadError::Io(std::io::Error::other(error)))? +} + fn inline_locators_for_pack( object_count: u64, git_checksum: &str, diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index 719d238f2..bf693af45 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -31,6 +31,12 @@ identity, integrity and admission checks; a journal ref change discards stale base indexes even when its generation is unchanged. Snapshots without indexes make no additional index requests. Publication of capsule-bound browse indexes is a separate maintenance responsibility, not part of opening or pushing. +Captured physical indexes resolve both OIDs and offset-delta bases through +authenticated pages. Snapshot opening checks their state, origin and prefixes; +queries retain storage admission, encoded-work limits and cancellation. Complete +index misses are definitive. Physical placement never assigns catalog ordinals +or authorizes ref visibility; ordinary unindexed snapshots retain their existing +pack-index inspection path until background readiness is published. Capsule graphs retain an immutable paged reader and captured commit roots; membership reads exact ancestor sets rather than allocating a full reachability bitmap. A full commit ID matching a verified captured root reuses that visibility diff --git a/crates/crab-remote-git/src/reader.rs b/crates/crab-remote-git/src/reader.rs index bd0c3697b..953196e46 100644 --- a/crates/crab-remote-git/src/reader.rs +++ b/crates/crab-remote-git/src/reader.rs @@ -8,7 +8,7 @@ use bytes::Bytes; use crab_git::{delta, pack::VerifiedPackIdentity}; use crab_metadata::git_object_locator::{ GitObjectLocation, GitObjectLocator, GitObjectLocatorSession, GitObjectLookup, - GitPackInventoryEntry, + GitPackInventoryEntry, PagedGitObjectIndex, }; use crab_storage::{ Store, repo_pack_index_path, repo_pack_kind_metadata_path, repo_pack_path, @@ -162,10 +162,12 @@ pub(crate) struct RemoteGitPackedEntry { /// Authenticated lookup data and immutable pack sources for a pinned snapshot. /// /// Callers must validate these against the snapshot before opening a repository. -/// Inline locators precede preferred indexes; remaining lookups use the complete -/// pinned inventory. An empty preferred set is distinct from an unspecified set. +/// A complete physical index is definitive; unindexed snapshots use inline locators +/// before preferred indexes and the complete pinned inventory. An empty preferred +/// set is distinct from an unspecified set. #[derive(Default)] pub struct SnapshotLookupSources { + object_index: Option>, preferred_pack_indexes: Option>, preferred_object_admission: Option>>>, inline_locators: Option>>, @@ -173,6 +175,31 @@ pub struct SnapshotLookupSources { } impl SnapshotLookupSources { + /// Use a completely verified captured physical index; misses do not probe pack indexes. + pub fn with_object_index(mut self, index: PagedGitObjectIndex) -> Self { + self.object_index = Some(Arc::new(index)); + self + } + + pub(crate) fn validate_object_index( + &self, + layout: &crab_storage::StoreLayout, + snapshot: &crab_metadata::manifest_store::RepositorySnapshot, + ) -> Result<()> { + if self.object_index.as_ref().is_some_and(|index| { + !index.matches_binding( + layout, + &snapshot.manifest_etag, + &snapshot.materialized_manifest(), + ) + }) { + return Err(Error::RepositoryState { + reason: RepositoryStateError::InconsistentGeneration, + }); + } + Ok(()) + } + /// Use verified in-memory locators before reading pack indexes. pub fn with_inline_locators(mut self, locators: HashMap<[u8; 20], GitObjectLocator>) -> Self { self.inline_locators = Some(Arc::new(locators)); @@ -432,6 +459,7 @@ pub(crate) struct RemoteGitReader { store: Store, repo_prefix: String, inventory: HashMap, + object_index: Option>, preferred_pack_indexes: Option>, preferred_object_admission: Option>>>, inline_locators: Option>>, @@ -515,6 +543,7 @@ impl RemoteGitReader { store, repo_prefix: repo_prefix.into(), inventory: canonical, + object_index: lookup_sources.object_index, preferred_pack_indexes, preferred_object_admission: lookup_sources.preferred_object_admission, inline_locators: lookup_sources.inline_locators, @@ -797,6 +826,23 @@ impl RemoteGitReader { budget: &OperationBudget, cancellation: &CancellationToken, ) -> Result> { + if let Some(index) = &self.object_index { + let store = self + .store + .clone() + .with_read_admission(budget.read_admission(cancellation.clone())); + let locations = tokio::select! { + biased; + () = cancellation.cancelled() => return Err(Error::Cancelled), + result = index.lookup(&store, requested, &self.inventory) => result?, + }; + // Complete coverage was verified before readiness. A miss is definitive; + // probing whole indexes here would hide corrupt or incomplete publication. + return Ok(locations + .into_iter() + .map(|location| location.map_or(GitObjectLookup::Miss, GitObjectLookup::Hit)) + .collect()); + } if let Some(locators) = &self.inline_locators { tracing::debug!(object_count = requested.len(), "remote Git locator lookup"); let mut lookups = requested @@ -2072,6 +2118,27 @@ impl RemoteGitReader { budget: &OperationBudget, cancellation: &CancellationToken, ) -> Result { + if let Some(index) = &self.object_index { + let store = self + .store + .clone() + .with_read_admission(budget.read_admission(cancellation.clone())); + let offsets = [(pack_id, pack_offset)]; + let oids = tokio::select! { + biased; + () = cancellation.cancelled() => return Err(Error::Cancelled), + result = index.oids_at_offsets(&store, &offsets, &self.inventory) => result?, + }; + return oids + .into_iter() + .next() + .flatten() + .map(gix_hash::ObjectId::from) + .ok_or(Error::DeltaBaseNotFound { + pack_id, + pack_offset, + }); + } let index = self.load_pack_index(pack_id, budget, cancellation).await?; index .oid_at_offset(pack_offset) @@ -3950,6 +4017,10 @@ fn usize_limit(value: u64, limit: &'static str) -> Result { }) } +#[cfg(test)] +#[path = "reader/object_index_tests.rs"] +mod object_index_tests; + #[cfg(test)] mod tests { use crab_metadata::git_object_locator::{ diff --git a/crates/crab-remote-git/src/reader/object_index_tests.rs b/crates/crab-remote-git/src/reader/object_index_tests.rs new file mode 100644 index 000000000..a90840add --- /dev/null +++ b/crates/crab-remote-git/src/reader/object_index_tests.rs @@ -0,0 +1,224 @@ +use super::*; +use crab_metadata::git_object_locator::{PagedGitObjectIndex, PagedGitObjectIndexWrite}; +use crab_metadata::manifests::{Manifest, PackManifestEntry, compact_pack_index}; +use crab_storage::StoreLayout; +use object_store::{ObjectStoreExt, memory::InMemory}; +use std::io::Write as _; + +async fn fixture() -> (Arc, gix_hash::ObjectId) { + let encode = |header: Header, data: &[u8]| { + let mut bytes = Vec::new(); + header.write_to(data.len() as u64, &mut bytes).unwrap(); + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + bytes.extend(encoder.finish().unwrap()); + bytes + }; + let oid = |data: &[u8]| { + gix_object::compute_hash(gix_hash::Kind::Sha1, gix_object::Kind::Blob, data).unwrap() + }; + let base = encode(Header::Blob, b"hello world"); + let delta = encode( + Header::OfsDelta { + base_distance: base.len() as u64, + }, + &[0x0b, 0x0c, 0x90, 0x0b, 0x01, b'!'], + ); + let mut pack = b"PACK\0\0\0\x02\0\0\0\x02".to_vec(); + pack.extend_from_slice(&base); + pack.extend_from_slice(&delta); + let checksum = sha1::Sha1::digest(&pack); + pack.extend_from_slice(&checksum); + let pack_id = MerkleHash::from_hex(blake3::hash(&pack).to_hex().as_str()).unwrap(); + let inventory = GitPackInventoryEntry { + pack_id, + object_count: 2, + pack_size: pack.len() as u64, + }; + let target = oid(b"hello world!"); + let entries = [ + (oid(b"hello world"), &base, 12), + (target, &delta, 12 + base.len() as u64), + ] + .map(|(oid, bytes, offset)| { + ( + oid.as_bytes().try_into().unwrap(), + GitObjectLocator { + ordinal: 0, + pack_id, + location: GitObjectLocation { + pack_offset: offset, + entry_len: bytes.len() as u64, + crc32: gix_features::hash::crc32(bytes), + }, + metadata: Default::default(), + }, + ) + }); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "repository".to_owned()); + store + .put(&repo_pack_path("repository", &pack_id), Bytes::from(pack)) + .await + .unwrap(); + let mut manifest = Manifest::default_for_repo("refs/heads/main"); + manifest.generation = 1; + manifest.git_validation_digest = "b".repeat(64); + manifest.pack_index_hash = compact_pack_index( + 1, + &[PackManifestEntry { + pack_id: pack_id.to_string(), + size: inventory.pack_size, + content_hash: pack_id.to_string(), + ref_tips: Vec::new(), + object_count: 2, + }], + ) + .unwrap() + .0; + let write = PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &[inventory], &entries) + .unwrap(); + let hash = write.upload(&store, &layout).await.unwrap(); + let index = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 128 * 1024, + ) + .await + .unwrap(); + index + .verify_complete(&store, &manifest, 128 * 1024) + .await + .unwrap(); + let runtime = Arc::new(RemoteGitRuntime::default()); + let reader = RemoteGitReader::from_pinned_with_preferred_pack_indexes( + store, + "repository", + [inventory], + SnapshotLookupSources::default().with_object_index(index), + ReaderLimits::default(), + runtime, + RepositoryIdentity::new("provider", "repository", 1).unwrap(), + 1, + ) + .unwrap(); + (Arc::new(reader), target) +} + +#[tokio::test] +async fn paged_lookup_reconstructs_offset_deltas_without_pack_sidecars() { + let (reader, target) = fixture().await; + let budget = OperationBudget::new(crate::OperationLimits::default(), reader.runtime.clone()); + // Only the pack and paged index exist: any hidden IDX/REV lookup fails. + let result = reader + .read_with_session( + &GitObjectLocatorSession::without_catalog(), + target, + &budget, + &CancellationToken::new(), + ) + .await; + reader.runtime.shutdown().await; + assert_eq!(result.unwrap().data.as_ref(), b"hello world!"); +} + +#[tokio::test] +async fn paged_lookup_misses_are_definitive_without_pack_sidecars() { + let (reader, _) = fixture().await; + let budget = OperationBudget::new(crate::OperationLimits::default(), reader.runtime.clone()); + let result = reader + .lookup_batch_for_read( + &GitObjectLocatorSession::without_catalog(), + &[[9; 20], [9; 20]], + &budget, + &CancellationToken::new(), + ) + .await; + reader.runtime.shutdown().await; + assert_eq!( + result.unwrap(), + vec![GitObjectLookup::Miss, GitObjectLookup::Miss] + ); +} + +#[tokio::test] +async fn corrupt_physical_pages_are_errors_without_pack_index_fallback() { + let (reader, target) = fixture().await; + let path = reader.object_index.as_ref().unwrap().body_path(); + let mut bytes = reader.store.get_with_etag(path).await.unwrap().0.to_vec(); + bytes[14] ^= 1; + // Bypass immutable Store writes to inject corruption at the provider boundary. + reader + .store + .inner() + .put(path, Bytes::from(bytes).into()) + .await + .unwrap(); + let budget = OperationBudget::new(crate::OperationLimits::default(), reader.runtime.clone()); + let result = reader + .lookup_batch_for_read( + &GitObjectLocatorSession::without_catalog(), + &[target.as_bytes().try_into().unwrap()], + &budget, + &CancellationToken::new(), + ) + .await; + reader.runtime.shutdown().await; + assert!(matches!(result, Err(Error::Metadata(_)))); +} + +#[tokio::test] +async fn cancelled_physical_lookup_does_not_touch_origin() { + let (reader, target) = fixture().await; + let budget = OperationBudget::new(crate::OperationLimits::default(), reader.runtime.clone()); + let cancel = CancellationToken::new(); + cancel.cancel(); + let result = reader + .lookup_batch_for_read( + &GitObjectLocatorSession::without_catalog(), + &[target.as_bytes().try_into().unwrap()], + &budget, + &cancel, + ) + .await; + let requests = budget + .usage() + .await + .amount(BudgetDimension::StorageRequests); + reader.runtime.shutdown().await; + assert!(matches!(result, Err(Error::Cancelled)) && requests == 0); +} + +#[tokio::test] +async fn physical_pages_obey_the_operations_storage_budget() { + let (reader, target) = fixture().await; + let budget = OperationBudget::new( + crate::OperationLimits { + max_storage_requests: 1, + ..Default::default() + }, + reader.runtime.clone(), + ); + budget + .charge(BudgetDimension::StorageRequests, 1) + .await + .unwrap(); + let result = reader + .lookup_batch_for_read( + &GitObjectLocatorSession::without_catalog(), + &[target.as_bytes().try_into().unwrap()], + &budget, + &CancellationToken::new(), + ) + .await; + reader.runtime.shutdown().await; + let Err(Error::Metadata(crab_metadata::error::MetadataError::Storage { source })) = result + else { + panic!("expected a source-preserving storage admission failure"); + }; + assert!(crab_storage::read_rejection(&source).is_some()); +} diff --git a/crates/crab-remote-git/src/repository.rs b/crates/crab-remote-git/src/repository.rs index 5f25f28a3..65b6927ad 100644 --- a/crates/crab-remote-git/src/repository.rs +++ b/crates/crab-remote-git/src/repository.rs @@ -421,6 +421,7 @@ impl RemoteGitRepository { RepositoryOptions::new(options.object_limits(), options.operation_limits())?; check_cancelled(cancellation)?; check_cancelled(&runtime.background_cancellation())?; + lookup_sources.validate_object_index(&layout, snapshot)?; let entries = snapshot .journal .packs diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index e415713bd..49923e50f 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -4,7 +4,11 @@ snapshot under renewed generation ownership and GC fences, reuses the shared Git index builders, and publishes `v2/browse-indexes` only after checking a fresh root and visible ref positions. Conditional record publication cannot overwrite -a newer maintenance result; readers independently reject a stale binding. +a newer maintenance result; readers independently reject a stale binding. Physical +OID and pack-offset pages are built from authenticated complete source sidecars, +verified in full before readiness, and carried through final path publication. +The elected owner repairs missing/corrupt derived physical pages; foreground +reads propagate those failures rather than probing whole pack indexes. Ordinary push/clone/fetch do not load this derived record. Internal orchestration shared by Crab SDK, CLI, HTTP, and managed-service diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index aaf158a66..f929fd9c3 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -84,11 +84,24 @@ async fn main() -> Result<()> { .ok_or_else(|| { io::Error::other("index build was superseded or repository was empty") })?; - if complete.path_state_hash().is_none() { - return Err( - io::Error::other("path attribution was not completely published").into(), - ); + if complete.path_state_hash().is_none() || complete.object_index_hash().is_none() { + return Err(io::Error::other( + "physical lookup and path attribution were not completely published", + ) + .into()); } + let hash = complete.object_index_hash().ok_or_else(|| io::Error::other("physical index readiness is absent"))?; + let root = crab_metadata::capsule_protocol::load_root(&layout).await?; + let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( + &layout, root, + crab_read::capsule_protocol::CapsuleReadLimits { max_capsule_bytes: 2 * 1024 * 1024 * 1024, max_frontier_bytes: 2 * 1024 * 1024 * 1024 }, + ).await?; + let snapshot = view.git_snapshot()?; + let index = crab_metadata::git_object_locator::PagedGitObjectIndex::load( + layout.store(), &layout, hash, &snapshot.manifest_etag, &snapshot.manifest, + options.object_limits().max_pack_index_bytes, + ).await?; + println!("{}", json!({"kind":"physical-index", "descriptor_hash":hash, "body_bytes":layout.store().head(index.body_path()).await?.size})); Ok(()) }) .await; diff --git a/crates/crab-remote/src/browse_indexes.rs b/crates/crab-remote/src/browse_indexes.rs index 8f6d1ff5f..ebf1fa4f2 100644 --- a/crates/crab-remote/src/browse_indexes.rs +++ b/crates/crab-remote/src/browse_indexes.rs @@ -25,7 +25,7 @@ pub enum Error { Coordination(#[from] crab_coordination::CoordinationError), } -/// Publish graph readiness, then complete attribution, for one captured state. +/// Publish graph and physical readiness, then complete attribution, for one captured state. /// /// This is opt-in background work, never a prerequisite for ref publication. A /// superseded snapshot is not published; readers independently reject stale records. @@ -60,13 +60,13 @@ pub async fn ensure( .git_repository_from_store( layout.clone(), identity.clone(), - runtime, + runtime.clone(), options, max_bytes, cancel, ) .await?; - let indexes = crab_write::generation::browse::build_graph( + let mut indexes = crab_write::generation::browse::build_graph( layout, &repository, &snapshot, @@ -75,6 +75,28 @@ pub async fn ensure( cancel, ) .await?; + let physical_ready = match indexes.object_index_hash() { + Some(hash) => verify_object_index(layout, &snapshot, hash, max_bytes, options, cancel).await?, + None => false, + }; + if !physical_ready { + let write = view.build_object_index(layout, max_bytes, cancel).await?; + let hash = tokio::select! { + biased; + () = cancel.cancelled() => return Err(crab_read::ReadError::Cancelled.into()), + result = write.upload(layout.store(), layout) => result?, + }; + let reader = crab_metadata::git_object_locator::PagedGitObjectIndex::load( + layout.store(), layout, &hash, &snapshot.manifest_etag, &snapshot.manifest, + options.object_limits().max_pack_index_bytes, + ).await?; + tokio::select! { + biased; + () = cancel.cancelled() => return Err(crab_read::ReadError::Cancelled.into()), + result = reader.verify_complete(layout.store(), &snapshot.manifest, max_bytes) => result?, + } + indexes = indexes.with_object_index(hash)?; + } if previous.as_ref() == Some(&indexes) && indexes.path_state_hash().is_some() { ready = Some(indexes); return Ok(()); @@ -87,6 +109,11 @@ pub async fn ensure( None => return Ok(()), } }; + // Bind the just-verified physical pages to this same view before path + // construction. Reopening a fresh root could mix index and ref identities. + let repository = view.with_browse_indexes(Some(indexes.clone())) + .git_repository_from_store(layout.clone(), identity.clone(), runtime, options, max_bytes, cancel) + .await?; let complete = crab_write::generation::browse::build_path_state( layout, &repository, @@ -135,6 +162,40 @@ async fn publish_current( .map_err(Into::into) } +async fn verify_object_index( + layout: &StoreLayout, + snapshot: &crab_metadata::manifest_store::RepositorySnapshot, + hash: &str, + max_bytes: u64, + options: RepositoryOptions, + cancel: &CancellationToken, +) -> Result { + let result = tokio::select! { + biased; + () = cancel.cancelled() => return Err(crab_read::ReadError::Cancelled.into()), + result = async { + let index = crab_metadata::git_object_locator::PagedGitObjectIndex::load( + layout.store(), layout, hash, &snapshot.manifest_etag, &snapshot.manifest, + options.object_limits().max_pack_index_bytes, + ).await?; + index.verify_complete(layout.store(), &snapshot.manifest, max_bytes).await + } => result, + }; + // Only the elected maintenance owner repairs derived corruption. Foreground + // lookup propagates these errors; provider/auth failures never trigger rebuilds. + match result { + Ok(()) => Ok(true), + Err( + crab_metadata::error::MetadataError::CorruptObject { .. } + | crab_metadata::error::MetadataError::GitObjectIndexRecord { .. } + | crab_metadata::error::MetadataError::Storage { + source: StorageError::NotFound { .. } | StorageError::CorruptObject { .. }, + }, + ) => Ok(false), + Err(error) => Err(error.into()), + } +} + async fn previous_record( layout: &StoreLayout, ) -> Result<(Option, Option), Error> { diff --git a/crates/crab-write/src/generation/browse.rs b/crates/crab-write/src/generation/browse.rs index ded8b9443..115e0bcb1 100644 --- a/crates/crab-write/src/generation/browse.rs +++ b/crates/crab-write/src/generation/browse.rs @@ -167,12 +167,19 @@ pub async fn build_path_state( let write = append_path_state(base, &graph, inputs)?; upload_path_state(store, layout, &write, limits.max_path_state_bytes).await?; if write.commit_count() == graph.descriptor.commit_count { - return BrowseIndexes::new( + let complete = BrowseIndexes::new( snapshot.manifest_etag.clone(), graph_hash.to_owned(), Some(write.descriptor_hash), - ) - .map_err(Into::into); + )?; + // Path completion must retain already verified physical readiness; + // replacing the record would otherwise send reads back to whole indexes. + return match indexes.object_index_hash() { + Some(hash) => complete + .with_object_index(hash.to_owned()) + .map_err(Into::into), + None => Ok(complete), + }; } publish_path_state_checkpoint( store, From 4711a5ce1905000e259014b0991cc96cd5f95d0d Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 22:29:54 -0700 Subject: [PATCH 017/127] perf(metadata): compact physical index rows Replace unreleased descriptor v1 with authenticated pack-slot directories and presence-aware metadata. Maintenance rebuilds retired descriptors. Record storage savings and unresolved shared-host latency regression. --- .../docs/architecture/indexed-remote-reads.md | 81 +++ .../src/git_object_locator/paged.rs | 510 +++--------------- .../src/git_object_locator/paged/codec.rs | 186 +++++++ .../src/git_object_locator/paged/reader.rs | 444 +++++++++++++++ .../src/git_object_locator/paged/tests.rs | 204 ++++++- crates/crab-metadata/src/pages.rs | 54 +- 6 files changed, 994 insertions(+), 485 deletions(-) create mode 100644 crates/crab-metadata/src/git_object_locator/paged/codec.rs create mode 100644 crates/crab-metadata/src/git_object_locator/paged/reader.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index ca718e532..49266a77a 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -599,3 +599,84 @@ compaction and batch-request behavior must be measured before calling the design fully efficient. Outer control/ref/inventory opening, the complete stage-4 edge matrix, old-reader lifecycle, representative provider and source-API release gates remain open. Stages 5–8 and the separate unified-staging workstream are not complete. + +## Compact physical-directory format + +Unreleased physical descriptor version 2 replaces version 1. OID rows contain a +pack-slot/offset pointer and logical facts. Packed-entry ranges, CRCs and delta +base facts occur only in physical offset rows. Metadata stores presence flags and +only fields that are present; known zero sizes and all-zero base IDs retain their +presence, rather than becoming missing facts. The existing metadata codec validates +flags and object-kind codes after compact decoding. + +| Directory | Key bytes | Value | +| --- | ---: | --- | +| OID → placement | 20 | slot (4), offset (8), present logical facts | +| Physical offset → entry | 12 | OID (20), length (8), CRC (4), present physical metadata | +| Pack ID → slot | 32 | slot (4), count (8), size (8) | +| Slot → pack | 4 | pack ID (32), count (8), size (8) | + +Slots are canonical in encoded pack-hash order. Both dictionary directions are +authenticated and joined against the caller's pinned inventory; queries do not +scan the entire dictionary to translate a pack ID. Slot pages use the shared dense +positional coverage checks. Full verification reconstructs all four directories +and compares canonical descriptor and body bytes. Point reads cross-check OID +pointers with the physical entry's OID; offset reads also verify the primary +placement and compatible logical facts without moving delta-base facts between +aliases. All nested reads share one encoded-work budget. + +Normal readers reject retired physical descriptors; the elected maintenance +owner rebuilds them into version 2 before publishing the new hash. No release tag +contains the version-1 implementation. Readiness remains version 3; graph, path, +released catalog formats and reference authorization retain their existing rules. +The reader and row codec now have separate modules to keep construction, I/O and +wire validation inspectable. Fifteen component tests cover compact-field presence, +truncation/reserved flags, dictionary disagreement, pack/entry permutations and +retired-format rejection alongside the original coverage and binding tests. The +exact forged-pointer negative control fails when OID agreement is removed and +passes after source restoration. The five runtime, 21 path, 27 graph and shared-body +GC regressions pass; metadata Clippy and payload/storage feature checks pass. + +Compact-format RustFS proof completes 301 native-Git query processes and three +maintenance passes. Resolve, attribution, object-point, object-batch and blob +cells each have 20 cold and 20 warm samples at 100/1,000/10,000 commits. Warm queries +read zero backend bytes. Maintenance recognizes old physical descriptors and +rebuilds current readiness before these query comparisons. Earlier timing and +storage receipts above describe the prior format. + +| History | Prior body bytes | Compact body bytes | Reduction | +| --- | ---: | ---: | ---: | +| 100 | 91,522 | 37,216 | 59.34% | +| 1,000 | 676,936 | 275,663 | 59.28% | +| 10,000 | 6,530,830 | 2,658,423 | 59.29% | + +The 10k body is now 3.15 times the old 844,460-byte Git index, down from 7.73 times. +Cold fixed-query traffic is 37,365 / 66,534 / 72,285 bytes for resolve and +61,880 / 273,003 / 377,318 bytes for attribution. Compact dictionaries introduce +an extra authenticated join on larger fixtures: resolve uses 5 / 7 / 7 requests; +attribution uses 61 / 50 / 26. These costs remain part of provider/batch qualification, +not a claim that every workload improves. Construction, storage and layout changes +add verification responsibilities for both dictionary directions; they do not add +new configuration or runtime dependencies. + +The correctness run overlapped a local CLI build. An overlapping timing-only attempt +is explicitly excluded; final timing comes from a later run without those tasks. +Source/binary hashes, checked summaries, migration results and traces remain +external. Opening, the full edge/lifecycle matrix, representative S3/release proof +and subsequent initiative stages remain open. + +The separate timing run completes 120 additional native-Git comparisons with 20 +samples per cold/warm cell. Its cold p95 measurements are: + +| History | Resolve p95, ms | Attribution p95, ms | +| --- | ---: | ---: | +| 100 | 15.634 | 297.608 | +| 1,000 | 15.245 | 509.871 | +| 10,000 | 123.526 | 182.518 | + +These timings regress against the prior-format local run despite lower transfer +and storage. In particular, the 1k attribution p95 rises from 110.685 to 509.871 ms. +The two runs are on a shared host rather than a controlled paired release build; +the cause is not established. Repeated authenticated dictionary joins and page +traversal need profiling, followed by controlled provider/release qualification. +This is an unresolved performance gate; smaller index bytes do not close it. diff --git a/crates/crab-metadata/src/git_object_locator/paged.rs b/crates/crab-metadata/src/git_object_locator/paged.rs index b61fa76aa..2ae2eb883 100644 --- a/crates/crab-metadata/src/git_object_locator/paged.rs +++ b/crates/crab-metadata/src/git_object_locator/paged.rs @@ -13,6 +13,10 @@ use crate::manifests::Manifest; use crate::pages::{Kind, PageCache, PageRef, PageWork, Rows, Writer}; use crate::validation::{corrupt_object, validate_content_hash}; +mod codec; +mod reader; +use codec::*; + const MAX_DESCRIPTOR_BYTES: u64 = 16 * 1024; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -28,6 +32,7 @@ struct Descriptor { objects: PageRef, offsets: PageRef, packs: PageRef, + slots: PageRef, } /// Canonical construction of OID locations, physical offsets and complete pack coverage. @@ -163,29 +168,46 @@ impl PagedGitObjectIndexWrite { return bad("physical entries do not cover the declared pack"); } } + // Pack hashes occur once per dictionary, not in every object and offset row. + // Slot assignment is canonical by encoded hash bytes, independent of input order. + let mut ordered_packs = packs.values().copied().collect::>(); + ordered_packs.sort_unstable_by_key(|pack| <[u8; 32]>::from(pack.pack_id)); + let mut slot_by_pack = BTreeMap::new(); + let mut pack_rows = Vec::new(); + let mut slot_rows = Vec::new(); + for (position, pack) in ordered_packs.into_iter().enumerate() { + let slot = u32::try_from(position).map_err(|_| corruption("pack slots overflow"))?; + slot_by_pack.insert(pack.pack_id, slot); + pack_rows.push(( + <[u8; 32]>::from(pack.pack_id).to_vec(), + encode_binding(slot, pack), + )); + slot_rows.push((slot.to_be_bytes().to_vec(), encode_pack(pack))); + } let object_rows = objects .into_iter() - .map(|(oid, locator)| (oid.to_vec(), encode_locator(locator))) - .collect(); + .map(|(oid, locator)| { + let slot = slot_by_pack + .get(&locator.pack_id) + .ok_or_else(|| corruption("primary pack has no slot"))?; + Ok((oid.to_vec(), encode_pointer(*slot, locator))) + }) + .collect::>()?; let offset_rows = offsets .into_iter() .map(|((pack_id, offset), (oid, locator))| { + let slot = slot_by_pack + .get(&pack_id) + .ok_or_else(|| corruption("physical pack has no slot"))?; let mut value = oid.to_vec(); value.extend_from_slice(&encode_location(locator.location, locator.metadata)); - (offset_key(pack_id, offset), value) + Ok((offset_key(*slot, offset), value)) }) - .collect(); - let pack_rows = packs - .into_iter() - .map(|(pack_id, pack)| { - let mut value = pack.object_count.to_be_bytes().to_vec(); - value.extend_from_slice(&pack.pack_size.to_be_bytes()); - (<[u8; 32]>::from(pack_id).to_vec(), value) - }) - .collect(); - let (body, objects, offsets, packs) = encode_trees(object_rows, offset_rows, pack_rows)?; + .collect::>()?; + let (body, [objects, offsets, packs, slots]) = + encode_trees(object_rows, offset_rows, pack_rows, slot_rows)?; let descriptor = Descriptor { - version: 1, + version: 2, state_digest: state_digest.to_owned(), generation: manifest.generation, pack_index_hash: manifest.pack_index_hash.clone(), @@ -195,6 +217,7 @@ impl PagedGitObjectIndexWrite { objects, offsets, packs, + slots, }; descriptor.validate()?; Ok(Self { @@ -226,381 +249,9 @@ impl PagedGitObjectIndexWrite { } } -impl PagedGitObjectIndex { - /// Capture an authenticated descriptor without loading its physical inventory or locations. - /// - /// Callers accept readiness only after complete verification; placement never authorizes refs. - pub async fn load( - store: &Store, - layout: &StoreLayout, - hash: &str, - state_digest: &str, - manifest: &Manifest, - maximum: u64, - ) -> Result { - validate_content_hash(hash, "object index descriptor", "Git object index")?; - let expected = hash_bytes(hash)?; - let bytes = store - .verify_bounded( - &layout.bulk_manifest_path("git-object-index", hash), - &expected, - maximum.min(MAX_DESCRIPTOR_BYTES), - ) - .await?; - let descriptor: Descriptor = serde_json::from_slice(&bytes) - .map_err(|source| crate::error::MetadataError::GitObjectIndexRecord { source })?; - descriptor.validate()?; - if descriptor.state_digest != state_digest - || descriptor.generation != manifest.generation - || descriptor.pack_index_hash != manifest.pack_index_hash - || descriptor.git_validation_digest != manifest.git_validation_digest - { - return bad("object index does not match captured state"); - } - let path = body_path(layout, &descriptor.body_hash); - Ok(Self { - descriptor, - descriptor_bytes: bytes.len() as u64, - path, - origin: Arc::clone(layout.store().inner()), - global_prefix: layout.global_prefix().to_owned(), - maximum, - cache: PageCache::default(), - }) - } - - /// Check captured state and in-process origin before attaching lookup to a repository. - #[must_use] - pub fn matches_binding( - &self, - layout: &StoreLayout, - state_digest: &str, - manifest: &Manifest, - ) -> bool { - Arc::ptr_eq(&self.origin, layout.store().inner()) - && self.global_prefix == layout.global_prefix() - && self.path == body_path(layout, &self.descriptor.body_hash) - && self.descriptor.state_digest == state_digest - && self.descriptor.generation == manifest.generation - && self.descriptor.pack_index_hash == manifest.pack_index_hash - && self.descriptor.git_validation_digest == manifest.git_validation_digest - } - - /// Return the immutable body containing every page for retained-root GC. - #[must_use] - pub fn body_path(&self) -> &object_store::path::Path { - &self.path - } - - /// Resolve OIDs against the captured physical inventory, preserving duplicates and misses. - /// - /// The caller admits request/output counts and supplies an admitted store for the captured origin. - pub async fn lookup( - &self, - store: &Store, - oids: &[[u8; 20]], - inventory: &HashMap, - ) -> Result>> { - let mut work = self.work(); - let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); - let values = self - .lookup_rows( - store, - &self.descriptor.objects, - Kind::ObjectLocators, - &keys, - &mut work, - ) - .await?; - let output = values - .into_iter() - .map(|value| value.map(|value| decode_locator(&value)).transpose()) - .collect::>>()?; - let locators = output.iter().flatten().copied().collect::>(); - let offset_keys = locators - .iter() - .map(|locator| offset_key(locator.pack_id, locator.location.pack_offset)) - .collect::>(); - let physical = self - .lookup_rows( - store, - &self.descriptor.offsets, - Kind::PackOffsets, - &offset_keys, - &mut work, - ) - .await?; - let mut physical = physical.into_iter(); - for (oid, locator) in oids.iter().zip(&output) { - let Some(locator) = locator else { - continue; - }; - let row = physical - .next() - .flatten() - .ok_or_else(|| corruption("object location has no physical offset"))?; - if row.len() != 64 || array::<20>(&row, 0)? != *oid { - return bad("object and physical offset identities differ"); - } - let (location, metadata) = decode_location(locator.location.pack_offset, &row[20..])?; - if location != locator.location - || metadata.delta_base_oid != locator.metadata.delta_base_oid - || metadata - .kind - .zip(locator.metadata.kind) - .is_some_and(|(left, right)| left != right) - || metadata - .logical_size - .zip(locator.metadata.logical_size) - .is_some_and(|(left, right)| left != right) - { - return bad("object and physical offset facts differ"); - } - } - self.validate_inventory(store, &locators, inventory, &mut work) - .await?; - Ok(output) - } - - /// Resolve physical delta-base offsets without reading an entire reverse pack index. - /// - /// The caller admits request/output counts and supplies an admitted store for the captured origin. - pub async fn oids_at_offsets( - &self, - store: &Store, - offsets: &[(MerkleHash, u64)], - inventory: &HashMap, - ) -> Result>> { - let mut work = self.work(); - let keys = offsets - .iter() - .map(|(pack, offset)| offset_key(*pack, *offset)) - .collect::>(); - let values = self - .lookup_rows( - store, - &self.descriptor.offsets, - Kind::PackOffsets, - &keys, - &mut work, - ) - .await?; - let mut locators = Vec::new(); - let mut output = Vec::with_capacity(values.len()); - for ((pack_id, offset), value) in offsets.iter().zip(values) { - let Some(value) = value else { - output.push(None); - continue; - }; - if value.len() != 64 { - return bad("invalid offset row length"); - } - let oid = array(&value, 0)?; - let (location, metadata) = decode_location(*offset, &value[20..])?; - locators.push(GitObjectLocator { - ordinal: 0, - pack_id: *pack_id, - location, - metadata, - }); - output.push(Some(oid)); - } - self.validate_inventory(store, &locators, inventory, &mut work) - .await?; - let oid_keys = output - .iter() - .flatten() - .map(|oid| oid.to_vec()) - .collect::>(); - let primary = self - .lookup_rows( - store, - &self.descriptor.objects, - Kind::ObjectLocators, - &oid_keys, - &mut work, - ) - .await?; - if primary.iter().any(Option::is_none) { - return bad("physical offset has no object identity"); - } - let primary = primary - .into_iter() - .flatten() - .map(|value| decode_locator(&value)) - .collect::>>()?; - self.validate_inventory(store, &primary, inventory, &mut work) - .await?; - Ok(output) - } - - /// Verify complete coverage and canonical bytes before publishing readiness. - pub async fn verify_complete( - &self, - store: &Store, - manifest: &Manifest, - maximum: u64, - ) -> Result<()> { - let bytes = store - .verify_bounded( - &self.path, - &hash_bytes(&self.descriptor.body_hash)?, - maximum, - ) - .await?; - if bytes.len() as u64 != self.descriptor.body_bytes { - return bad("object index body length mismatch"); - } - let mut audit = BTreeMap::new(); - crate::pages::scan( - &bytes, - &self.descriptor.objects, - Kind::ObjectLocators, - self.path.as_ref(), - &mut audit, - )?; - let packs = crate::pages::scan( - &bytes, - &self.descriptor.packs, - Kind::PackInventory, - self.path.as_ref(), - &mut audit, - )?; - let inventory = packs - .into_iter() - .map(|(key, value)| { - if value.len() != 16 { - return bad("invalid inventory row length"); - } - Ok(GitPackInventoryEntry { - pack_id: MerkleHash::from(array(&key, 0)?), - object_count: u64::from_be_bytes(array(&value, 0)?), - pack_size: u64::from_be_bytes(array(&value, 8)?), - }) - }) - .collect::>>()?; - let offsets = crate::pages::scan( - &bytes, - &self.descriptor.offsets, - Kind::PackOffsets, - self.path.as_ref(), - &mut audit, - )?; - let entries = offsets - .into_iter() - .map(|(key, value)| { - if value.len() != 64 { - return bad("invalid offset row length"); - } - let offset = u64::from_be_bytes(array(&key, 32)?); - let (location, metadata) = decode_location(offset, &value[20..])?; - Ok(( - array(&value, 0)?, - GitObjectLocator { - ordinal: 0, - pack_id: MerkleHash::from(array(&key, 0)?), - location, - metadata, - }, - )) - }) - .collect::>>()?; - let rebuilt = PagedGitObjectIndexWrite::build( - &self.descriptor.state_digest, - manifest, - &inventory, - &entries, - )?; - if rebuilt.descriptor != self.descriptor || rebuilt.body.as_ref() != bytes.as_ref() { - return bad("object index coverage or canonical placement mismatch"); - } - Ok(()) - } - - fn work(&self) -> PageWork { - PageWork { - maximum: self.maximum, - bytes: self.descriptor_bytes, - seen: BTreeSet::new(), - } - } - - async fn lookup_rows( - &self, - store: &Store, - root: &PageRef, - kind: Kind, - keys: &[Vec], - work: &mut PageWork, - ) -> Result>>> { - self.cache - .lookup( - store, - &self.path, - 0, - self.descriptor.body_bytes, - root, - kind, - keys, - work, - ) - .await - } - - async fn validate_inventory( - &self, - store: &Store, - locators: &[GitObjectLocator], - inventory: &HashMap, - work: &mut PageWork, - ) -> Result<()> { - let pack_ids = locators - .iter() - .map(|locator| locator.pack_id) - .collect::>() - .into_iter() - .collect::>(); - let keys = pack_ids - .iter() - .map(|pack| <[u8; 32]>::from(*pack).to_vec()) - .collect::>(); - let values = self - .lookup_rows( - store, - &self.descriptor.packs, - Kind::PackInventory, - &keys, - work, - ) - .await?; - for (pack_id, value) in pack_ids.into_iter().zip(values) { - let value = value.ok_or_else(|| corruption("required pack binding is absent"))?; - if value.len() != 16 { - return bad("invalid inventory row length"); - } - let expected = inventory - .get(&pack_id) - .ok_or_else(|| corruption("pack is outside captured inventory"))?; - if u64::from_be_bytes(array(&value, 0)?) != expected.object_count - || u64::from_be_bytes(array(&value, 8)?) != expected.pack_size - { - return bad("pack binding differs from captured inventory"); - } - } - for locator in locators { - if !inventory.get(&locator.pack_id).is_some_and(|pack| { - format::validate_location_for_pack(locator.location, pack.pack_size) - }) { - return bad("location escapes captured pack"); - } - } - Ok(()) - } -} - impl Descriptor { fn validate(&self) -> Result<()> { - if self.version != 1 { + if self.version != 2 { return bad("unsupported object index descriptor"); } for hash in [ @@ -615,10 +266,16 @@ impl Descriptor { (&self.objects, Kind::ObjectLocators), (&self.offsets, Kind::PackOffsets), (&self.packs, Kind::PackInventory), + (&self.slots, Kind::PackSlots), ] { root.validate(kind, self.body_bytes)?; } - if self.objects.count > self.offsets.count || self.packs.count > self.offsets.count { + if self.objects.count > self.offsets.count + || self.packs.count > self.offsets.count + || self.packs.count != self.slots.count + || self.slots.first != 0u32.to_be_bytes() + || self.slots.last != (self.slots.count - 1).to_be_bytes() + { return bad("object index counts are inconsistent"); } Ok(()) @@ -629,15 +286,26 @@ fn encode_trees( mut objects: Rows, mut offsets: Rows, mut packs: Rows, -) -> Result<(Vec, PageRef, PageRef, PageRef)> { - for rows in [&mut objects, &mut offsets, &mut packs] { + mut slots: Rows, +) -> Result<(Vec, [PageRef; 4])> { + for rows in [&mut objects, &mut offsets, &mut packs, &mut slots] { rows.sort_unstable_by(|left, right| left.0.cmp(&right.0)); } let mut builder = Writer::default(); - let objects = builder.tree(Kind::ObjectLocators, objects)?; - let offsets = builder.tree(Kind::PackOffsets, offsets)?; - let packs = builder.tree(Kind::PackInventory, packs)?; - Ok((builder.bytes, objects, offsets, packs)) + let roots = [ + builder.tree(Kind::ObjectLocators, objects)?, + builder.tree(Kind::PackOffsets, offsets)?, + builder.tree(Kind::PackInventory, packs)?, + builder.tree(Kind::PackSlots, slots)?, + ]; + Ok((builder.bytes, roots)) +} +fn logical_conflict(left: super::GitObjectMetadata, right: super::GitObjectMetadata) -> bool { + left.kind.zip(right.kind).is_some_and(|(a, b)| a != b) + || left + .logical_size + .zip(right.logical_size) + .is_some_and(|(a, b)| a != b) } fn placement_rank(locator: GitObjectLocator) -> (bool, MerkleHash, u64) { ( @@ -646,60 +314,6 @@ fn placement_rank(locator: GitObjectLocator) -> (bool, MerkleHash, u64) { locator.location.pack_offset, ) } -fn offset_key(pack: MerkleHash, offset: u64) -> Vec { - let mut key = <[u8; 32]>::from(pack).to_vec(); - key.extend_from_slice(&offset.to_be_bytes()); - key -} -fn encode_location(location: GitObjectLocation, metadata: super::GitObjectMetadata) -> Vec { - let mut value = location.entry_len.to_be_bytes().to_vec(); - value.extend_from_slice(&location.crc32.to_be_bytes()); - value.extend_from_slice(&format::encode_object_metadata(metadata)); - value -} -fn encode_locator(locator: GitObjectLocator) -> Vec { - let mut value = <[u8; 32]>::from(locator.pack_id).to_vec(); - value.extend_from_slice(&locator.location.pack_offset.to_be_bytes()); - value.extend_from_slice(&encode_location(locator.location, locator.metadata)); - value -} -fn decode_location( - offset: u64, - bytes: &[u8], -) -> Result<(GitObjectLocation, super::GitObjectMetadata)> { - if bytes.len() != 44 { - return bad("invalid location row length"); - } - let metadata = format::decode_object_metadata(&bytes[12..]) - .ok_or_else(|| corruption("invalid object metadata"))?; - Ok(( - GitObjectLocation { - pack_offset: offset, - entry_len: u64::from_be_bytes(array(bytes, 0)?), - crc32: u32::from_be_bytes(array(bytes, 8)?), - }, - metadata, - )) -} -fn decode_locator(bytes: &[u8]) -> Result { - if bytes.len() != 84 { - return bad("invalid locator row length"); - } - let (location, metadata) = - decode_location(u64::from_be_bytes(array(bytes, 32)?), &bytes[40..])?; - Ok(GitObjectLocator { - ordinal: 0, - pack_id: MerkleHash::from(array(bytes, 0)?), - location, - metadata, - }) -} -fn array(bytes: &[u8], offset: usize) -> Result<[u8; N]> { - bytes - .get(offset..offset + N) - .and_then(|value| value.try_into().ok()) - .ok_or_else(|| corruption("truncated object index value")) -} fn hash_bytes(hash: &str) -> Result<[u8; 32]> { blake3::Hash::from_hex(hash) .map(|hash| *hash.as_bytes()) diff --git a/crates/crab-metadata/src/git_object_locator/paged/codec.rs b/crates/crab-metadata/src/git_object_locator/paged/codec.rs new file mode 100644 index 000000000..2211f6767 --- /dev/null +++ b/crates/crab-metadata/src/git_object_locator/paged/codec.rs @@ -0,0 +1,186 @@ +use super::*; +use crate::git_object_locator::GitObjectMetadata; + +pub(super) struct Placement { + pub(super) slot: u32, + pub(super) offset: u64, + pub(super) logical: GitObjectMetadata, +} + +pub(super) fn encode_pointer(slot: u32, locator: GitObjectLocator) -> Vec { + let mut value = offset_key(slot, locator.location.pack_offset); + let mut logical = locator.metadata; + logical.delta_base_oid = None; + value.extend_from_slice(&encode_metadata(logical)); + value +} + +pub(super) fn decode_pointer(bytes: &[u8]) -> Result { + let logical = decode_metadata( + bytes + .get(12..) + .ok_or_else(|| corruption("truncated object pointer"))?, + )?; + if logical.delta_base_oid.is_some() { + return bad("object pointer contains encoding-specific base facts"); + } + Ok(Placement { + slot: u32::from_be_bytes(array(bytes, 0)?), + offset: u64::from_be_bytes(array(bytes, 4)?), + logical, + }) +} + +pub(super) fn offset_key(slot: u32, offset: u64) -> Vec { + let mut key = slot.to_be_bytes().to_vec(); + key.extend_from_slice(&offset.to_be_bytes()); + key +} + +pub(super) fn encode_location(location: GitObjectLocation, metadata: GitObjectMetadata) -> Vec { + let mut value = location.entry_len.to_be_bytes().to_vec(); + value.extend_from_slice(&location.crc32.to_be_bytes()); + value.extend_from_slice(&encode_metadata(metadata)); + value +} + +pub(super) fn decode_location( + offset: u64, + bytes: &[u8], +) -> Result<(GitObjectLocation, GitObjectMetadata)> { + let metadata = decode_metadata( + bytes + .get(12..) + .ok_or_else(|| corruption("truncated physical location"))?, + )?; + Ok(( + GitObjectLocation { + pack_offset: offset, + entry_len: u64::from_be_bytes(array(bytes, 0)?), + crc32: u32::from_be_bytes(array(bytes, 8)?), + }, + metadata, + )) +} + +fn encode_metadata(metadata: GitObjectMetadata) -> Vec { + let full = format::encode_object_metadata(metadata); + let mut value = vec![full[0]]; + for (flag, range) in [(1, 1..2), (2, 2..10), (4, 10..30)] { + if full[0] & flag != 0 { + value.extend_from_slice(&full[range]); + } + } + value +} + +fn decode_metadata(bytes: &[u8]) -> Result { + let flags = *bytes + .first() + .ok_or_else(|| corruption("missing metadata flags"))?; + let mut full = [0; 32]; + full[0] = flags; + let mut position = 1; + for (flag, range) in [(1, 1..2), (2, 2..10), (4, 10..30)] { + if flags & flag == 0 { + continue; + } + let end = position + range.len(); + full[range].copy_from_slice( + bytes + .get(position..end) + .ok_or_else(|| corruption("truncated metadata fields"))?, + ); + position = end; + } + if position != bytes.len() { + return bad("metadata has trailing bytes"); + } + format::decode_object_metadata(&full).ok_or_else(|| corruption("invalid object metadata")) +} + +pub(super) fn encode_pack(pack: GitPackInventoryEntry) -> Vec { + let mut value = <[u8; 32]>::from(pack.pack_id).to_vec(); + value.extend_from_slice(&pack.object_count.to_be_bytes()); + value.extend_from_slice(&pack.pack_size.to_be_bytes()); + value +} + +pub(super) fn decode_pack(bytes: &[u8]) -> Result { + if bytes.len() != 48 { + return bad("invalid pack-slot row length"); + } + Ok(GitPackInventoryEntry { + pack_id: MerkleHash::from(array(bytes, 0)?), + object_count: u64::from_be_bytes(array(bytes, 32)?), + pack_size: u64::from_be_bytes(array(bytes, 40)?), + }) +} + +pub(super) fn encode_binding(slot: u32, pack: GitPackInventoryEntry) -> Vec { + let mut value = slot.to_be_bytes().to_vec(); + value.extend_from_slice(&pack.object_count.to_be_bytes()); + value.extend_from_slice(&pack.pack_size.to_be_bytes()); + value +} + +pub(super) fn decode_binding( + pack_id: MerkleHash, + bytes: &[u8], +) -> Result<(u32, GitPackInventoryEntry)> { + if bytes.len() != 20 { + return bad("invalid pack binding row length"); + } + Ok(( + u32::from_be_bytes(array(bytes, 0)?), + GitPackInventoryEntry { + pack_id, + object_count: u64::from_be_bytes(array(bytes, 4)?), + pack_size: u64::from_be_bytes(array(bytes, 12)?), + }, + )) +} + +pub(super) fn array(bytes: &[u8], offset: usize) -> Result<[u8; N]> { + offset + .checked_add(N) + .and_then(|end| bytes.get(offset..end)) + .and_then(|value| value.try_into().ok()) + .ok_or_else(|| corruption("truncated object index value")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::git_object_locator::GitObjectKind; + + proptest::proptest! { + #[test] + fn compact_metadata_preserves_presence_and_extreme_values( + kind in 0usize..5, + size in proptest::option::of(proptest::num::u64::ANY), + base in proptest::option::of(proptest::array::uniform20(proptest::num::u8::ANY)), + ) { + let kinds = [None, Some(GitObjectKind::Commit), Some(GitObjectKind::Tree), Some(GitObjectKind::Blob), Some(GitObjectKind::Tag)]; + let expected = GitObjectMetadata { kind: kinds[kind], logical_size: size, delta_base_oid: base }; + proptest::prop_assert_eq!(decode_metadata(&encode_metadata(expected)).unwrap(), expected); + } + } + + #[test] + fn compact_metadata_rejects_truncation_reserved_flags_and_trailing_data() { + let metadata = GitObjectMetadata { + kind: Some(GitObjectKind::Blob), + logical_size: Some(0), + delta_base_oid: Some([0; 20]), + }; + let bytes = encode_metadata(metadata); + for end in 0..bytes.len() { + assert!(decode_metadata(&bytes[..end]).is_err()); + } + for invalid in [vec![8], vec![1, 0], vec![0, 0]] { + assert!(decode_metadata(&invalid).is_err()); + } + assert_eq!(decode_metadata(&bytes).unwrap(), metadata); + } +} diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs new file mode 100644 index 000000000..27061a773 --- /dev/null +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -0,0 +1,444 @@ +use super::*; + +impl PagedGitObjectIndex { + /// Capture an authenticated descriptor without loading its physical inventory or locations. + /// + /// Callers accept readiness only after complete verification; placement never authorizes refs. + pub async fn load( + store: &Store, + layout: &StoreLayout, + hash: &str, + state_digest: &str, + manifest: &Manifest, + maximum: u64, + ) -> Result { + validate_content_hash(hash, "object index descriptor", "Git object index")?; + let expected = hash_bytes(hash)?; + let bytes = store + .verify_bounded( + &layout.bulk_manifest_path("git-object-index", hash), + &expected, + maximum.min(MAX_DESCRIPTOR_BYTES), + ) + .await?; + let descriptor: Descriptor = serde_json::from_slice(&bytes) + .map_err(|source| crate::error::MetadataError::GitObjectIndexRecord { source })?; + descriptor.validate()?; + if descriptor.state_digest != state_digest + || descriptor.generation != manifest.generation + || descriptor.pack_index_hash != manifest.pack_index_hash + || descriptor.git_validation_digest != manifest.git_validation_digest + { + return bad("object index does not match captured state"); + } + let path = body_path(layout, &descriptor.body_hash); + Ok(Self { + descriptor, + descriptor_bytes: bytes.len() as u64, + path, + origin: Arc::clone(layout.store().inner()), + global_prefix: layout.global_prefix().to_owned(), + maximum, + cache: PageCache::default(), + }) + } + + /// Check captured state and in-process origin before attaching lookup to a repository. + #[must_use] + pub fn matches_binding( + &self, + layout: &StoreLayout, + state_digest: &str, + manifest: &Manifest, + ) -> bool { + Arc::ptr_eq(&self.origin, layout.store().inner()) + && self.global_prefix == layout.global_prefix() + && self.path == body_path(layout, &self.descriptor.body_hash) + && self.descriptor.state_digest == state_digest + && self.descriptor.generation == manifest.generation + && self.descriptor.pack_index_hash == manifest.pack_index_hash + && self.descriptor.git_validation_digest == manifest.git_validation_digest + } + + /// Return the immutable body containing every page for retained-root GC. + #[must_use] + pub fn body_path(&self) -> &object_store::path::Path { + &self.path + } + + /// Resolve OIDs against the captured physical inventory, preserving duplicates and misses. + /// + /// The caller admits request/output counts and supplies an admitted store for the captured origin. + pub async fn lookup( + &self, + store: &Store, + oids: &[[u8; 20]], + inventory: &HashMap, + ) -> Result>> { + self.lookup_with_work(store, oids, inventory, &mut self.work()) + .await + } + + async fn lookup_with_work( + &self, + store: &Store, + oids: &[[u8; 20]], + inventory: &HashMap, + work: &mut PageWork, + ) -> Result>> { + let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + let values = self + .lookup_rows( + store, + &self.descriptor.objects, + Kind::ObjectLocators, + &keys, + work, + ) + .await?; + let pointers = values + .into_iter() + .map(|value| value.map(|value| decode_pointer(&value)).transpose()) + .collect::>>()?; + let slots = pointers + .iter() + .flatten() + .map(|pointer| pointer.slot) + .collect::>(); + let packs = self.packs_for_slots(store, &slots, inventory, work).await?; + let keys = pointers + .iter() + .flatten() + .map(|pointer| offset_key(pointer.slot, pointer.offset)) + .collect::>(); + let rows = self + .lookup_rows( + store, + &self.descriptor.offsets, + Kind::PackOffsets, + &keys, + work, + ) + .await?; + let mut rows = rows.into_iter(); + let mut output = Vec::with_capacity(oids.len()); + for (oid, pointer) in oids.iter().zip(pointers) { + let Some(pointer) = pointer else { + output.push(None); + continue; + }; + let row = rows + .next() + .flatten() + .ok_or_else(|| corruption("object pointer has no physical offset"))?; + if array::<20>(&row, 0)? != *oid { + return bad("object and physical offset identities differ"); + } + let pack = packs + .get(&pointer.slot) + .ok_or_else(|| corruption("primary pack slot is absent"))?; + let (location, mut metadata) = decode_location( + pointer.offset, + row.get(20..) + .ok_or_else(|| corruption("truncated offset row"))?, + )?; + if !format::validate_location_for_pack(location, pack.pack_size) + || logical_conflict(metadata, pointer.logical) + { + return bad("physical location or logical facts differ"); + } + // Logical facts can be proven by another alias. Encoding-specific + // base facts remain solely in the selected physical record. + metadata.kind = pointer.logical.kind.or(metadata.kind); + metadata.logical_size = pointer.logical.logical_size.or(metadata.logical_size); + output.push(Some(GitObjectLocator { + ordinal: 0, + pack_id: pack.pack_id, + location, + metadata, + })); + } + Ok(output) + } + + /// Resolve physical delta-base offsets without reading an entire reverse pack index. + /// + /// The caller admits request/output counts and supplies an admitted store for the captured origin. + pub async fn oids_at_offsets( + &self, + store: &Store, + offsets: &[(MerkleHash, u64)], + inventory: &HashMap, + ) -> Result>> { + let mut work = self.work(); + let pack_ids = offsets.iter().map(|(pack, _)| *pack).collect::>(); + let slots = self + .slots_for_packs(store, &pack_ids, inventory, &mut work) + .await?; + let keys = slots + .iter() + .zip(offsets) + .map(|(slot, (_, offset))| offset_key(*slot, *offset)) + .collect::>(); + let rows = self + .lookup_rows( + store, + &self.descriptor.offsets, + Kind::PackOffsets, + &keys, + &mut work, + ) + .await?; + let mut output = Vec::with_capacity(rows.len()); + let mut physical_facts = Vec::new(); + for ((pack_id, offset), row) in offsets.iter().zip(rows) { + let Some(row) = row else { + output.push(None); + continue; + }; + let oid = array(&row, 0)?; + let (location, metadata) = decode_location( + *offset, + row.get(20..) + .ok_or_else(|| corruption("truncated offset row"))?, + )?; + if !inventory + .get(pack_id) + .is_some_and(|pack| format::validate_location_for_pack(location, pack.pack_size)) + { + return bad("location escapes captured pack"); + } + output.push(Some(oid)); + physical_facts.push(metadata); + } + let oids = output.iter().flatten().copied().collect::>(); + let primary = self + .lookup_with_work(store, &oids, inventory, &mut work) + .await?; + for (physical, primary) in physical_facts.into_iter().zip(primary) { + let primary = + primary.ok_or_else(|| corruption("physical offset has no object identity"))?; + if logical_conflict(physical, primary.metadata) { + return bad("physical aliases disagree about logical facts"); + } + } + Ok(output) + } + + /// Verify complete coverage and canonical bytes before publishing readiness. + pub async fn verify_complete( + &self, + store: &Store, + manifest: &Manifest, + maximum: u64, + ) -> Result<()> { + let bytes = store + .verify_bounded( + &self.path, + &hash_bytes(&self.descriptor.body_hash)?, + maximum, + ) + .await?; + if bytes.len() as u64 != self.descriptor.body_bytes { + return bad("object index body length mismatch"); + } + let mut audit = BTreeMap::new(); + crate::pages::scan( + &bytes, + &self.descriptor.objects, + Kind::ObjectLocators, + self.path.as_ref(), + &mut audit, + )?; + crate::pages::scan( + &bytes, + &self.descriptor.packs, + Kind::PackInventory, + self.path.as_ref(), + &mut audit, + )?; + let slots = crate::pages::scan( + &bytes, + &self.descriptor.slots, + Kind::PackSlots, + self.path.as_ref(), + &mut audit, + )?; + let inventory = slots + .into_iter() + .map(|(_, value)| decode_pack(&value)) + .collect::>>()?; + let offsets = crate::pages::scan( + &bytes, + &self.descriptor.offsets, + Kind::PackOffsets, + self.path.as_ref(), + &mut audit, + )?; + let entries = offsets + .into_iter() + .map(|(key, value)| { + let slot = u32::from_be_bytes(array(&key, 0)?); + let pack = inventory + .get(slot as usize) + .ok_or_else(|| corruption("physical pack slot is absent"))?; + let offset = u64::from_be_bytes(array(&key, 4)?); + let (location, metadata) = decode_location( + offset, + value + .get(20..) + .ok_or_else(|| corruption("truncated offset row"))?, + )?; + Ok(( + array(&value, 0)?, + GitObjectLocator { + ordinal: 0, + pack_id: pack.pack_id, + location, + metadata, + }, + )) + }) + .collect::>>()?; + let rebuilt = PagedGitObjectIndexWrite::build( + &self.descriptor.state_digest, + manifest, + &inventory, + &entries, + )?; + if rebuilt.descriptor != self.descriptor || rebuilt.body.as_ref() != bytes.as_ref() { + return bad("object index coverage or canonical placement mismatch"); + } + Ok(()) + } + + fn work(&self) -> PageWork { + PageWork { + maximum: self.maximum, + bytes: self.descriptor_bytes, + seen: BTreeSet::new(), + } + } + + async fn lookup_rows( + &self, + store: &Store, + root: &PageRef, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + self.cache + .lookup( + store, + &self.path, + 0, + self.descriptor.body_bytes, + root, + kind, + keys, + work, + ) + .await + } + + async fn packs_for_slots( + &self, + store: &Store, + slots: &[u32], + inventory: &HashMap, + work: &mut PageWork, + ) -> Result> { + let slots = slots + .iter() + .copied() + .collect::>() + .into_iter() + .collect::>(); + let keys = slots + .iter() + .map(|slot| slot.to_be_bytes().to_vec()) + .collect::>(); + let rows = self + .lookup_rows(store, &self.descriptor.slots, Kind::PackSlots, &keys, work) + .await?; + let mut packs = HashMap::new(); + for (slot, row) in slots.iter().zip(rows) { + let row = row.ok_or_else(|| corruption("required pack slot is absent"))?; + let pack = decode_pack(&row)?; + if inventory.get(&pack.pack_id) != Some(&pack) { + return bad("pack slot differs from captured inventory"); + } + packs.insert(*slot, pack); + } + let keys = slots + .iter() + .map(|slot| { + packs + .get(slot) + .map(|pack| <[u8; 32]>::from(pack.pack_id).to_vec()) + .ok_or_else(|| corruption("pack slot result is absent")) + }) + .collect::>>()?; + let rows = self + .lookup_rows( + store, + &self.descriptor.packs, + Kind::PackInventory, + &keys, + work, + ) + .await?; + for (slot, row) in slots.into_iter().zip(rows) { + let pack = *packs + .get(&slot) + .ok_or_else(|| corruption("pack slot result is absent"))?; + let row = row.ok_or_else(|| corruption("required pack binding is absent"))?; + if decode_binding(pack.pack_id, &row)? != (slot, pack) { + return bad("pack directories disagree"); + } + } + Ok(packs) + } + + async fn slots_for_packs( + &self, + store: &Store, + ids: &[MerkleHash], + inventory: &HashMap, + work: &mut PageWork, + ) -> Result> { + let keys = ids + .iter() + .map(|id| <[u8; 32]>::from(*id).to_vec()) + .collect::>(); + let rows = self + .lookup_rows( + store, + &self.descriptor.packs, + Kind::PackInventory, + &keys, + work, + ) + .await?; + let slots = ids + .iter() + .zip(rows) + .map(|(id, row)| { + let row = row.ok_or_else(|| corruption("required pack binding is absent"))?; + let (slot, pack) = decode_binding(*id, &row)?; + if inventory.get(id) != Some(&pack) { + return bad("pack binding differs from captured inventory"); + } + Ok(slot) + }) + .collect::>>()?; + let packs = self.packs_for_slots(store, &slots, inventory, work).await?; + for (id, slot) in ids.iter().zip(&slots) { + if packs.get(slot).is_none_or(|pack| pack.pack_id != *id) { + return bad("pack directories disagree"); + } + } + Ok(slots) + } +} diff --git a/crates/crab-metadata/src/git_object_locator/paged/tests.rs b/crates/crab-metadata/src/git_object_locator/paged/tests.rs index dec24e9fe..849dc6189 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/tests.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/tests.rs @@ -246,8 +246,8 @@ async fn descriptors_cannot_cross_captured_git_states() { ); } -#[test] -fn object_aliases_prefer_full_encodings_and_keep_every_physical_offset() { +#[tokio::test] +async fn object_aliases_prefer_full_encodings_and_keep_every_physical_offset() { let (mut manifest, mut inventory, mut entries) = inputs(1); entries[0].1.metadata.delta_base_oid = Some(oid(0)); let mut full = entries[0].1; @@ -274,17 +274,32 @@ fn object_aliases_prefer_full_encodings_and_keep_every_physical_offset() { manifest.pack_index_hash = crate::manifests::compact_pack_index(1, &packs).unwrap().0; let write = PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); - let mut audit = BTreeMap::new(); - let rows = crate::pages::scan( - &write.body, - &write.descriptor.objects, - Kind::ObjectLocators, - "test index", - &mut audit, + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "alias-index".to_owned()); + let hash = write.upload(&store, &layout).await.unwrap(); + let reader = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 128 * 1024, ) + .await .unwrap(); - let selected = decode_locator(&rows[0].1).unwrap(); - assert_eq!((selected, write.descriptor.offsets.count), (full, 2)); + let inventory = inventory + .into_iter() + .map(|pack| (pack.pack_id, pack)) + .collect(); + let selected = reader.lookup(&store, &[oid(0)], &inventory).await.unwrap(); + reader + .verify_complete(&store, &manifest, 128 * 1024) + .await + .unwrap(); + assert_eq!( + (selected, write.descriptor.offsets.count), + (vec![Some(full)], 2) + ); } proptest::proptest! { @@ -330,15 +345,26 @@ async fn resigned_oid_locations_must_agree_with_the_physical_offset_directory() &mut audit, ) .unwrap(); + let slots = crate::pages::scan( + &write.body, + &write.descriptor.slots, + Kind::PackSlots, + "test index", + &mut audit, + ) + .unwrap(); + let pointer = decode_pointer(&objects[0].1).unwrap(); let mut forged = entries[0].1; forged.location.pack_offset = entries[1].1.location.pack_offset; - objects[0].1 = encode_locator(forged); - let (body, objects, offsets, packs) = encode_trees(objects, offsets, packs).unwrap(); + objects[0].1 = encode_pointer(pointer.slot, forged); + let (body, [objects, offsets, packs, slots]) = + encode_trees(objects, offsets, packs, slots).unwrap(); write.descriptor.body_hash = blake3::hash(&body).to_hex().to_string(); write.descriptor.body_bytes = body.len() as u64; write.descriptor.objects = objects; write.descriptor.offsets = offsets; write.descriptor.packs = packs; + write.descriptor.slots = slots; write.body = body.into(); let store = Store::new(Arc::new(InMemory::new())); let layout = StoreLayout::new(store.clone(), "forged-object-index".into()); @@ -423,3 +449,155 @@ async fn physical_index_attachment_binds_origin_prefixes_and_captured_state() { [true, false, false, false, false] ); } + +#[test] +fn pack_slots_are_canonical_under_inventory_and_entry_permutations() { + let (mut manifest, _, one) = inputs(1); + let mut inventory = Vec::new(); + let mut entries = Vec::new(); + for (index, bytes) in [[3; 32], [0xff; 32], [0x55; 32]].into_iter().enumerate() { + let pack_id = MerkleHash::from(bytes); + inventory.push(GitPackInventoryEntry { + pack_id, + object_count: 1, + pack_size: 42, + }); + let mut locator = one[0].1; + locator.pack_id = pack_id; + locator.metadata.logical_size = Some(index as u64); + entries.push((oid(index as u32), locator)); + } + let mut packs = inventory + .iter() + .map(|pack| crate::manifests::PackManifestEntry { + pack_id: pack.pack_id.to_string(), + size: pack.pack_size, + content_hash: pack.pack_id.to_string(), + ref_tips: Vec::new(), + object_count: pack.object_count, + }) + .collect::>(); + packs.sort_unstable_by(|a, b| a.pack_id.cmp(&b.pack_id)); + manifest.pack_index_hash = crate::manifests::compact_pack_index(1, &packs).unwrap().0; + let first = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + inventory.reverse(); + entries.rotate_left(1); + let second = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + assert_eq!( + (first.descriptor, first.body), + (second.descriptor, second.body) + ); +} + +#[tokio::test] +async fn resigned_pack_directories_must_agree_before_point_lookup() { + let (manifest, inventory, entries) = inputs(10); + let mut write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let mut audit = BTreeMap::new(); + let objects = crate::pages::scan( + &write.body, + &write.descriptor.objects, + Kind::ObjectLocators, + "test", + &mut audit, + ) + .unwrap(); + let offsets = crate::pages::scan( + &write.body, + &write.descriptor.offsets, + Kind::PackOffsets, + "test", + &mut audit, + ) + .unwrap(); + let mut packs = crate::pages::scan( + &write.body, + &write.descriptor.packs, + Kind::PackInventory, + "test", + &mut audit, + ) + .unwrap(); + let slots = crate::pages::scan( + &write.body, + &write.descriptor.slots, + Kind::PackSlots, + "test", + &mut audit, + ) + .unwrap(); + packs[0].1 = encode_binding(1, inventory[0]); + let (body, [objects, offsets, packs, slots]) = + encode_trees(objects, offsets, packs, slots).unwrap(); + write.descriptor.body_hash = blake3::hash(&body).to_hex().to_string(); + write.descriptor.body_bytes = body.len() as u64; + write.descriptor.objects = objects; + write.descriptor.offsets = offsets; + write.descriptor.packs = packs; + write.descriptor.slots = slots; + write.body = body.into(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "forged-pack-slot".to_owned()); + let hash = write.upload(&store, &layout).await.unwrap(); + let reader = PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 128 * 1024, + ) + .await + .unwrap(); + let inventory = inventory + .into_iter() + .map(|pack| (pack.pack_id, pack)) + .collect(); + assert_eq!( + ( + reader.lookup(&store, &[oid(0)], &inventory).await.is_err(), + reader + .verify_complete(&store, &manifest, 128 * 1024) + .await + .is_err() + ), + (true, true) + ); +} + +#[tokio::test] +async fn retired_physical_descriptor_shape_is_rejected_before_body_lookup() { + let (manifest, inventory, entries) = inputs(1); + let write = + PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); + let mut descriptor = serde_json::to_value(&write.descriptor).unwrap(); + descriptor["version"] = serde_json::json!(1); + descriptor.as_object_mut().unwrap().remove("slots"); + let bytes = serde_json::to_vec(&descriptor).unwrap(); + let hash = blake3::hash(&bytes).to_hex().to_string(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "retired-physical-index".to_owned()); + store + .put( + &layout.bulk_manifest_path("git-object-index", &hash), + Bytes::from(bytes), + ) + .await + .unwrap(); + // No body is stored: retirement must be recognized from the authenticated descriptor. + assert!(matches!( + PagedGitObjectIndex::load( + &store, + &layout, + &hash, + &"c".repeat(64), + &manifest, + 128 * 1024 + ) + .await, + Err(crate::error::MetadataError::GitObjectIndexRecord { .. }) + )); +} diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index b7be2fd69..f78a4a0e0 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -55,9 +55,20 @@ pub(crate) enum Kind { PackOffsets, #[cfg(feature = "remote-index")] PackInventory, + #[cfg(feature = "remote-index")] + PackSlots, } impl Kind { + fn is_positional(self) -> bool { + match self { + Self::Records | Self::Nodes | Self::CommitRecords | Self::CommitAncestryRoots => true, + #[cfg(feature = "remote-index")] + Self::PackSlots => true, + _ => false, + } + } + pub(crate) fn key_bytes(self) -> usize { match self { Self::Names | Self::CommitAncestryNodes => 32, @@ -67,7 +78,7 @@ impl Kind { #[cfg(feature = "remote-index")] Self::ObjectLocators => 20, #[cfg(feature = "remote-index")] - Self::PackOffsets => 40, + Self::PackOffsets => 12, _ => 4, } } @@ -97,10 +108,7 @@ impl PageRef { { return Err(corruption("invalid metadata page reference")); } - if matches!( - kind, - Kind::Records | Kind::Nodes | Kind::CommitRecords | Kind::CommitAncestryRoots - ) { + if kind.is_positional() { let first = integer(&self.first)?; let last = integer(&self.last)?; if last.checked_sub(first).and_then(|span| span.checked_add(1)) != Some(self.count) { @@ -303,15 +311,14 @@ pub(crate) fn decode( { return corrupt_at(path, "invalid page directory routing or ancestry"); } - if matches!( - kind, - Kind::Records | Kind::Nodes | Kind::CommitRecords | Kind::CommitAncestryRoots - ) && children.last().is_some_and(|previous| { - integer(&previous.last) - .ok() - .and_then(|last| last.checked_add(1)) - != integer(&child.first).ok() - }) { + if kind.is_positional() + && children.last().is_some_and(|previous| { + integer(&previous.last) + .ok() + .and_then(|last| last.checked_add(1)) + != integer(&child.first).ok() + }) + { return corrupt_at(path, "positional directory has a key gap"); } total = total @@ -331,7 +338,7 @@ pub(crate) fn decode( tracing::debug!(target:"crab_metadata::index_decode", index_format=match kind { Kind::CommitRecords | Kind::CommitOids | Kind::CommitAncestryRoots | Kind::CommitAncestryNodes => "commit_graph", #[cfg(feature = "remote-index")] - Kind::ObjectLocators | Kind::PackOffsets | Kind::PackInventory => "git_object_index", + Kind::ObjectLocators | Kind::PackOffsets | Kind::PackInventory | Kind::PackSlots => "git_object_index", _ => "path_state", }, decoded_bytes=bytes.len() as u64, "index page decoded"); @@ -390,15 +397,14 @@ fn validate_rows(kind: Kind, rows: &Rows) -> Result<()> { { return Err(corruption("page keys are empty, invalid or unordered")); } - if matches!( - kind, - Kind::Records | Kind::Nodes | Kind::CommitRecords | Kind::CommitAncestryRoots - ) && rows.windows(2).any(|pair| { - integer(&pair[0].0) - .ok() - .and_then(|value| value.checked_add(1)) - != integer(&pair[1].0).ok() - }) { + if kind.is_positional() + && rows.windows(2).any(|pair| { + integer(&pair[0].0) + .ok() + .and_then(|value| value.checked_add(1)) + != integer(&pair[1].0).ok() + }) + { return Err(corruption("positional leaf has a key gap")); } Ok(()) From f9b489c85875d80060877105f8e7278edd55d0c4 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 22:53:38 -0700 Subject: [PATCH 018/127] perf(metadata): reuse verified pack bindings Offset lookup already authenticates the forward dictionary row. Join its reverse slot once while retaining captured-inventory and pack-ID checks. A re-signed valid-slot redirect regression protects offset identity. Local join medians improve; provider and opening gates remain unresolved. --- .../docs/architecture/indexed-remote-reads.md | 44 +++++++++++++ .../src/git_object_locator/paged/reader.rs | 64 ++++++++++--------- .../src/git_object_locator/paged/tests.rs | 41 ++++++++++-- 3 files changed, 113 insertions(+), 36 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 49266a77a..039969a48 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -680,3 +680,47 @@ The two runs are on a shared host rather than a controlled paired release build; the cause is not established. Repeated authenticated dictionary joins and page traversal need profiling, followed by controlled provider/release qualification. This is an unresolved performance gate; smaller index bytes do not close it. + + +## Remove redundant compact-directory traversal + +Offset-delta lookup previously read pack ID → slot, then slot → pack, then the +same pack ID → slot again. The first forward row is already authenticated and +checked against captured inventory. Both lookup directions now share one typed +slot-row reader; offset lookup joins its original forward row directly to that +reverse row. The reverse pack identity check remains mandatory. OID lookup still +checks both directions, and all reads retain the same aggregate work admission. +This changes no stored bytes, descriptor version, public API or configuration. + +A stronger re-signed-directory test redirects a pack to another existing valid +slot with equal size/count. Point and offset lookup reject it. Removing only the +reverse identity comparison makes offset lookup accept another pack's object; +the exact one-test negative control fails and restored source passes. All 15 +physical-index tests, five runtime regressions, metadata strict Clippy and both +qualification-tool builds pass. Temporary probes are removed from source. + +Twenty interleaved in-memory samples of 1,000 warm joins compare the actual new +helper with the redundant third traversal. Median times are 35.700 versus +40.805 ms (12.5% lower); the optimized path is faster in 14/20 pairs. Its p95 is +156.667 versus 108.873 ms, so the noisy shared-host experiment does not establish +an improved latency tail or a completed end-to-end performance gate. Other +worktree compilers were observed using CPU during diagnosis. Earlier preliminary +prototype timings are not substituted for this production-helper measurement. + +The remaining opening audit identifies a distinct boundary: +`open_view_from_root_with_metadata` → `assemble_layered_control_view` loads the +complete checkpoint, then constructs its visibility index to order ref controls. +`control_is_ready` checks new-branch base membership in that index; the complete +capsule sibling applies the same rule. Footer-only fetch deliberately has a +narrower admission contract. Bounded opening therefore needs authenticated +visibility/catalog lookups and transaction reconciliation, not a silent switch +to the fetch loader. Existing metadata-view tests cover snapshot equivalence, +multi-ref activation and missing dependencies; those invariants must remain in +the next implementation slice. This audit does not close the opening gate. + +The optimized binary's RustFS run passes the full ten-query native oracle at +100 commits and all 20 cold/warm resolve/attribution samples at 100 and 1,000 +commits. Warm queries read zero backend bytes and completed operations retain no +active backend work. The 10k history sampling remains in progress; source/binary +hashes and completed receipts are retained externally. This partial observation +is not recorded as a completed 121-process qualification. diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index 27061a773..5e3374131 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -342,19 +342,14 @@ impl PagedGitObjectIndex { .await } - async fn packs_for_slots( + async fn slot_packs( &self, store: &Store, slots: &[u32], inventory: &HashMap, work: &mut PageWork, - ) -> Result> { - let slots = slots - .iter() - .copied() - .collect::>() - .into_iter() - .collect::>(); + ) -> Result> { + let slots = slots.iter().copied().collect::>(); let keys = slots .iter() .map(|slot| slot.to_be_bytes().to_vec()) @@ -362,24 +357,32 @@ impl PagedGitObjectIndex { let rows = self .lookup_rows(store, &self.descriptor.slots, Kind::PackSlots, &keys, work) .await?; - let mut packs = HashMap::new(); - for (slot, row) in slots.iter().zip(rows) { - let row = row.ok_or_else(|| corruption("required pack slot is absent"))?; - let pack = decode_pack(&row)?; - if inventory.get(&pack.pack_id) != Some(&pack) { - return bad("pack slot differs from captured inventory"); - } - packs.insert(*slot, pack); - } - let keys = slots - .iter() - .map(|slot| { - packs - .get(slot) - .map(|pack| <[u8; 32]>::from(pack.pack_id).to_vec()) - .ok_or_else(|| corruption("pack slot result is absent")) + slots + .into_iter() + .zip(rows) + .map(|(slot, row)| { + let row = row.ok_or_else(|| corruption("required pack slot is absent"))?; + let pack = decode_pack(&row)?; + if inventory.get(&pack.pack_id) != Some(&pack) { + return bad("pack slot differs from captured inventory"); + } + Ok((slot, pack)) }) - .collect::>>()?; + .collect() + } + + async fn packs_for_slots( + &self, + store: &Store, + slots: &[u32], + inventory: &HashMap, + work: &mut PageWork, + ) -> Result> { + let packs = self.slot_packs(store, slots, inventory, work).await?; + let keys = packs + .values() + .map(|pack| <[u8; 32]>::from(pack.pack_id).to_vec()) + .collect::>(); let rows = self .lookup_rows( store, @@ -389,12 +392,9 @@ impl PagedGitObjectIndex { work, ) .await?; - for (slot, row) in slots.into_iter().zip(rows) { - let pack = *packs - .get(&slot) - .ok_or_else(|| corruption("pack slot result is absent"))?; + for ((slot, pack), row) in packs.iter().zip(rows) { let row = row.ok_or_else(|| corruption("required pack binding is absent"))?; - if decode_binding(pack.pack_id, &row)? != (slot, pack) { + if decode_binding(pack.pack_id, &row)? != (*slot, *pack) { return bad("pack directories disagree"); } } @@ -433,7 +433,9 @@ impl PagedGitObjectIndex { Ok(slot) }) .collect::>>()?; - let packs = self.packs_for_slots(store, &slots, inventory, work).await?; + // The forward binding is already authenticated above. Read its reverse once; + // repeating the forward traversal adds work for every offset-delta base. + let packs = self.slot_packs(store, &slots, inventory, work).await?; for (id, slot) in ids.iter().zip(&slots) { if packs.get(slot).is_none_or(|pack| pack.pack_id != *id) { return bad("pack directories disagree"); diff --git a/crates/crab-metadata/src/git_object_locator/paged/tests.rs b/crates/crab-metadata/src/git_object_locator/paged/tests.rs index 849dc6189..bef5084ef 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/tests.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/tests.rs @@ -492,8 +492,33 @@ fn pack_slots_are_canonical_under_inventory_and_entry_permutations() { } #[tokio::test] -async fn resigned_pack_directories_must_agree_before_point_lookup() { - let (manifest, inventory, entries) = inputs(10); +async fn resigned_pack_binding_cannot_redirect_reads_to_another_pack() { + let (mut manifest, mut inventory, mut entries) = inputs(10); + let mut second_pack = inventory[0]; + second_pack.pack_id = MerkleHash::from([4; 32]); + inventory.push(second_pack); + let second_entries = entries + .iter() + .enumerate() + .map(|(index, (_, locator))| { + let mut locator = *locator; + locator.pack_id = second_pack.pack_id; + (oid(10 + index as u32), locator) + }) + .collect::>(); + entries.extend(second_entries); + let mut packs = inventory + .iter() + .map(|pack| crate::manifests::PackManifestEntry { + pack_id: pack.pack_id.to_string(), + size: pack.pack_size, + content_hash: pack.pack_id.to_string(), + ref_tips: Vec::new(), + object_count: pack.object_count, + }) + .collect::>(); + packs.sort_unstable_by(|left, right| left.pack_id.cmp(&right.pack_id)); + manifest.pack_index_hash = crate::manifests::compact_pack_index(1, &packs).unwrap().0; let mut write = PagedGitObjectIndexWrite::build(&"c".repeat(64), &manifest, &inventory, &entries).unwrap(); let mut audit = BTreeMap::new(); @@ -529,7 +554,9 @@ async fn resigned_pack_directories_must_agree_before_point_lookup() { &mut audit, ) .unwrap(); - packs[0].1 = encode_binding(1, inventory[0]); + // Both slots and inventories are valid. Only the binding redirects this pack; + // dropping the reverse-ID check would return the other pack's object. + packs[1].1 = encode_binding(0, inventory[1]); let (body, [objects, offsets, packs, slots]) = encode_trees(objects, offsets, packs, slots).unwrap(); write.descriptor.body_hash = blake3::hash(&body).to_hex().to_string(); @@ -558,13 +585,17 @@ async fn resigned_pack_directories_must_agree_before_point_lookup() { .collect(); assert_eq!( ( - reader.lookup(&store, &[oid(0)], &inventory).await.is_err(), + reader.lookup(&store, &[oid(10)], &inventory).await.is_err(), + reader + .oids_at_offsets(&store, &[(second_pack.pack_id, 12)], &inventory) + .await + .is_err(), reader .verify_complete(&store, &manifest, 128 * 1024) .await .is_err() ), - (true, true) + (true, true, true) ); } From a4dc4b36d4a626bb63dfab46c4175c2709535e34 Mon Sep 17 00:00:00 2001 From: forhappy Date: Tue, 13 Oct 2026 23:57:09 -0700 Subject: [PATCH 019/127] perf(read): omit unused indexed placement hints Paged object lookup resolves placements before hint paths run. Borrow ordinal admission directly and avoid a second per-OID dictionary. Preserve source/member and capsule-run validation for indexed reads. Record final-source native-Git replay and remaining opening costs. --- .../docs/architecture/indexed-remote-reads.md | 55 ++++++++++++++-- crates/crab-read/src/capsule_protocol.rs | 65 ++++++++----------- 2 files changed, 77 insertions(+), 43 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 039969a48..3c6dd4465 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -718,9 +718,52 @@ to the fetch loader. Existing metadata-view tests cover snapshot equivalence, multi-ref activation and missing dependencies; those invariants must remain in the next implementation slice. This audit does not close the opening gate. -The optimized binary's RustFS run passes the full ten-query native oracle at -100 commits and all 20 cold/warm resolve/attribution samples at 100 and 1,000 -commits. Warm queries read zero backend bytes and completed operations retain no -active backend work. The 10k history sampling remains in progress; source/binary -hashes and completed receipts are retained externally. This partial observation -is not recorded as a completed 121-process qualification. +The dictionary-join binary at `d68a9177915` completes all 121 RustFS native-Git +query processes: all ten queries at 100 commits and 20 cold/warm resolve and +attribution samples at 100/1,000/10,000 commits. Warm queries read zero backend +bytes, and measured operations finish with zero active backend work. Exact source +and binary hashes, native results and independently checked summaries are external. + +| History | Resolve p95, ms | Attribution p95, ms | Maximum attribution decoded bytes | +| --- | ---: | ---: | ---: | +| 100 | 18.843 | 398.433 | 51,111 | +| 1,000 | 30.370 | 163.718 | 232,194 | +| 10,000 | 32.646 | 871.628 | 345,933 | + +The fixed-query byte bounds pass. These noisy shared-host debug timings do not +close the latency gate. Opening still reads 2,183,801 bytes at 10k and process +memory/CPU grow with history; correctness proof is not bounded-opening proof. + + +## Indexed opening without redundant placement hints + +`git_repository_from_store` previously copied every ordinal admission into an +intermediate OID vector and built another per-object placement dictionary even +when a complete physical index was attached. `lookup_batch_for_read` returns from +that index before consulting any placement hints. Opening now borrows ordinal +admission directly and retains hint dictionaries only for unindexed reads. +Source/member range checks, hash parsing and capsule-run admission/count validation +remain unconditional. Ref reconciliation, visibility proofs, catalog ordinals, +thin-pack client-base authorization and reconstruction checks are unchanged. +The refactor removes nine production lines and adds no format/config/API surface. + +Two metadata-view equivalence/activation tests, five runtime regressions and the +unindexed admitted-source lookup test pass. Read-library strict Clippy and both +qualification-tool builds pass. The final-source RustFS replay passes 61 native-Git +processes: all ten queries at 100 commits plus 20 cold/warm opening and resolve +samples at 100/1,000/10,000 commits. Warm resolve queries read zero backend bytes; +measurements finish with no active backend work. Source/binary receipts and checked +summaries remain external. + +| History | Cold open median / p95, ms | Opening backend bytes | Process median RSS, bytes | Process median CPU, s | +| --- | --- | ---: | ---: | ---: | +| 100 | 41.672 / 51.923 | 89,134 | 32,931,840 | 0.070 | +| 1,000 | 141.332 / 164.715 | 557,942 | 36,544,512 | 0.225 | +| 10,000 | 675.488 / 747.510 | 2,183,801 | 61,693,952 | 1.050 | + +The previous dictionary-join resolve cells used 78,217,216 bytes median RSS and +1.23 s median process CPU at 10k; this replay uses 61,693,952 bytes and 1.05 s. +These are shared-host debug observations, not a controlled release/provider +latency result. Metadata transfer is unchanged. Complete visibility decoding, +transaction reconciliation and source/ref inventory capture still scale; this +cleanup does not satisfy bounded opening or finish stage 4. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 4d44449b2..6aeeaf36f 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -936,30 +936,16 @@ impl CapsuleRepositoryView { // The footer still authenticates source descriptors and frontier // admission, while the complete ordinal join is only available to // maintenance/strict readers that loaded the full checkpoint. - let preferred_object_admission = if let Some(checkpoint) = checkpoint + // A ready physical index resolves every placement before hint paths run. + // Keep validating source/member admission, but avoid a second OID dictionary. + let mut preferred_object_admission_by_oid = + object_index.is_none().then(std::collections::HashMap::new); + if let Some(checkpoint) = checkpoint && !checkpoint.is_control_only() + && let Some(snapshot) = checkpoint.visibility_ordinal_snapshot()? + && let Some(admission) = snapshot.member_admission() { - checkpoint - .visibility_ordinal_snapshot()? - .and_then(|snapshot| { - snapshot.member_admission().map(|admission| { - snapshot - .objects() - .iter() - .zip(admission) - .map(|(oid, member)| (*oid, *member)) - .collect::>() - }) - }) - } else { - None - }; - let mut preferred_object_admission_by_oid: std::collections::HashMap< - [u8; 20], - Vec, - > = std::collections::HashMap::new(); - if let Some(admission) = preferred_object_admission.as_ref() { - for (oid, member) in admission { + for (oid, member) in snapshot.objects().iter().zip(admission) { let source = sources .get(usize::from(member.source_index())) .ok_or_else(|| { @@ -973,19 +959,20 @@ impl CapsuleRepositoryView { })?; let pack_id = crab_xet::hash::MerkleHash::from_hex(pack.pack().blake3()) .map_err(|error| corrupt_path("layered visibility", error.to_string()))?; - preferred_object_admission_by_oid - .entry(*oid) - .or_default() - .push(pack_id); + if let Some(admission) = preferred_object_admission_by_oid.as_mut() { + admission.entry(*oid).or_insert_with(Vec::new).push(pack_id); + } } } for (oid, pack_ids) in self.frontier_object_admission() { - let entry = preferred_object_admission_by_oid.entry(*oid).or_default(); for pack_id in pack_ids { let pack_id = crab_xet::hash::MerkleHash::from_hex(pack_id) .map_err(|error| corrupt_path("frontier admission", error.to_string()))?; - if !entry.contains(&pack_id) { - entry.push(pack_id); + if let Some(admission) = preferred_object_admission_by_oid.as_mut() { + let entry = admission.entry(*oid).or_insert_with(Vec::new); + if !entry.contains(&pack_id) { + entry.push(pack_id); + } } } } @@ -995,15 +982,18 @@ impl CapsuleRepositoryView { for (pack_id, object_ids) in capsule_run_member_oids_by_pack(self)? { let pack_id = crab_xet::hash::MerkleHash::from_hex(&pack_id) .map_err(|error| corrupt_path("capsule run admission", error.to_string()))?; - for oid in object_ids { - let entry = preferred_object_admission_by_oid.entry(oid).or_default(); - if !entry.contains(&pack_id) { - entry.push(pack_id); + if let Some(admission) = preferred_object_admission_by_oid.as_mut() { + for oid in object_ids { + let entry = admission.entry(oid).or_insert_with(Vec::new); + if !entry.contains(&pack_id) { + entry.push(pack_id); + } } } } let admitted_pack_ids = preferred_object_admission_by_oid - .values() + .iter() + .flat_map(|admission| admission.values()) .flat_map(|pack_ids| pack_ids.iter().map(ToString::to_string)) .collect::>(); let frontier_pack_ids = self @@ -1197,9 +1187,10 @@ impl CapsuleRepositoryView { if let Some(index) = object_index { lookup_sources = lookup_sources.with_object_index(index); } - if !preferred_object_admission_by_oid.is_empty() { - lookup_sources = - lookup_sources.with_preferred_object_admission(preferred_object_admission_by_oid); + if let Some(admission) = preferred_object_admission_by_oid + && !admission.is_empty() + { + lookup_sources = lookup_sources.with_preferred_object_admission(admission); } crab_remote_git::RemoteGitRepository::from_snapshot_with_lookup_sources( layout, From 034891124a334a09a1a676441684c5aedcff61ca Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 05:40:40 -0700 Subject: [PATCH 020/127] feat(metadata): page exact run admission Commit OID/member and dense count page roots in run footer version 2. Full readers verify both directories and canonical bytes before exposure. Retire unshipped version 1 without a compatibility reader. Keep pure codecs independent of storage features and meter admission pages. Lazy opening remains a dependent slice. --- .../docs/architecture/indexed-remote-reads.md | 33 ++ crates/crab-metadata/README.md | 12 +- .../crab-metadata/src/capsule_protocol/run.rs | 244 ++---------- .../src/capsule_protocol/run/admission.rs | 375 ++++++++++++++++++ crates/crab-metadata/src/pages.rs | 46 +-- crates/crab-remote/examples/support/mod.rs | 36 +- 6 files changed, 493 insertions(+), 253 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/run/admission.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 3c6dd4465..1d5257c79 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -767,3 +767,36 @@ These are shared-host debug observations, not a controlled release/provider latency result. Metadata transfer is unchanged. Complete visibility decoding, transaction reconciliation and source/ref inventory capture still scale; this cleanup does not satisfy bounded opening or finish stage 4. + + +## Paged run admission: format work in progress + +The frontier fixture opens from capsule runs, not a checkpoint. Its 10k transport +receipts contain capsule-control ranges and no `v2/checkpoints` reads. Paging only +checkpoint visibility would therefore miss the measured source of linear opening. + +Run footer/admission version 2 commits two existing-engine page trees: OID-to-member +lists and dense per-member object counts. Full readers authenticate the entire +sidecar, cross-check its page descriptor against the footer and reconstruct canonical +bytes, including counts. Member lists retain ordering, aliases and bounds; zero-count +members remain represented. Explicit page-kind discriminants prevent Cargo feature +selection from changing wire bytes. Pure page decoding remains payload-only; the +existing qualifier now counts capsule-admission page decoding separately. + +Twenty-one focused run/admission tests pass with default and storage/remote-index +features. Payload-only compilation, strict metadata Clippy, two metadata-view tests +and qualifier observer tests pass. Re-signing incorrect member counts remains an +error; removing canonical body/count comparison makes the exact negative-control +test fail, and restored source passes. The obsolete format-1 positive assertion is +removed; the unsupported-version test now explicitly rejects version 1. + +The introducing capsule commit has no containing release tag, consistent with the +recorded upstream release audit. Released v1 manifest/catalog formats are unchanged. +Retired run prefixes must be republished; no compatibility reader is added. An +isolated `make install` passes for FUSE/NFS/cache-server binaries without changing +global installations. Fresh-prefix RustFS proof passes 63 harness checks and 45 +commands: Crab pushes 100/1,000-commit fixtures, two index-maintenance processes +finish, and all 20 native-Git query processes pass. Source/binary hashes and receipts +remain external. This establishes a real publication/read smoke for the new format. +The slice still performs complete admission decoding and does not satisfy bounded +opening or finish the initiative. diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index a4ccf3a40..982bc7c76 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -191,9 +191,15 @@ suffix. Detached large visibility/catalog sections retain their separate bounded reads. The unshipped `CRBRUN04`/`CRBRUN05`/`CRBRUN06`/`CRBRUN07` formats are rejected, not read through a compatibility path; readers and writers must cut over together. -The version reset retains the latest run layout, including 500-capsule rollups, -pooled indexes and exact admission. Repositories containing retired runs must -be republished into a fresh repository prefix; no automatic migration is provided. +Run footer and admission version 2 replace their unshipped version-1 shapes. +Exact admission uses authenticated OID-to-member pages and dense per-member +count pages. Their roots are committed by the footer; complete readers verify +both directories and canonical body bytes before exposing admission. Page kind +codes are explicit, so payload-only and storage-enabled producers share one wire +format. The layout retains 500-capsule rollups and pooled indexes. Repositories +containing retired runs must be republished into a fresh repository prefix; +no automatic migration is provided. Control-suffix reads currently still load the +complete admission; lazy admission/visibility opening remains unfinished. Payload modules cover manifests, segmented lists, pack metadata, commit-graph summaries, ref registries, chunk/file indexes, receipts, transactions, and diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index 78c28be82..49cb4e306 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -11,15 +11,10 @@ use crate::error::{MetadataError, Result}; use crate::validation::validate_content_hash; const RUN_MAGIC: &[u8; 8] = b"CRBRUN01"; -const RUN_VERSION: u32 = 1; +const RUN_VERSION: u32 = 2; const RUN_TRAILER_BYTES: usize = 8 + 32 + RUN_MAGIC.len(); const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_INLINE_RUN_CONTROL_SECTION_BYTES: usize = 512 * 1024; -const ADMISSION_MAGIC: &[u8; 8] = b"CRBADM01"; -const ADMISSION_VERSION: u32 = 1; -const ADMISSION_HEADER_BYTES: usize = ADMISSION_MAGIC.len() + 4 + 4 + 8; -const MAX_RUN_ADMISSION_BYTES: usize = 128 * 1024 * 1024; -const MAX_RUN_ADMISSION_ENTRIES: usize = 8_000_000; /// Largest number of push capsules coalesced before a repository checkpoint. pub const MAX_CAPSULES_PER_RUN: usize = 512; @@ -75,203 +70,12 @@ struct CapsuleRunFooter { index_pool: Option, #[serde(default)] admission: Option, + admission_pages: Option, } -/// Exact object-to-member admission for one immutable capsule run. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CapsuleRunAdmission { - members: BTreeMap<[u8; 20], Vec>, -} - -impl CapsuleRunAdmission { - fn from_member_oids(member_oids: &[Vec<[u8; 20]>], member_count: usize) -> Result { - if member_oids.len() != member_count { - return Err(contract_error( - "capsule run admission member count does not match the pack directory", - )); - } - let mut members = BTreeMap::new(); - for (member_index, oids) in member_oids.iter().enumerate() { - let member_index = u32::try_from(member_index) - .map_err(|_| contract_error("capsule run admission member index overflowed"))?; - let mut oids = oids.clone(); - oids.sort_unstable(); - oids.dedup(); - for oid in oids { - members - .entry(oid) - .or_insert_with(Vec::new) - .push(member_index); - } - } - Ok(Self { members }) - } - - fn append(&mut self, newer: Self, member_offset: usize) -> Result<()> { - let member_offset = u32::try_from(member_offset) - .map_err(|_| contract_error("capsule run admission member offset overflowed"))?; - for (oid, newer_members) in newer.members { - let entry = self.members.entry(oid).or_default(); - for member in newer_members { - entry.push(member.checked_add(member_offset).ok_or_else(|| { - contract_error("capsule run admission member index overflowed") - })?); - } - } - Ok(()) - } - - fn encode(&self, member_count: usize) -> Result { - if self.members.len() > MAX_RUN_ADMISSION_ENTRIES { - return Err(contract_error("capsule run admission has too many objects")); - } - let member_count = u32::try_from(member_count) - .map_err(|_| contract_error("capsule run admission member count overflowed"))?; - let entry_count = u64::try_from(self.members.len()) - .map_err(|_| contract_error("capsule run admission object count overflowed"))?; - let mut bytes = Vec::with_capacity( - ADMISSION_HEADER_BYTES.saturating_add(self.members.len().saturating_mul(28)), - ); - bytes.extend_from_slice(ADMISSION_MAGIC); - bytes.extend_from_slice(&ADMISSION_VERSION.to_be_bytes()); - bytes.extend_from_slice(&member_count.to_be_bytes()); - bytes.extend_from_slice(&entry_count.to_be_bytes()); - let mut previous = None; - for (oid, members) in &self.members { - if members.is_empty() - || members.windows(2).any(|pair| pair[0] >= pair[1]) - || members.iter().any(|member| *member >= member_count) - { - return Err(contract_error( - "capsule run admission members are not canonical", - )); - } - if previous.is_some_and(|previous| previous >= *oid) { - return Err(contract_error( - "capsule run admission objects are not canonical", - )); - } - previous = Some(*oid); - bytes.extend_from_slice(oid); - let count = u32::try_from(members.len()) - .map_err(|_| contract_error("capsule run admission member list overflowed"))?; - bytes.extend_from_slice(&count.to_be_bytes()); - for member in members { - bytes.extend_from_slice(&member.to_be_bytes()); - } - } - if bytes.len() > MAX_RUN_ADMISSION_BYTES { - return Err(contract_error( - "capsule run admission exceeds its size bound", - )); - } - Ok(Bytes::from(bytes)) - } - - fn decode(bytes: &[u8], expected_member_count: usize) -> Result { - if bytes.len() < ADMISSION_HEADER_BYTES || bytes.len() > MAX_RUN_ADMISSION_BYTES { - return Err(corrupt("capsule run admission has an invalid size")); - } - let mut cursor = 0; - let take = |cursor: &mut usize, length: usize| -> Result<&[u8]> { - let end = cursor - .checked_add(length) - .ok_or_else(|| corrupt("capsule run admission offset overflowed"))?; - let bytes = bytes - .get(*cursor..end) - .ok_or_else(|| corrupt("capsule run admission is truncated"))?; - *cursor = end; - Ok(bytes) - }; - if take(&mut cursor, ADMISSION_MAGIC.len())? != ADMISSION_MAGIC { - return Err(corrupt("capsule run admission magic is invalid")); - } - let version = u32::from_be_bytes( - take(&mut cursor, 4)? - .try_into() - .map_err(|_| corrupt("capsule run admission version is truncated"))?, - ); - if version != ADMISSION_VERSION { - return Err(corrupt("capsule run admission version is unsupported")); - } - let member_count = u32::from_be_bytes( - take(&mut cursor, 4)? - .try_into() - .map_err(|_| corrupt("capsule run admission member count is truncated"))?, - ); - if usize::try_from(member_count).ok() != Some(expected_member_count) { - return Err(corrupt( - "capsule run admission member count does not match the pack directory", - )); - } - let entry_count = u64::from_be_bytes( - take(&mut cursor, 8)? - .try_into() - .map_err(|_| corrupt("capsule run admission object count is truncated"))?, - ); - let entry_count = usize::try_from(entry_count) - .map_err(|_| corrupt("capsule run admission object count overflows"))?; - if entry_count > MAX_RUN_ADMISSION_ENTRIES { - return Err(corrupt("capsule run admission has too many objects")); - } - let mut members = BTreeMap::new(); - let mut previous = None; - for _ in 0..entry_count { - let oid: [u8; 20] = take(&mut cursor, 20)? - .try_into() - .map_err(|_| corrupt("capsule run admission object ID is truncated"))?; - if previous.is_some_and(|previous| previous >= oid) { - return Err(corrupt("capsule run admission objects are not sorted")); - } - previous = Some(oid); - let member_count = u32::from_be_bytes( - take(&mut cursor, 4)? - .try_into() - .map_err(|_| corrupt("capsule run admission member list is truncated"))?, - ); - if member_count == 0 { - return Err(corrupt("capsule run admission has an empty member list")); - } - let mut oid_members = Vec::with_capacity( - usize::try_from(member_count) - .map_err(|_| corrupt("capsule run admission member list overflows"))?, - ); - for _ in 0..member_count { - let member = u32::from_be_bytes( - take(&mut cursor, 4)? - .try_into() - .map_err(|_| corrupt("capsule run admission member is truncated"))?, - ); - if member >= u32::try_from(expected_member_count).unwrap_or(u32::MAX) - || oid_members - .last() - .is_some_and(|previous| *previous >= member) - { - return Err(corrupt("capsule run admission members are invalid")); - } - oid_members.push(member); - } - members.insert(oid, oid_members); - } - if cursor != bytes.len() { - return Err(corrupt("capsule run admission has trailing bytes")); - } - Ok(Self { members }) - } - - /// Return the member ordinals that contain one Git object. - #[must_use] - pub fn object_members(&self, oid: &[u8; 20]) -> Option<&[u32]> { - self.members.get(oid).map(Vec::as_slice) - } - - /// Return every admitted object and the run members that contain it. - pub fn entries(&self) -> impl Iterator { - self.members - .iter() - .map(|(oid, members)| (oid, members.as_slice())) - } -} +mod admission; +use admission::AdmissionPages; +pub use admission::CapsuleRunAdmission; /// Immutable bounded run of complete push capsules. #[derive(Debug, Clone, PartialEq, Eq)] @@ -443,6 +247,11 @@ impl CapsuleRunControl { } // The pointer covers admission and footer together. Validate the exact // boundary and admission hash before exposing any physical object hints. + if footer.admission.is_some() != footer.admission_pages.is_some() { + return Err(corrupt( + "capsule run admission descriptor presence differs from its body", + )); + } let admission = if let Some(range) = footer.admission.as_ref() { let footer_offset = control_offset .checked_add(footer_start as u64) @@ -452,6 +261,11 @@ impl CapsuleRunControl { if blake3::hash(admission_bytes).to_hex().as_str() != range.hash { return Err(corrupt("capsule run admission hash does not match")); } + if footer.admission_pages.as_ref() != Some(&AdmissionPages::decode(admission_bytes)?) { + return Err(corrupt( + "capsule run admission descriptor differs from its footer", + )); + } Some(CapsuleRunAdmission::decode( admission_bytes, footer.git_packs.len(), @@ -867,6 +681,13 @@ impl CapsuleRun { git_packs: git_packs.clone(), index_pool: index_pool.clone(), admission: admission.as_ref().map(|(_, range)| range.clone()), + admission_pages: admission + .as_ref() + .map(|(_, range)| { + let start = range.offset as usize; + AdmissionPages::decode(&body[start..start + range.length as usize]) + }) + .transpose()?, }; let footer_bytes = serde_json::to_vec(&footer).map_err(|source| { MetadataError::Internal(format!("capsule run serialization failed: {source}")) @@ -1007,6 +828,11 @@ impl CapsuleRun { } } expected_offset = pool_end; + if footer.admission.is_some() != footer.admission_pages.is_some() { + return Err(corrupt( + "capsule run admission descriptor presence differs from its body", + )); + } let admission = if let Some(admission) = footer.admission.as_ref() { validate_body_range(admission, expected_offset, footer_start as u64)?; let end = admission @@ -1024,6 +850,11 @@ impl CapsuleRun { return Err(corrupt("capsule run admission hash does not match")); } expected_offset = end as u64; + if footer.admission_pages.as_ref() != Some(&AdmissionPages::decode(bytes)?) { + return Err(corrupt( + "capsule run admission descriptor differs from its footer", + )); + } Some(CapsuleRunAdmission::decode(bytes, expected_packs.len())?) } else { None @@ -1750,15 +1581,6 @@ mod tests { } } - #[test] - fn run_serializes_format_one() { - let run = CapsuleRun::leaf(capsule('1', '2')).unwrap(); - assert_eq!( - (&run.bytes()[run.bytes().len() - 8..], run.footer.version), - (&b"CRBRUN01"[..], 1) - ); - } - #[test] fn retired_run_magic_is_rejected() { let run = CapsuleRun::leaf(capsule('1', '2')).unwrap(); @@ -1785,7 +1607,7 @@ mod tests { #[test] fn unsupported_run_footer_versions_are_rejected() { let run = CapsuleRun::leaf(capsule('1', '2')).unwrap(); - for version in [0, 2, 3, 4, 5, 6, 7] { + for version in [0, 1, 3, 4, 5, 6, 7] { let mut footer = run.footer.clone(); footer.version = version; let footer = serde_json::to_vec(&footer).unwrap(); diff --git a/crates/crab-metadata/src/capsule_protocol/run/admission.rs b/crates/crab-metadata/src/capsule_protocol/run/admission.rs new file mode 100644 index 000000000..f0669086d --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/run/admission.rs @@ -0,0 +1,375 @@ +//! Exact object admission and authenticated per-member counts for one run. +use super::{contract_error, corrupt}; +use crate::error::Result; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +const ADMISSION_MAGIC: &[u8; 8] = b"CRBADM01"; +const ADMISSION_VERSION: u32 = 2; +const ADMISSION_TRAILER_BYTES: usize = 4; +const MAX_RUN_ADMISSION_BYTES: usize = 128 * 1024 * 1024; +const MAX_RUN_ADMISSION_ENTRIES: usize = 8_000_000; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct AdmissionPages { + member_count: u32, + body_bytes: u64, + body_hash: [u8; 32], + objects: crate::pages::PageRef, + counts: crate::pages::PageRef, +} + +impl AdmissionPages { + fn validate(&self) -> Result<()> { + use crate::pages::Kind; + if self.member_count == 0 + || self.body_bytes == 0 + || self.body_bytes > MAX_RUN_ADMISSION_BYTES as u64 + || self.objects.count as usize > MAX_RUN_ADMISSION_ENTRIES + { + return Err(corrupt("capsule run admission has an invalid count")); + } + self.objects.validate(Kind::RunAdmission, self.body_bytes)?; + self.counts + .validate(Kind::RunAdmissionCounts, self.body_bytes)?; + if self.counts.count != self.member_count + || self.counts.first != 0u32.to_be_bytes() + || self.counts.last != (self.member_count - 1).to_be_bytes() + { + return Err(corrupt( + "capsule run admission member counts have incomplete coverage", + )); + } + Ok(()) + } + + fn header(&self) -> Vec { + let mut bytes = ADMISSION_MAGIC.to_vec(); + bytes.extend_from_slice(&ADMISSION_VERSION.to_be_bytes()); + bytes.extend_from_slice(&self.member_count.to_be_bytes()); + bytes.extend_from_slice(&self.body_hash); + crate::pages::encode_ref(&mut bytes, &self.objects); + crate::pages::encode_ref(&mut bytes, &self.counts); + bytes + } + + pub(super) fn decode(bytes: &[u8]) -> Result { + use crate::pages::{Input, Kind}; + if bytes.len() < ADMISSION_TRAILER_BYTES || bytes.len() > MAX_RUN_ADMISSION_BYTES { + return Err(corrupt("capsule run admission has an invalid size")); + } + let end = bytes.len() - ADMISSION_TRAILER_BYTES; + let size = u32::from_be_bytes( + bytes[end..] + .try_into() + .map_err(|_| corrupt("capsule run admission trailer is invalid"))?, + ) as usize; + let start = end + .checked_sub(size) + .ok_or_else(|| corrupt("capsule run admission header starts outside its body"))?; + let mut header = Input::new(&bytes[start..end], "capsule run admission header"); + if header.take(ADMISSION_MAGIC.len())? != ADMISSION_MAGIC + || u32::from_be_bytes(header.array()?) != ADMISSION_VERSION + { + return Err(corrupt( + "capsule run admission magic or version is unsupported", + )); + } + let member_count = u32::from_be_bytes(header.array()?); + let body_hash = header.array()?; + let objects = crate::pages::decode_ref(&mut header, Kind::RunAdmission)?; + let counts = crate::pages::decode_ref(&mut header, Kind::RunAdmissionCounts)?; + header.finish()?; + let result = Self { + member_count, + body_bytes: start as u64, + body_hash, + objects, + counts, + }; + result.validate()?; + Ok(result) + } +} + +/// Exact object-to-member admission for one immutable capsule run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CapsuleRunAdmission { + pub(super) members: BTreeMap<[u8; 20], Vec>, +} + +impl CapsuleRunAdmission { + pub(super) fn from_member_oids( + member_oids: &[Vec<[u8; 20]>], + member_count: usize, + ) -> Result { + if member_oids.len() != member_count { + return Err(contract_error( + "capsule run admission member count does not match the pack directory", + )); + } + let mut members = BTreeMap::new(); + for (member_index, oids) in member_oids.iter().enumerate() { + let member_index = u32::try_from(member_index) + .map_err(|_| contract_error("capsule run admission member index overflowed"))?; + let mut oids = oids.clone(); + oids.sort_unstable(); + oids.dedup(); + for oid in oids { + members + .entry(oid) + .or_insert_with(Vec::new) + .push(member_index); + } + } + Ok(Self { members }) + } + + pub(super) fn append(&mut self, newer: Self, member_offset: usize) -> Result<()> { + let member_offset = u32::try_from(member_offset) + .map_err(|_| contract_error("capsule run admission member offset overflowed"))?; + for (oid, newer_members) in newer.members { + let entry = self.members.entry(oid).or_default(); + for member in newer_members { + entry.push(member.checked_add(member_offset).ok_or_else(|| { + contract_error("capsule run admission member index overflowed") + })?); + } + } + Ok(()) + } + + pub(super) fn encode(&self, member_count: usize) -> Result { + use crate::pages::{Kind, Writer}; + if self.members.is_empty() + || self.members.len() > MAX_RUN_ADMISSION_ENTRIES + || member_count == 0 + { + return Err(contract_error("capsule run admission has an invalid count")); + } + let member_count = u32::try_from(member_count) + .map_err(|_| contract_error("capsule run admission member count overflowed"))?; + let minimum_bytes = self + .members + .len() + .checked_mul(28) + .and_then(|bytes| { + self.members.values().try_fold(bytes, |total, members| { + total.checked_add(members.len().checked_mul(4)?) + }) + }) + .and_then(|bytes| bytes.checked_add((member_count as usize).checked_mul(16)?)); + if minimum_bytes.is_none_or(|bytes| bytes > MAX_RUN_ADMISSION_BYTES) { + return Err(contract_error( + "capsule run admission exceeds its byte bound", + )); + } + let mut counts = vec![0u64; member_count as usize]; + let mut rows = Vec::with_capacity(self.members.len()); + for (oid, members) in &self.members { + if members.is_empty() || members.windows(2).any(|pair| pair[0] >= pair[1]) { + return Err(contract_error( + "capsule run admission members are not canonical", + )); + } + let mut value = Vec::new(); + crate::pages::put_u32(&mut value, members.len())?; + for member in members { + let count = counts.get_mut(*member as usize).ok_or_else(|| { + contract_error("capsule run admission references an absent member") + })?; + *count += 1; + value.extend_from_slice(&member.to_le_bytes()); + } + rows.push((oid.to_vec(), value)); + } + let mut writer = Writer::default(); + let objects = writer.tree(Kind::RunAdmission, rows)?; + let counts = writer.tree( + Kind::RunAdmissionCounts, + counts + .into_iter() + .enumerate() + .map(|(index, count)| { + ( + (index as u32).to_be_bytes().to_vec(), + count.to_le_bytes().to_vec(), + ) + }) + .collect(), + )?; + let pages = AdmissionPages { + member_count, + body_bytes: writer.bytes.len() as u64, + body_hash: *blake3::hash(&writer.bytes).as_bytes(), + objects, + counts, + }; + pages.validate()?; + let header = pages.header(); + let header_len = u32::try_from(header.len()) + .map_err(|_| contract_error("capsule run admission header size overflows"))?; + writer.bytes.extend_from_slice(&header); + writer.bytes.extend_from_slice(&header_len.to_be_bytes()); + if writer.bytes.len() > MAX_RUN_ADMISSION_BYTES { + return Err(contract_error( + "capsule run admission exceeds its byte bound", + )); + } + Ok(writer.bytes.into()) + } + + pub(super) fn decode(bytes: &[u8], expected_member_count: usize) -> Result { + use crate::pages::{Input, Kind}; + let pages = AdmissionPages::decode(bytes)?; + if pages.member_count as usize != expected_member_count { + return Err(corrupt( + "capsule run admission member count does not match the pack directory", + )); + } + let body = &bytes[..pages.body_bytes as usize]; + if blake3::hash(body).as_bytes() != &pages.body_hash { + return Err(corrupt( + "capsule run admission page body hash does not match", + )); + } + let mut audit = BTreeMap::new(); + let rows = crate::pages::scan( + body, + &pages.objects, + Kind::RunAdmission, + "capsule run admission", + &mut audit, + )?; + let mut members = BTreeMap::new(); + for (oid, value) in rows { + let mut input = Input::new(&value, "capsule run admission members"); + let count = input.u32()? as usize; + if count == 0 || count > expected_member_count || input.remaining() / 4 != count { + return Err(corrupt("capsule run admission has an invalid member list")); + } + let mut list = Vec::with_capacity(count); + for _ in 0..count { + let member = input.u32()?; + if member as usize >= expected_member_count + || list.last().is_some_and(|previous| *previous >= member) + { + return Err(corrupt("capsule run admission members are invalid")); + } + list.push(member); + } + input.finish()?; + let oid = oid + .try_into() + .map_err(|_| corrupt("capsule run admission OID has an invalid length"))?; + members.insert(oid, list); + } + crate::pages::scan( + body, + &pages.counts, + Kind::RunAdmissionCounts, + "capsule run admission", + &mut audit, + )?; + let result = Self { members }; + if result.encode(expected_member_count)?.as_ref() != bytes { + return Err(corrupt( + "capsule run admission pages are not canonical or counts disagree", + )); + } + Ok(result) + } + + /// Return the member ordinals that contain one Git object. + #[must_use] + pub fn object_members(&self, oid: &[u8; 20]) -> Option<&[u32]> { + self.members.get(oid).map(Vec::as_slice) + } + + /// Return every admitted object and the run members that contain it. + pub fn entries(&self) -> impl Iterator { + self.members + .iter() + .map(|(oid, members)| (oid, members.as_slice())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::pages::{Kind, Writer}; + + #[test] + fn admission_pages_preserve_aliases_and_empty_member_counts() { + let inputs = [vec![[1; 20], [2; 20], [1; 20]], Vec::new(), vec![[1; 20]]]; + let admission = CapsuleRunAdmission::from_member_oids(&inputs, 3).unwrap(); + let bytes = admission.encode(3).unwrap(); + let pages = AdmissionPages::decode(&bytes).unwrap(); + let counts = crate::pages::scan( + &bytes[..pages.body_bytes as usize], + &pages.counts, + Kind::RunAdmissionCounts, + "test", + &mut BTreeMap::new(), + ) + .unwrap(); + assert_eq!( + counts + .into_iter() + .map(|(_, value)| u64::from_le_bytes(value.try_into().unwrap())) + .collect::>(), + [2, 0, 1] + ); + assert_eq!(CapsuleRunAdmission::decode(&bytes, 3).unwrap(), admission); + } + + #[test] + fn resigned_member_counts_cannot_disagree_with_object_admission() { + let admission = CapsuleRunAdmission::from_member_oids(&[vec![[1; 20]]], 1).unwrap(); + let bytes = admission.encode(1).unwrap(); + let pages = AdmissionPages::decode(&bytes).unwrap(); + let objects = crate::pages::scan( + &bytes[..pages.body_bytes as usize], + &pages.objects, + Kind::RunAdmission, + "test", + &mut BTreeMap::new(), + ) + .unwrap(); + let mut writer = Writer::default(); + let objects = writer.tree(Kind::RunAdmission, objects).unwrap(); + let counts = writer + .tree( + Kind::RunAdmissionCounts, + vec![(0u32.to_be_bytes().to_vec(), 2u64.to_le_bytes().to_vec())], + ) + .unwrap(); + let pages = AdmissionPages { + member_count: 1, + body_bytes: writer.bytes.len() as u64, + body_hash: *blake3::hash(&writer.bytes).as_bytes(), + objects, + counts, + }; + let header = pages.header(); + writer.bytes.extend_from_slice(&header); + writer + .bytes + .extend_from_slice(&(header.len() as u32).to_be_bytes()); + assert!(CapsuleRunAdmission::decode(&writer.bytes, 1).is_err()); + } + + proptest::proptest! { + #[test] + fn admission_input_order_does_not_change_page_bytes( + oids in proptest::collection::vec(proptest::array::uniform20(proptest::num::u8::ANY), 1..100), + ) { + let mut reversed = oids.clone(); reversed.reverse(); + let left = CapsuleRunAdmission::from_member_oids(&[oids], 1).unwrap().encode(1).unwrap(); + let right = CapsuleRunAdmission::from_member_oids(&[reversed], 1).unwrap().encode(1).unwrap(); + proptest::prop_assert_eq!(left, right); + } + } +} diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index f78a4a0e0..9c733711c 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -7,14 +7,12 @@ mod cache; pub(crate) use cache::{PageCache, PageWork}; use crate::error::{MetadataError, Result}; -#[cfg(any(feature = "storage", test))] use crate::validation::corrupt_object; fn corruption(reason: &str) -> MetadataError { crate::validation::corrupt_object("metadata page", reason) } -#[cfg(any(feature = "storage", test))] fn corrupt_at(path: &str, reason: &str) -> Result { Err(corrupt_object(path, reason)) } @@ -41,28 +39,34 @@ pub struct PageRef { #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] pub(crate) enum Kind { - Records, - Nodes, - Children, - Names, - CommitRecords, - CommitOids, - CommitAncestryRoots, - CommitAncestryNodes, + Records = 0, + Nodes = 1, + Children = 2, + Names = 3, + CommitRecords = 4, + CommitOids = 5, + CommitAncestryRoots = 6, + CommitAncestryNodes = 7, #[cfg(feature = "remote-index")] - ObjectLocators, + ObjectLocators = 8, #[cfg(feature = "remote-index")] - PackOffsets, + PackOffsets = 9, #[cfg(feature = "remote-index")] - PackInventory, + PackInventory = 10, #[cfg(feature = "remote-index")] - PackSlots, + PackSlots = 11, + RunAdmission = 12, + RunAdmissionCounts = 13, } impl Kind { fn is_positional(self) -> bool { match self { - Self::Records | Self::Nodes | Self::CommitRecords | Self::CommitAncestryRoots => true, + Self::Records + | Self::Nodes + | Self::CommitRecords + | Self::CommitAncestryRoots + | Self::RunAdmissionCounts => true, #[cfg(feature = "remote-index")] Self::PackSlots => true, _ => false, @@ -72,7 +76,7 @@ impl Kind { pub(crate) fn key_bytes(self) -> usize { match self { Self::Names | Self::CommitAncestryNodes => 32, - Self::CommitOids => 20, + Self::CommitOids | Self::RunAdmission => 20, #[cfg(feature = "remote-index")] Self::PackInventory => 32, #[cfg(feature = "remote-index")] @@ -85,7 +89,6 @@ impl Kind { } pub(crate) type Rows = Vec<(Vec, Vec)>; -#[cfg(any(feature = "storage", test))] #[derive(Debug)] pub(crate) enum Page { Leaf(Rows), @@ -250,7 +253,6 @@ fn header(kind: Kind, height: u8, count: u32) -> Vec { bytes } -#[cfg(any(feature = "storage", test))] pub(crate) fn decode( bytes: &[u8], reference: &PageRef, @@ -339,13 +341,13 @@ pub(crate) fn decode( Kind::CommitRecords | Kind::CommitOids | Kind::CommitAncestryRoots | Kind::CommitAncestryNodes => "commit_graph", #[cfg(feature = "remote-index")] Kind::ObjectLocators | Kind::PackOffsets | Kind::PackInventory | Kind::PackSlots => "git_object_index", + Kind::RunAdmission | Kind::RunAdmissionCounts => "capsule_admission", _ => "path_state", }, decoded_bytes=bytes.len() as u64, "index page decoded"); Ok(page) } -#[cfg(any(feature = "storage", test))] pub(crate) fn scan( source: &[u8], root: &PageRef, @@ -374,7 +376,6 @@ pub(crate) fn scan( Ok(rows) } -#[cfg(any(feature = "storage", test))] pub(crate) fn page_bytes<'a>( source: &'a [u8], reference: &PageRef, @@ -431,7 +432,6 @@ pub(crate) fn encode_ref(bytes: &mut Vec, reference: &PageRef) { bytes.extend_from_slice(&reference.last); } -#[cfg(any(feature = "storage", test))] pub(crate) fn decode_ref(input: &mut Input<'_, '_>, kind: Kind) -> Result { Ok(PageRef { offset: u64::from_le_bytes(input.array()?), @@ -453,12 +453,10 @@ pub(crate) fn put_u32(bytes: &mut Vec, value: usize) -> Result<()> { Ok(()) } -#[cfg(any(feature = "storage", test))] pub(crate) struct Input<'a, 'p> { bytes: &'a [u8], path: &'p str, } -#[cfg(any(feature = "storage", test))] impl<'a, 'p> Input<'a, 'p> { pub(crate) fn new(bytes: &'a [u8], path: &'p str) -> Self { Self { bytes, path } @@ -490,7 +488,7 @@ impl<'a, 'p> Input<'a, 'p> { } } -#[cfg(any(feature = "storage", test))] +#[cfg(feature = "storage")] impl Page { pub(crate) fn resident_bytes(&self) -> usize { match self { diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index 6365baba1..4463f3d60 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -25,8 +25,8 @@ pub struct Measurements { read_bytes: AtomicU64, written_bytes: AtomicU64, active: AtomicU64, - decoded_blocks: [AtomicU64; 3], - decoded_bytes: [AtomicU64; 3], + decoded_blocks: [AtomicU64; 4], + decoded_bytes: [AtomicU64; 4], } #[derive(Clone, Copy)] @@ -35,8 +35,8 @@ struct Snapshot { outcomes: [u64; 9], read_bytes: u64, written_bytes: u64, - decoded_blocks: [u64; 3], - decoded_bytes: [u64; 3], + decoded_blocks: [u64; 4], + decoded_bytes: [u64; 4], } impl StorageObserver for Measurements { @@ -71,6 +71,7 @@ impl tracing::field::Visit for DecodeEvent { "commit_graph" => Some(0), "path_state" => Some(1), "git_object_index" => Some(2), + "capsule_admission" => Some(3), _ => None, }; } @@ -160,16 +161,20 @@ impl Measurements { ) }) .collect::>(); - let decoded = ["commit_graph", "path_state", "git_object_index"] - .into_iter() - .enumerate() - .map(|(index, name)| { - let blocks = - after.decoded_blocks[index].saturating_sub(before.decoded_blocks[index]); - let bytes = after.decoded_bytes[index].saturating_sub(before.decoded_bytes[index]); - (name.to_owned(), json!({"blocks":blocks, "bytes":bytes})) - }) - .collect::>(); + let decoded = [ + "commit_graph", + "path_state", + "git_object_index", + "capsule_admission", + ] + .into_iter() + .enumerate() + .map(|(index, name)| { + let blocks = after.decoded_blocks[index].saturating_sub(before.decoded_blocks[index]); + let bytes = after.decoded_bytes[index].saturating_sub(before.decoded_bytes[index]); + (name.to_owned(), json!({"blocks":blocks, "bytes":bytes})) + }) + .collect::>(); println!( "{}", json!({ @@ -236,12 +241,13 @@ mod tests { tracing::debug!(target:"crab_metadata::index_decode", index_format="commit_graph", decoded_bytes=41_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="path_state", decoded_bytes=123_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="git_object_index", decoded_bytes=67_u64); + tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_admission", decoded_bytes=29_u64); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), - ([1, 1, 1], [41, 123, 67]) + ([1, 1, 1, 1], [41, 123, 67, 29]) ); } From 48f5b1ae631a2ec19058e8c4ebc074bebe1e49ad Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 06:22:39 -0700 Subject: [PATCH 021/127] perf(read): keep run OID admission cold --- .../docs/architecture/indexed-remote-reads.md | 38 ++++ crab/src/read/mod.rs | 18 +- crates/crab-http-server/src/projection.rs | 2 +- crates/crab-http-server/src/server.rs | 2 +- crates/crab-metadata/README.md | 17 +- .../crab-metadata/src/capsule_protocol/mod.rs | 9 +- .../crab-metadata/src/capsule_protocol/run.rs | 184 +++++++++++++++--- .../src/capsule_protocol/run/admission.rs | 58 ++++-- .../capsule_protocol/run/admission/reader.rs | 160 +++++++++++++++ .../src/capsule_protocol/store.rs | 41 ++++ .../src/capsule_protocol/store/tests.rs | 115 +++++++++++ .../src/git_object_locator/paged/reader.rs | 2 +- crates/crab-metadata/src/pages/cache.rs | 38 ++-- crates/crab-metadata/src/path_state/reader.rs | 10 +- .../src/split_commit_graph/reader.rs | 8 +- crates/crab-read/README.md | 9 + crates/crab-read/src/capsule_protocol.rs | 88 ++++++++- crates/crab-remote/examples/support/mod.rs | 2 +- crates/crab-s3-gateway/src/repository.rs | 2 +- 19 files changed, 706 insertions(+), 97 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 1d5257c79..076c4ece4 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -800,3 +800,41 @@ finish, and all 20 native-Git query processes pass. Source/binary hashes and rec remain external. This establishes a real publication/read smoke for the new format. The slice still performs complete admission decoding and does not satisfy bounded opening or finish the initiative. + +## Git control opening with cold OID admission + +Git read consumers now use `open_git_view_from_root`. Metadata authenticates the +run footer against its captured pointer and validates paged member counts against +the pack directory. It keeps the OID admission body cold and omits eager frontier +placement-hint dictionaries. Complete maintenance/publication readers still verify +the entire admission sidecar and its canonical trees. Reference reconciliation, +visibility edits, catalog precedence and source/member ranges retain their owners. +The wire format is unchanged from the preceding version-2 format slice. + +`CapsuleRunAdmissionReader` provides duplicate-preserving batched exact hints and +authenticated misses. Both directories share one encoded-page budget; warm pages +remain charged. The shared cache now accepts embedded body ranges and keys pages +by absolute source offset. Graph, path-state and physical-index consumers retain +their zero-based body ranges. Page and object authentication remain mandatory; +placement membership never authorizes reference visibility. + +Five metadata-store tests, 21 payload-only run tests and the Git/complete view +equivalence test pass. The shared pager's 21 path-state, 27 graph and 15 physical +index tests pass. Metadata/read strict Clippy, frontend build, CLI/HTTP/S3 consumer +checks and qualifier builds pass. Fault injection leaves unrequested OID damage +cold during opening; exact lookup and complete inspection reject that damage. + +The final-source RustFS replay reuses the preceding format slice's immutable +100/1,000-commit fixtures. It passes 41 qualifier processes: all ten supported +queries at 100 commits and 20 cold/warm resolve samples per history. Native Git +oracles pass, warm resolve reads fetch zero backend bytes, and every measurement +ends without active backend work. Opening reads 78,142/471,334 backend bytes and +decodes one admission count page at 100/1,000 commits. Source/binary hashes, +transport receipts and results remain external; frozen fixtures are unchanged. + +This removes full run-OID decoding from these read opens, not all linear opening +work. Checkpoint visibility/catalog decoding, ref/source inventories and transaction +reconciliation remain eager. The unindexed pack-index path remains existing product +behavior. HTTP/S3 service E2E, 10k version-2 scaling and representative-provider +latency qualification are still missing for this slice. Stages 5–8 and unified +native/Git staging remain separate work. The PR stays draft and is not merged. diff --git a/crab/src/read/mod.rs b/crab/src/read/mod.rs index 9126f54bc..93dfa94bd 100644 --- a/crab/src/read/mod.rs +++ b/crab/src/read/mod.rs @@ -568,17 +568,15 @@ impl Inner { remote.router.global_prefix().to_owned(), ); let root = crab_metadata::capsule_protocol::load_root(&layout).await?; - Ok( - crab_read::capsule_protocol::open_view_from_root_with_metadata( - &layout, - root, - crab_read::capsule_protocol::CapsuleReadLimits { - max_capsule_bytes: maximum, - max_frontier_bytes: maximum, - }, - ) - .await?, + Ok(crab_read::capsule_protocol::open_git_view_from_root( + &layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: maximum, + max_frontier_bytes: maximum, + }, ) + .await?) } } diff --git a/crates/crab-http-server/src/projection.rs b/crates/crab-http-server/src/projection.rs index 66b12371b..e033400c2 100644 --- a/crates/crab-http-server/src/projection.rs +++ b/crates/crab-http-server/src/projection.rs @@ -790,7 +790,7 @@ pub(crate) async fn open_view( .map_err(metadata_error)?; // Origin-backed readers need complete visibility controls, while Git pack // bodies remain in their authenticated sources for bounded range reads. - let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( + let view = crab_read::capsule_protocol::open_git_view_from_root( layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-http-server/src/server.rs b/crates/crab-http-server/src/server.rs index cf09de03c..0086214bf 100644 --- a/crates/crab-http-server/src/server.rs +++ b/crates/crab-http-server/src/server.rs @@ -792,7 +792,7 @@ impl Repository { .map_err(|source| crate::Error::Settings { source: Box::new(source), })?; - let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( + let view = crab_read::capsule_protocol::open_git_view_from_root( &self.layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 982bc7c76..76151326c 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -169,7 +169,7 @@ capsules and encodes/authenticates the final run once. Run level is the `ceil(log2(count))` size class; the authenticated capsule count remains exact. Mixed-level carries retain canonical capsule bytes and member ordinals without encoding discarded intermediate runs; complete capsule verification remains -mandatory. The v2 cutover uses CRBRUN01 with footer version 1 and +mandatory. The v2 cutover uses CRBRUN01 with footer version 2 and root/history/ref-head versions 4/3/5; there is no reader fallback to earlier run or pointer semantics. Runs may contain byte-identical packs from different transactions. Their source @@ -184,9 +184,10 @@ unchanged for recovery, installation and repack. Control-only readers derive pool ranges in member order with the original index hashes; full decoding checks the pool and every copied index. Leaves do not duplicate indexes. The run's authenticated control suffix covers its exact member-admission bytes together -with the footer, so control loading needs no second admission request. The -decoder verifies the entire admission hash and exact suffix boundary before -exposing placement hints. Payload bytes and the lookup pool remain outside the +with the footer. Complete control loading verifies the entire admission hash +and exact suffix boundary before exposing placement hints. Git control loading +authenticates the footer and reads member-count pages while keeping the OID +directory cold. Payload bytes and the lookup pool remain outside the suffix. Detached large visibility/catalog sections retain their separate bounded reads. The unshipped `CRBRUN04`/`CRBRUN05`/`CRBRUN06`/`CRBRUN07` formats are rejected, not read through a compatibility path; readers and writers must cut @@ -198,8 +199,12 @@ both directories and canonical body bytes before exposing admission. Page kind codes are explicit, so payload-only and storage-enabled producers share one wire format. The layout retains 500-capsule rollups and pooled indexes. Repositories containing retired runs must be republished into a fresh repository prefix; -no automatic migration is provided. Control-suffix reads currently still load the -complete admission; lazy admission/visibility opening remains unfinished. +no automatic migration is provided. `load_capsule_run_git_control` preserves +complete reference controls and verifies counts against the pack directory. +Its paged admission reader returns authenticated misses and preserves request +order and duplicates; cached and fetched pages share one encoded-work budget. +Complete inspection still scans both admission trees. Complete visibility and +catalog decoding and source/ref capture remain separate opening costs. Payload modules cover manifests, segmented lists, pack metadata, commit-graph summaries, ref registries, chunk/file indexes, receipts, transactions, and diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 33b33ebf6..29e8c1b34 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -52,15 +52,18 @@ pub use root::{ CapsulePointer, CheckpointPointer, GcFence, MAX_CAPSULE_FRONTIER, MAX_ROOT_BYTES, RepositoryRoot, RootRecord, }; +#[cfg(feature = "storage")] +pub use run::CapsuleRunAdmissionReader; pub use run::{ CapsuleControl, CapsuleControlLocation, CapsuleRun, CapsuleRunAdmission, CapsuleRunControl, MAX_CAPSULES_PER_RUN, }; #[cfg(feature = "storage")] pub use store::{ - RootSnapshot, create_root, load_capsule_run, load_capsule_run_control, load_history_chain, - load_history_segment, load_layered_checkpoint, load_layered_checkpoint_control, - load_pointer_catalog, load_pointer_catalog_from_root, load_root, + RootSnapshot, create_root, load_capsule_run, load_capsule_run_control, + load_capsule_run_git_control, load_history_chain, load_history_segment, + load_layered_checkpoint, load_layered_checkpoint_control, load_pointer_catalog, + load_pointer_catalog_from_root, load_root, }; pub use transaction::{CapsuleRefEdit, CapsuleTransaction}; pub use transaction_record::{ diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index 49cb4e306..5f774d508 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -13,7 +13,7 @@ use crate::validation::validate_content_hash; const RUN_MAGIC: &[u8; 8] = b"CRBRUN01"; const RUN_VERSION: u32 = 2; const RUN_TRAILER_BYTES: usize = 8 + 32 + RUN_MAGIC.len(); -const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; +pub(super) const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_INLINE_RUN_CONTROL_SECTION_BYTES: usize = 512 * 1024; /// Largest number of push capsules coalesced before a repository checkpoint. pub const MAX_CAPSULES_PER_RUN: usize = 512; @@ -76,6 +76,8 @@ struct CapsuleRunFooter { mod admission; use admission::AdmissionPages; pub use admission::CapsuleRunAdmission; +#[cfg(feature = "storage")] +pub use admission::CapsuleRunAdmissionReader; /// Immutable bounded run of complete push capsules. #[derive(Debug, Clone, PartialEq, Eq)] @@ -102,6 +104,7 @@ pub struct CapsuleRunControl { git_packs: Vec, index_pool: Option, admission: Option, + admission_info: Option<(RunSectionLocation, AdmissionPages)>, } /// Ranges for the transaction and optional metadata sections of one capsule. @@ -132,7 +135,7 @@ pub struct CapsuleControl { } impl CapsuleRunControl { - /// Decode an authenticated run footer from its control suffix. + /// Decode complete control and admission bytes from the authenticated suffix. pub fn decode_suffix( bytes: Bytes, object_size: u64, @@ -140,6 +143,101 @@ impl CapsuleRunControl { expected_level: u8, expected_transactions: &[String], expected_newest_base: &str, + ) -> Result { + let offset = object_size + .checked_sub(bytes.len() as u64) + .ok_or_else(|| corrupt("capsule run suffix starts outside its object"))?; + Self::decode_control( + bytes, + object_size, + expected_hash, + expected_level, + expected_transactions, + expected_newest_base, + offset, + ) + } + + #[cfg(feature = "storage")] + pub(crate) fn footer_range( + trailer: &[u8], + pointer: &crate::capsule_protocol::CapsulePointer, + ) -> Result> { + if trailer.len() != RUN_TRAILER_BYTES || &trailer[40..] != RUN_MAGIC { + return Err(corrupt("capsule run footer trailer is invalid")); + } + let length = u64::from_be_bytes( + trailer[..8] + .try_into() + .map_err(|_| corrupt("capsule run footer length is invalid"))?, + ); + let hash = blake3::Hash::from_bytes( + trailer[8..40] + .try_into() + .map_err(|_| corrupt("capsule run footer hash is invalid"))?, + ); + if length == 0 + || length > MAX_RUN_FOOTER_BYTES as u64 + || hash.to_hex().as_str() != pointer.footer_hash() + { + return Err(corrupt( + "capsule run footer trailer does not match its pointer", + )); + } + let start = pointer + .size() + .checked_sub(RUN_TRAILER_BYTES as u64) + .and_then(|end| end.checked_sub(length)) + .ok_or_else(|| corrupt("capsule run footer range overflows"))?; + if start < pointer.control_offset() { + return Err(corrupt( + "capsule run footer starts outside its control range", + )); + } + Ok(start..pointer.size()) + } + + #[cfg(feature = "storage")] + pub(crate) const fn trailer_bytes() -> u64 { + RUN_TRAILER_BYTES as u64 + } + + /// Decode the footer alone, retaining authenticated page roots without materializing admission. + pub fn decode_footer( + bytes: Bytes, + pointer: &crate::capsule_protocol::CapsulePointer, + ) -> Result { + crate::capsule_protocol::root::validate_capsule_pointer(pointer)?; + let result = Self::decode_control( + bytes, + pointer.size(), + pointer.hash(), + pointer.level(), + pointer.transaction_ids(), + pointer.newest_base_root_digest(), + pointer.control_offset(), + )?; + if result.footer_hash != pointer.footer_hash() + || result.control_size != pointer.control_size() + || result.admission.is_some() + { + return Err(corrupt( + "capsule run footer does not match its pointer or contains admission bytes", + )); + } + Ok(result) + } + + // Both entry points share footer/range validation; complete inspection + // additionally authenticates every admission byte before exposing OIDs. + fn decode_control( + bytes: Bytes, + object_size: u64, + expected_hash: &str, + expected_level: u8, + expected_transactions: &[String], + expected_newest_base: &str, + control_offset: u64, ) -> Result { if bytes.len() < RUN_TRAILER_BYTES { return Err(corrupt("capsule run control suffix is truncated")); @@ -172,12 +270,10 @@ impl CapsuleRunControl { } validate_level_count(footer.level, footer.capsules.len()) .map_err(|error| corrupt(error.to_string()))?; - let control_offset = - object_size - .checked_sub(u64::try_from(bytes.len()).map_err(|_| { - corrupt("capsule run control suffix length cannot be represented") - })?) - .ok_or_else(|| corrupt("capsule run control suffix starts outside its object"))?; + let footer_offset = object_size + .checked_sub(bytes.len() as u64) + .and_then(|offset| offset.checked_add(footer_start as u64)) + .ok_or_else(|| corrupt("capsule run footer starts outside its object"))?; let mut controls = Vec::with_capacity(footer.capsules.len()); let mut expected_offset = 0_u64; for location in &footer.capsules { @@ -252,26 +348,33 @@ impl CapsuleRunControl { "capsule run admission descriptor presence differs from its body", )); } - let admission = if let Some(range) = footer.admission.as_ref() { - let footer_offset = control_offset - .checked_add(footer_start as u64) - .ok_or_else(|| corrupt("capsule run footer offset overflowed"))?; + let admission_info = footer.admission.clone().zip(footer.admission_pages.clone()); + let admission = if let Some((range, pages)) = &admission_info { validate_body_range(range, control_offset, footer_offset)?; - let admission_bytes = &bytes[..footer_start]; - if blake3::hash(admission_bytes).to_hex().as_str() != range.hash { - return Err(corrupt("capsule run admission hash does not match")); - } - if footer.admission_pages.as_ref() != Some(&AdmissionPages::decode(admission_bytes)?) { - return Err(corrupt( - "capsule run admission descriptor differs from its footer", - )); + pages.validate_region(range.length, footer.git_packs.len())?; + if footer_start == 0 { + None + } else { + let admission_bytes = &bytes[..footer_start]; + if admission_bytes.len() as u64 != range.length + || blake3::hash(admission_bytes).to_hex().as_str() != range.hash + { + return Err(corrupt( + "capsule run admission hash or length does not match", + )); + } + if pages != &AdmissionPages::decode(admission_bytes)? { + return Err(corrupt( + "capsule run admission descriptor differs from its footer", + )); + } + Some(CapsuleRunAdmission::decode( + admission_bytes, + footer.git_packs.len(), + )?) } - Some(CapsuleRunAdmission::decode( - admission_bytes, - footer.git_packs.len(), - )?) } else { - if footer_start != 0 { + if footer_start != 0 || footer_offset != control_offset { return Err(corrupt("capsule run control suffix has unclaimed bytes")); } None @@ -296,17 +399,42 @@ impl CapsuleRunControl { hash: expected_hash.to_owned(), object_size, control_offset, - control_size: u64::try_from(bytes.len()) - .map_err(|_| corrupt("capsule run control suffix length overflows"))?, + control_size: object_size + .checked_sub(control_offset) + .ok_or_else(|| corrupt("capsule run control range is invalid"))?, level: footer.level, footer_hash, capsules: controls, git_packs: footer.git_packs, index_pool: footer.index_pool, admission, + admission_info, }) } + #[cfg(feature = "storage")] + /// Open bounded admission lookup from this run's authenticated footer. + pub fn admission_reader( + &self, + layout: &crab_storage::StoreLayout, + maximum: u64, + ) -> Result> { + self.admission_info + .as_ref() + .map(|(range, pages)| { + CapsuleRunAdmissionReader::new( + layout, + &self.hash, + range.offset, + range.length, + pages.clone(), + self.git_packs.len(), + maximum, + ) + }) + .transpose() + } + /// Return the run's authenticated capsule controls. #[must_use] pub fn capsule_locations(&self) -> &[CapsuleControlLocation] { @@ -330,7 +458,7 @@ impl CapsuleRunControl { git_index_ranges(&self.git_packs, self.index_pool.as_ref()) } - /// Return the verified exact frontier admission from the control suffix. + /// Return materialized exact admission; footer-only reads retain a separate paged reader. #[must_use] pub fn admission(&self) -> Option<&CapsuleRunAdmission> { self.admission.as_ref() diff --git a/crates/crab-metadata/src/capsule_protocol/run/admission.rs b/crates/crab-metadata/src/capsule_protocol/run/admission.rs index f0669086d..99c5dc412 100644 --- a/crates/crab-metadata/src/capsule_protocol/run/admission.rs +++ b/crates/crab-metadata/src/capsule_protocol/run/admission.rs @@ -5,6 +5,11 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +#[cfg(feature = "storage")] +mod reader; +#[cfg(feature = "storage")] +pub use reader::CapsuleRunAdmissionReader; + const ADMISSION_MAGIC: &[u8; 8] = b"CRBADM01"; const ADMISSION_VERSION: u32 = 2; const ADMISSION_TRAILER_BYTES: usize = 4; @@ -45,6 +50,20 @@ impl AdmissionPages { Ok(()) } + pub(super) fn validate_region(&self, length: u64, member_count: usize) -> Result<()> { + self.validate()?; + let encoded = self + .body_bytes + .checked_add(self.header().len() as u64) + .and_then(|bytes| bytes.checked_add(ADMISSION_TRAILER_BYTES as u64)); + if self.member_count as usize != member_count || encoded != Some(length) { + return Err(corrupt( + "capsule run admission descriptor does not match its range or inventory", + )); + } + Ok(()) + } + fn header(&self) -> Vec { let mut bytes = ADMISSION_MAGIC.to_vec(); bytes.extend_from_slice(&ADMISSION_VERSION.to_be_bytes()); @@ -222,7 +241,7 @@ impl CapsuleRunAdmission { } pub(super) fn decode(bytes: &[u8], expected_member_count: usize) -> Result { - use crate::pages::{Input, Kind}; + use crate::pages::Kind; let pages = AdmissionPages::decode(bytes)?; if pages.member_count as usize != expected_member_count { return Err(corrupt( @@ -245,22 +264,7 @@ impl CapsuleRunAdmission { )?; let mut members = BTreeMap::new(); for (oid, value) in rows { - let mut input = Input::new(&value, "capsule run admission members"); - let count = input.u32()? as usize; - if count == 0 || count > expected_member_count || input.remaining() / 4 != count { - return Err(corrupt("capsule run admission has an invalid member list")); - } - let mut list = Vec::with_capacity(count); - for _ in 0..count { - let member = input.u32()?; - if member as usize >= expected_member_count - || list.last().is_some_and(|previous| *previous >= member) - { - return Err(corrupt("capsule run admission members are invalid")); - } - list.push(member); - } - input.finish()?; + let list = decode_members(&value, expected_member_count)?; let oid = oid .try_into() .map_err(|_| corrupt("capsule run admission OID has an invalid length"))?; @@ -296,6 +300,26 @@ impl CapsuleRunAdmission { } } +fn decode_members(value: &[u8], member_count: usize) -> Result> { + let mut input = crate::pages::Input::new(value, "capsule run admission members"); + let count = input.u32()? as usize; + if count == 0 || count > member_count || input.remaining() / 4 != count { + return Err(corrupt("capsule run admission has an invalid member list")); + } + let mut list = Vec::with_capacity(count); + for _ in 0..count { + let member = input.u32()?; + if member as usize >= member_count + || list.last().is_some_and(|previous| *previous >= member) + { + return Err(corrupt("capsule run admission members are invalid")); + } + list.push(member); + } + input.finish()?; + Ok(list) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs b/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs new file mode 100644 index 000000000..52c3d5430 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs @@ -0,0 +1,160 @@ +//! Bounded admission lookups within one authenticated immutable run. +use super::*; +use crate::pages::{Kind, PageCache, PageWork}; +use crab_storage::{Store, StoreLayout}; +use std::collections::BTreeSet; + +/// Paged exact placement hints; membership never grants reference visibility. +#[derive(Debug)] +pub struct CapsuleRunAdmissionReader { + path: object_store::path::Path, + offset: u64, + pages: AdmissionPages, + maximum: u64, + cache: PageCache, +} + +impl CapsuleRunAdmissionReader { + pub(in crate::capsule_protocol::run) fn new( + layout: &StoreLayout, + hash: &str, + offset: u64, + length: u64, + pages: AdmissionPages, + member_count: usize, + maximum: u64, + ) -> Result { + pages.validate_region(length, member_count)?; + offset + .checked_add(length) + .ok_or_else(|| corrupt("capsule run admission range overflows"))?; + Ok(Self { + path: layout.capsule_path(hash), + offset, + pages, + maximum, + cache: PageCache::default(), + }) + } + + /// Return exact member lists or authenticated misses, preserving duplicates and input order. + /// + /// The caller admits input/output cardinality; the reader bounds aggregate encoded page work. + pub async fn lookup(&self, store: &Store, oids: &[[u8; 20]]) -> Result>>> { + let mut work = self.work(); + let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + let values = self + .rows( + store, + &self.pages.objects, + Kind::RunAdmission, + &keys, + &mut work, + ) + .await?; + let result = values + .into_iter() + .map(|value| { + value + .map(|value| decode_members(&value, self.pages.member_count as usize)) + .transpose() + }) + .collect::>>()?; + let members = result + .iter() + .flatten() + .flatten() + .copied() + .collect::>() + .into_iter() + .collect::>(); + if self + .member_counts(store, &members, &mut work) + .await? + .contains(&0) + { + return Err(corrupt( + "capsule run admission names a member with zero objects", + )); + } + Ok(result) + } + + /// Return authenticated counts in member order without decoding the OID directory. + pub async fn counts(&self, store: &Store) -> Result> { + let members = (0..self.pages.member_count).collect::>(); + self.member_counts(store, &members, &mut self.work()).await + } + + async fn member_counts( + &self, + store: &Store, + members: &[u32], + work: &mut PageWork, + ) -> Result> { + let keys = members + .iter() + .map(|member| member.to_be_bytes().to_vec()) + .collect::>(); + let values = self + .rows( + store, + &self.pages.counts, + Kind::RunAdmissionCounts, + &keys, + work, + ) + .await?; + values + .into_iter() + .map(|value| { + let value = + value.ok_or_else(|| corrupt("required capsule member count is absent"))?; + let count = u64::from_le_bytes( + value + .as_slice() + .try_into() + .map_err(|_| corrupt("capsule member count has an invalid size"))?, + ); + if count > u64::from(self.pages.objects.count) { + return Err(corrupt("capsule member count exceeds total objects")); + } + Ok(count) + }) + .collect() + } + + async fn rows( + &self, + store: &Store, + root: &crate::pages::PageRef, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + let end = self + .offset + .checked_add(self.pages.body_bytes) + .ok_or_else(|| corrupt("capsule admission body range overflows"))?; + self.cache + .lookup( + store, + &self.path, + 0, + self.offset..end, + root, + kind, + keys, + work, + ) + .await + } + + fn work(&self) -> PageWork { + PageWork { + maximum: self.maximum, + bytes: 0, + seen: BTreeSet::new(), + } + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index bb8d15f0e..5ba2b5a99 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -398,6 +398,47 @@ pub async fn load_capsule_run_control( Ok((control, capsules)) } +/// Load complete reference controls while validating admission counts through authenticated pages. +/// +/// OID admission remains cold; reference reconciliation still validates complete visibility edits. +pub async fn load_capsule_run_git_control( + router: &StoreLayout, + pointer: &super::CapsulePointer, +) -> Result<(CapsuleRunControl, Vec)> { + super::root::validate_capsule_pointer(pointer)?; + let path = router.capsule_path(pointer.hash()); + let start = pointer + .size() + .checked_sub(CapsuleRunControl::trailer_bytes()) + .ok_or_else(|| corrupt(&path, "capsule run is shorter than its trailer"))?; + let trailer = router + .store() + .range_get(&path, start..pointer.size()) + .await?; + let range = CapsuleRunControl::footer_range(&trailer, pointer)?; + let footer = router.store().range_get(&path, range).await?; + let control = CapsuleRunControl::decode_footer(footer, pointer)?; + if let Some(reader) = + control.admission_reader(router, super::run::MAX_RUN_FOOTER_BYTES as u64)? + { + let counts = reader.counts(router.store()).await?; + if counts.len() != control.git_packs().len() + || counts + .iter() + .zip(control.git_packs()) + .any(|(count, member)| *count != member.object_count()) + { + return Err(corrupt( + &path, + "capsule member counts do not match its authenticated pack directory", + )); + } + } + let detached = load_detached_control_sections(router, &path, &control).await?; + let capsules = control.materialize_capsules_with_external_controls(&detached)?; + Ok((control, capsules)) +} + async fn load_detached_control_sections( router: &StoreLayout, path: &object_store::path::Path, diff --git a/crates/crab-metadata/src/capsule_protocol/store/tests.rs b/crates/crab-metadata/src/capsule_protocol/store/tests.rs index a30d5717c..74528ce23 100644 --- a/crates/crab-metadata/src/capsule_protocol/store/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/store/tests.rs @@ -54,6 +54,9 @@ async fn run_pointer_binding_is_identical_for_full_and_control_reads() { let (layout, pointer, _) = run_fixture().await; load_capsule_run(&layout, &pointer).await.unwrap(); load_capsule_run_control(&layout, &pointer).await.unwrap(); + load_capsule_run_git_control(&layout, &pointer) + .await + .unwrap(); for field in ["footer_hash", "control_boundary"] { let mut changed = serde_json::to_value(&pointer).unwrap(); if field == "footer_hash" { @@ -71,6 +74,12 @@ async fn run_pointer_binding_is_identical_for_full_and_control_reads() { load_capsule_run_control(&layout, &changed).await.is_err(), "control read: {field}" ); + assert!( + load_capsule_run_git_control(&layout, &changed) + .await + .is_err(), + "paged Git control: {field}" + ); } } @@ -276,3 +285,109 @@ async fn pointer_catalog_resolves_dependencies_only_for_current_epoch_heads() { } } } + +#[tokio::test] +async fn git_control_keeps_oid_admission_cold_and_paged_queries_obey_budgets() { + use crate::capsule_protocol::{CapsuleGitPack, CapsulePointer}; + use std::sync::atomic::{AtomicU64, Ordering}; + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_read_byte_observer(Arc::new(move |read| { + observed.fetch_add(read, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store, "repositories/lazy-admission".into()); + let transaction = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let pack = CapsuleGitPack::new( + bytes::Bytes::from_static(b"PACK"), + bytes::Bytes::from_static(b"index"), + bytes::Bytes::from_static(b"reverse"), + bytes::Bytes::from_static(b"locator"), + "4".repeat(40), + 10000, + ) + .unwrap(); + let capsule = Capsule::build(&transaction, vec![pack], vec![]).unwrap(); + let oids = (0u32..10000) + .map(|index| { + let mut oid = [0; 20]; + oid[..4].copy_from_slice(&index.to_be_bytes()); + oid + }) + .collect::>(); + let run = CapsuleRun::leaf_with_member_oids(capsule, vec![oids.clone()]).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + bytes.store(0, Ordering::Relaxed); + let (control, controls) = load_capsule_run_git_control(&layout, &pointer) + .await + .unwrap(); + let opening_bytes = bytes.swap(0, Ordering::Relaxed); + assert!(opening_bytes < 16 * 1024 && control.admission().is_none()); + assert_eq!(controls.len(), 1); + let reader = control + .admission_reader(&layout, 128 * 1024) + .unwrap() + .unwrap(); + let requested = [oids[9999], oids[0], oids[9999], [0xff; 20]]; + let expected = vec![Some(vec![0]), Some(vec![0]), Some(vec![0]), None]; + assert_eq!( + reader.lookup(layout.store(), &requested).await.unwrap(), + expected + ); + assert!(bytes.swap(0, Ordering::Relaxed) < 128 * 1024); + assert_eq!( + reader.lookup(layout.store(), &requested).await.unwrap(), + expected + ); + assert_eq!(bytes.swap(0, Ordering::Relaxed), 0); + let limited = control.admission_reader(&layout, 1).unwrap().unwrap(); + assert!(limited.lookup(layout.store(), &requested).await.is_err()); + assert_eq!(bytes.load(Ordering::Relaxed), 0); + // Unrequested OID corruption must remain cold during Git control opening; + // exact lookup and complete inspection still reject the damaged dependency. + let mut damaged = run.bytes().to_vec(); + damaged[run.control_offset() as usize] ^= 1; + layout + .store() + .put_overwrite(&layout.capsule_path(run.hash()), damaged.into()) + .await + .unwrap(); + let fresh = control + .admission_reader(&layout, 128 * 1024) + .unwrap() + .unwrap(); + assert_eq!( + ( + load_capsule_run_git_control(&layout, &pointer) + .await + .is_ok(), + load_capsule_run_control(&layout, &pointer).await.is_err(), + fresh.lookup(layout.store(), &[oids[0]]).await.is_err(), + ), + (true, true, true), + ); +} diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index 5e3374131..c1f39af8d 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -333,7 +333,7 @@ impl PagedGitObjectIndex { store, &self.path, 0, - self.descriptor.body_bytes, + 0..self.descriptor.body_bytes, root, kind, keys, diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs index caa9fb1d2..74a37f0f2 100644 --- a/crates/crab-metadata/src/pages/cache.rs +++ b/crates/crab-metadata/src/pages/cache.rs @@ -34,12 +34,16 @@ impl PageCache { store: &Store, path: &object_store::path::Path, layer: usize, - source_bytes: u64, + source: std::ops::Range, root: &PageRef, kind: Kind, keys: &[Vec], work: &mut PageWork, ) -> Result>>> { + let source_bytes = source + .end + .checked_sub(source.start) + .ok_or_else(|| corruption("invalid page source range"))?; if keys.iter().any(|key| key.len() != kind.key_bytes()) { return Err(corruption("invalid lookup key")); } @@ -51,7 +55,7 @@ impl PageCache { while !pending.is_empty() { for (reference, _) in &pending { reference.validate(kind, source_bytes)?; - if work.seen.insert((layer, reference.offset)) { + if work.seen.insert((layer, source.start + reference.offset)) { work.bytes = work .bytes .checked_add(u64::from(reference.bytes)) @@ -63,11 +67,14 @@ impl PageCache { } // Charge the entire wave before starting I/O, including warm pages. let pages = futures_util::stream::iter(std::mem::take(&mut pending)) - .map(|(reference, positions)| async move { - let page = self - .fetch(store, path, layer, source_bytes, &reference, kind) - .await?; - Ok::<_, MetadataError>((page, positions)) + .map(|(reference, positions)| { + let source = source.clone(); + async move { + let page = self + .fetch(store, path, layer, source.clone(), &reference, kind) + .await?; + Ok::<_, MetadataError>((page, positions)) + } }) .buffer_unordered(4) .try_collect::>() @@ -118,12 +125,20 @@ impl PageCache { store: &Store, path: &object_store::path::Path, layer: usize, - source_bytes: u64, + source: std::ops::Range, reference: &PageRef, kind: Kind, ) -> Result> { + let source_bytes = source + .end + .checked_sub(source.start) + .ok_or_else(|| corruption("invalid page source range"))?; reference.validate(kind, source_bytes)?; - let key = (layer, reference.offset); + let offset = source + .start + .checked_add(reference.offset) + .ok_or_else(|| corruption("page source offset overflows"))?; + let key = (layer, offset); if let Some((previous_kind, previous, page)) = self.state()?.pages.get(&key) { if *previous_kind != kind || previous != reference { return Err(corruption("cached page routing metadata changed")); @@ -131,10 +146,7 @@ impl PageCache { return Ok(page.clone()); } let bytes = store - .range_get( - path, - reference.offset..reference.offset + u64::from(reference.bytes), - ) + .range_get(path, offset..offset + u64::from(reference.bytes)) .await?; let page = Arc::new(decode( &bytes, diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index 03c778711..c84f78953 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -136,7 +136,7 @@ impl PathStateReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, names, Kind::Names, &keys, @@ -178,7 +178,7 @@ impl PathStateReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, &batch.root, Kind::Children, &keys, @@ -254,7 +254,7 @@ impl PathStateReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, &reference.pages.records, Kind::Records, &keys, @@ -314,7 +314,7 @@ impl PathStateReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, &reference.pages.nodes, Kind::Nodes, &keys, @@ -372,7 +372,7 @@ impl PathStateReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, reference, kind, ) diff --git a/crates/crab-metadata/src/split_commit_graph/reader.rs b/crates/crab-metadata/src/split_commit_graph/reader.rs index e9b2fa9d5..bec1ac05a 100644 --- a/crates/crab-metadata/src/split_commit_graph/reader.rs +++ b/crates/crab-metadata/src/split_commit_graph/reader.rs @@ -97,7 +97,7 @@ impl CommitGraphReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, &self.roots(&self.descriptor.layers[layer])?.ancestry_roots, Kind::CommitAncestryRoots, &[root.to_be_bytes().to_vec()], @@ -171,7 +171,7 @@ impl CommitGraphReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, &self.roots(reference)?.ancestry_nodes, Kind::CommitAncestryNodes, &keys, @@ -248,7 +248,7 @@ impl CommitGraphReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, &roots.oids, Kind::CommitOids, &keys, @@ -380,7 +380,7 @@ impl CommitGraphReader { store, &self.paths[layer], layer, - self.descriptor.layers[layer].bytes, + 0..self.descriptor.layers[layer].bytes, root, Kind::CommitRecords, &keys, diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 60eeed93e..e3e3006d2 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -37,6 +37,15 @@ Replica selection reports readiness, generations, fallbacks, and routing choices so callers can distinguish an unavailable replica from a corrupt object. +`capsule_protocol::open_git_view_from_root` captures complete refs, visibility +and pointer catalogs without materializing run OID admission. It validates +authenticated member counts and omits placement-hint dictionaries. CLI reads, +HTTP browse/projection and S3 gateway reads use this view. Physical lookup uses +the captured index; unindexed reads retain their existing pack-index path. +Maintenance and writers requiring exact admission sets continue using the +complete metadata entry point. Visibility, catalog and source/ref enumeration +are still eager; this entry point does not establish bounded repository opening. + `ShardHydrator` provides memory, file, and half-open byte-range reconstruction. Before full-file reconstruction it checks that the pointer's shard terms are covered; after reconstruction it verifies the requested file hash. A shared diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 6aeeaf36f..e251448d0 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -4124,7 +4124,15 @@ pub async fn open_view_from_root_for_repack( ) .await; } - assemble_layered_control_view(router, snapshot, limits, heads, active, false).await + assemble_layered_control_view( + router, + snapshot, + limits, + heads, + active, + ControlRead::Complete, + ) + .await } /// Load complete checkpoint catalog/visibility metadata and frontier controls. @@ -4261,7 +4269,35 @@ pub async fn open_view_from_root_with_metadata( limits: CapsuleReadLimits, ) -> Result { let (heads, active) = capture_ref_heads(router, snapshot.record().root()).await?; - assemble_layered_control_view(router, snapshot, limits, heads, active, false).await + assemble_layered_control_view( + router, + snapshot, + limits, + heads, + active, + ControlRead::Complete, + ) + .await +} + +/// Capture complete reference/visibility metadata while keeping run OID admission cold. +/// +/// Paged member counts validate pack coverage. Placement reads use the captured origin; +/// callers requiring exact admission sets use the complete metadata entry point. +pub async fn open_git_view_from_root( + router: &StoreLayout, + snapshot: crab_metadata::capsule_protocol::RootSnapshot, + limits: CapsuleReadLimits, +) -> Result { + let (heads, active) = capture_ref_heads(router, snapshot.record().root()).await?; + assemble_layered_control_view(router, snapshot, limits, heads, active, ControlRead::Git).await +} + +#[derive(Clone, Copy)] +enum ControlRead { + Complete, + Git, + Fetch, } #[derive(Debug, Clone, Copy)] @@ -4289,7 +4325,11 @@ async fn assemble_view( limits, heads, active, - matches!(checkpoint_load, CheckpointLoad::LayeredControl), + if matches!(checkpoint_load, CheckpointLoad::LayeredControl) { + ControlRead::Fetch + } else { + ControlRead::Complete + }, ) .await; } @@ -4513,8 +4553,9 @@ async fn assemble_layered_control_view( limits: CapsuleReadLimits, heads: Vec, active: BTreeSet, - footer_only: bool, + read: ControlRead, ) -> Result { + let footer_only = matches!(read, ControlRead::Fetch); let mut refs = snapshot.record().root().refs().clone(); let mut peeled_refs = snapshot.record().root().peeled_refs().clone(); let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; @@ -4548,7 +4589,12 @@ async fn assemble_layered_control_view( let (control, capsules) = run.verified_controls()?; Ok::<_, ReadError>((control, capsules, Some(run.bytes().clone()))) } else { - let (control, capsules) = load_run_control(router, pointer).await?; + let (control, capsules) = if matches!(read, ControlRead::Git) { + crab_metadata::capsule_protocol::load_capsule_run_git_control(router, pointer) + .await? + } else { + load_run_control(router, pointer).await? + }; Ok((control, capsules, None)) } })) @@ -4621,7 +4667,9 @@ async fn assemble_layered_control_view( capsule_run_member_oids.insert(pointer.hash().to_owned(), member_oids); } for capsule in capsules { - if let Some(visibility) = capsule.visibility_delta() { + if !matches!(read, ControlRead::Git) + && let Some(visibility) = capsule.visibility_delta() + { extend_frontier_object_admission( &mut frontier_object_admission, visibility, @@ -6414,6 +6462,34 @@ mod tests { ); } + #[tokio::test] + async fn git_metadata_view_preserves_references_visibility_and_catalog() { + for committed in [false, true] { + let inner = Arc::new(InMemory::new()); + seed_prepared_multi_ref(inner.clone(), committed, committed).await; + let router = StoreLayout::new(Store::new(inner), "repositories/test".into()); + let root = load_root(&router).await.unwrap(); + let complete = open_view_from_root_with_metadata(&router, root.clone(), TEST_LIMITS) + .await + .unwrap(); + let git = open_git_view_from_root(&router, root, TEST_LIMITS) + .await + .unwrap(); + assert_eq!( + ( + git.git_snapshot().unwrap(), + git.git_visibility_index().unwrap(), + git.pointer_catalog().unwrap() + ), + ( + complete.git_snapshot().unwrap(), + complete.git_visibility_index().unwrap(), + complete.pointer_catalog().unwrap() + ) + ); + } + } + #[tokio::test] async fn metadata_view_preserves_multi_ref_activation_and_visibility() { for committed in [false, true] { diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index 4463f3d60..d34332dd1 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -203,7 +203,7 @@ pub async fn open_repository( ) -> Result { let root = crab_metadata::capsule_protocol::load_root(&layout).await?; let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&layout).await?; - let view = crab_read::capsule_protocol::open_view_from_root_with_metadata( + let view = crab_read::capsule_protocol::open_git_view_from_root( &layout, root, crab_read::capsule_protocol::CapsuleReadLimits { diff --git a/crates/crab-s3-gateway/src/repository.rs b/crates/crab-s3-gateway/src/repository.rs index 5e5e9e20f..6487bc654 100644 --- a/crates/crab-s3-gateway/src/repository.rs +++ b/crates/crab-s3-gateway/src/repository.rs @@ -340,7 +340,7 @@ impl ReadViewCache { } match crab_metadata::capsule_protocol::load_root(&repository.layout).await { Ok(root) => { - let capsule = crab_read::capsule_protocol::open_view_from_root_with_metadata( + let capsule = crab_read::capsule_protocol::open_git_view_from_root( &repository.layout, root, crab_read::capsule_protocol::CapsuleReadLimits { From bf5ea14b51a138ff48553005aa2fc18a35fa3c44 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 06:39:08 -0700 Subject: [PATCH 022/127] perf(read): borrow shared visibility checks --- .../docs/architecture/indexed-remote-reads.md | 31 +++ crates/crab-read/README.md | 3 + crates/crab-read/src/capsule_protocol.rs | 242 +++++++++--------- 3 files changed, 151 insertions(+), 125 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 076c4ece4..7597ec655 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -838,3 +838,34 @@ reconciliation remain eager. The unindexed pack-index path remains existing prod behavior. HTTP/S3 service E2E, 10k version-2 scaling and representative-provider latency qualification are still missing for this slice. Stages 5–8 and unified native/Git staging remain separate work. The PR stays draft and is not merged. + +### Shared visibility checks and 10k qualification + +Full-capsule and control-only readiness/application now share borrowed checks. +The old four implementations cloned entire added/removed OID lists to remove ref +keys from a temporary map. Canonical transaction encoding/decoding already proves +unique refs, so counting matched borrowed rows preserves rejection of uncommitted +evidence. Ref-tip matching, deletion checks, cross-ref base readiness and prior +closure validation remain unchanged. The production refactor removes 74 lines; +complete JSON visibility decoding remains eager. + +All 36 capsule-read tests and strict all-target read Clippy pass. A new paired +full/control regression rejects extra/missing evidence, deleted-ref evidence and +wrong old/new tips. Removing the unmatched-row guard makes its exact test fail; +restoring the source passes the suite. Both qualifier builds pass. Final-source +RustFS replay passes 121 qualifier processes: all ten queries at 100 commits, +plus 20 cold/warm resolve and attribution samples at each of 100/1,000/10,000 +commits. Warm query backend bytes are zero; source hashes match the receipts. +Opening bytes are 78,142/471,334/1,341,201, with one admission count page decoded +at every history size. These shared-host debug results do not establish a +representative-provider latency target or bounded total opening. + +The fresh 10k fixture was published through the format slice's isolated installed +CLI; the read qualifier contains the current lazy-opening source. The complete +ten-query 10k qualification **fails** at path history: tree traversal charges +100,010 entries against the unchanged 100,000-entry budget. The failed report is +retained outside the checkout. Resolve/attribution proof does not override that +failure. Path history still visits unrelated commits/trees and needs the planned +change-point traversal; absent/deleted paths require explicit change records +because the current path-state trie drops deleted children. Limits were not raised +and no baseline or expected-failure file was changed. diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index e3e3006d2..0f82acf70 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -45,6 +45,9 @@ the captured index; unindexed reads retain their existing pack-index path. Maintenance and writers requiring exact admission sets continue using the complete metadata entry point. Visibility, catalog and source/ref enumeration are still eager; this entry point does not establish bounded repository opening. +Full-capsule and control-only views share borrowed visibility readiness/application +checks. Canonical transaction decoding guarantees unique ref names; matched-row +counts reject extra evidence without copying the per-object edit lists. `ShardHydrator` provides memory, file, and half-open byte-range reconstruction. Before full-file reconstruction it checks that the pointer's shard terms are diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index e251448d0..606fe3a04 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -5345,7 +5345,20 @@ fn control_is_ready( refs: &BTreeMap, visibility_index: &crab_metadata::git_visibility::GitVisibilityIndex, ) -> Result { - let transaction = capsule.transaction(); + visibility_is_ready( + capsule.transaction(), + capsule.visibility_delta(), + refs, + visibility_index, + ) +} + +fn visibility_is_ready( + transaction: &crab_metadata::capsule_protocol::CapsuleTransaction, + delta: Option<&crab_metadata::capsule_protocol::CapsuleVisibilityDelta>, + refs: &BTreeMap, + visibility_index: &crab_metadata::git_visibility::GitVisibilityIndex, +) -> Result { if transaction .edits() .iter() @@ -5353,13 +5366,16 @@ fn control_is_ready( { return Ok(false); } - let Some(delta) = capsule.visibility_delta() else { + let Some(delta) = delta else { return Ok(true); }; - let mut evidence = delta.edits().clone(); + // Transactions have unique refs. Count matched rows without copying OID lists; + // unmatched evidence still fails before it can authorize a publication. + let evidence = delta.edits(); + let mut matched = 0; for edit in transaction.edits() { let Some(new_oid) = edit.new_oid() else { - if evidence.remove(edit.ref_name()).is_some() { + if evidence.get(edit.ref_name()).is_some() { return Err(corrupt_path( "capsule Git visibility", "deleted ref has visibility evidence", @@ -5367,12 +5383,13 @@ fn control_is_ready( } continue; }; - let Some(visibility) = evidence.remove(edit.ref_name()) else { + let Some(visibility) = evidence.get(edit.ref_name()) else { return Err(corrupt_path( "capsule Git visibility", "live ref edit has no visibility evidence", )); }; + matched += 1; visibility.validate()?; if visibility.new_oid != new_oid { return Err(corrupt_path( @@ -5403,7 +5420,7 @@ fn control_is_ready( } } } - if !evidence.is_empty() { + if evidence.len() != matched { return Err(corrupt_path( "capsule Git visibility", "visibility evidence contains an uncommitted ref", @@ -5451,84 +5468,30 @@ fn capsule_is_ready( visibility_index: &crab_metadata::git_visibility::GitVisibilityIndex, ) -> Result { let transaction = capsule.transaction()?; - if transaction - .edits() - .iter() - .any(|edit| refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old()) - { - return Ok(false); - } - let Some(delta) = capsule.visibility_delta()? else { - // Ref ordering is authenticated by expected-old links. Visibility is a - // separate read-admission proof and remains fail-closed when requested. - return Ok(true); - }; - let mut evidence = delta.edits().clone(); - for edit in transaction.edits() { - let Some(new_oid) = edit.new_oid() else { - if evidence.remove(edit.ref_name()).is_some() { - return Err(corrupt_path( - "capsule Git visibility", - "deleted ref has visibility evidence", - )); - } - continue; - }; - let Some(visibility) = evidence.remove(edit.ref_name()) else { - return Err(corrupt_path( - "capsule Git visibility", - "live ref edit has no visibility evidence", - )); - }; - visibility.validate()?; - if visibility.new_oid != new_oid { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match its ref edit", - )); - } - match edit.expected_old() { - Some(expected_old) => { - if visibility.old_oid.as_deref() != Some(expected_old) { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match the expected old ref", - )); - } - if !visibility_index.contains_ref(edit.ref_name()) { - return Ok(false); - } - } - None if visibility.replaces => {} - None => { - let Some(old_oid) = visibility.old_oid.as_deref() else { - continue; - }; - if !visibility_index.contains_hex_in_any_ref(old_oid) { - return Ok(false); - } - } - } - } - if !evidence.is_empty() { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence contains an uncommitted ref", - )); - } - Ok(true) + let delta = capsule.visibility_delta()?; + visibility_is_ready(&transaction, delta.as_ref(), refs, visibility_index) } fn apply_capsule_control_visibility_index( capsule: &CapsuleControl, index: &mut crab_metadata::git_visibility::GitVisibilityIndex, ) -> Result<()> { - let transaction = capsule.transaction(); - let delta = capsule.visibility_delta(); - let mut evidence = delta.map(|delta| delta.edits().clone()).unwrap_or_default(); + apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta(), index) +} + +fn apply_transaction_visibility_index( + transaction: &crab_metadata::capsule_protocol::CapsuleTransaction, + delta: Option<&crab_metadata::capsule_protocol::CapsuleVisibilityDelta>, + index: &mut crab_metadata::git_visibility::GitVisibilityIndex, +) -> Result<()> { + let evidence = delta.map(|delta| delta.edits()); + let mut matched = 0; for edit in transaction.edits() { let Some(new_oid) = edit.new_oid() else { - if evidence.remove(edit.ref_name()).is_some() { + if evidence + .and_then(|rows| rows.get(edit.ref_name())) + .is_some() + { return Err(corrupt_path( "capsule Git visibility", "deleted ref has visibility evidence", @@ -5537,12 +5500,15 @@ fn apply_capsule_control_visibility_index( index.remove_ref(edit.ref_name()); continue; }; - let visibility = evidence.remove(edit.ref_name()).ok_or_else(|| { - corrupt_path( - "capsule Git visibility", - "live ref edit has no visibility evidence", - ) - })?; + let visibility = evidence + .and_then(|rows| rows.get(edit.ref_name())) + .ok_or_else(|| { + corrupt_path( + "capsule Git visibility", + "live ref edit has no visibility evidence", + ) + })?; + matched += 1; if visibility.new_oid != new_oid { return Err(corrupt_path( "capsule Git visibility", @@ -5557,9 +5523,9 @@ fn apply_capsule_control_visibility_index( "visibility evidence does not match the expected old ref", )); } - index.apply_ref_edit(edit.ref_name().to_owned(), &visibility)?; + index.apply_ref_edit(edit.ref_name().to_owned(), visibility)?; } - if !evidence.is_empty() { + if evidence.is_some_and(|rows| rows.len() != matched) { return Err(corrupt_path( "capsule Git visibility", "visibility evidence contains an uncommitted ref", @@ -5574,47 +5540,7 @@ fn apply_capsule_visibility_index( ) -> Result<()> { let transaction = capsule.transaction()?; let delta = capsule.visibility_delta()?; - let mut evidence = delta.map(|delta| delta.edits().clone()).unwrap_or_default(); - for edit in transaction.edits() { - let Some(new_oid) = edit.new_oid() else { - if evidence.remove(edit.ref_name()).is_some() { - return Err(corrupt_path( - "capsule Git visibility", - "deleted ref has visibility evidence", - )); - } - index.remove_ref(edit.ref_name()); - continue; - }; - let visibility = evidence.remove(edit.ref_name()).ok_or_else(|| { - corrupt_path( - "capsule Git visibility", - "live ref edit has no visibility evidence", - ) - })?; - if visibility.new_oid != new_oid { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match its ref edit", - )); - } - if let Some(expected_old) = edit.expected_old() - && visibility.old_oid.as_deref() != Some(expected_old) - { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match the expected old ref", - )); - } - index.apply_ref_edit(edit.ref_name().to_owned(), &visibility)?; - } - if !evidence.is_empty() { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence contains an uncommitted ref", - )); - } - Ok(()) + apply_transaction_visibility_index(&transaction, delta.as_ref(), index) } fn apply_capsule_refs( @@ -7260,6 +7186,72 @@ mod tests { ); } + #[test] + fn full_and_control_visibility_reject_uncommitted_evidence() { + for case in ["extra", "missing", "deleted", "new-tip", "old-tip"] { + let old = "1".repeat(40); + let new = "2".repeat(40); + let transaction = CapsuleTransaction::new( + &"a".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + Some(old.clone()), + (case != "deleted").then_some(new.clone()), + None, + )], + ) + .unwrap(); + let evidence_tip = if case == "new-tip" { + "3".repeat(40) + } else { + new + }; + let evidence_old = if case == "old-tip" { + "3".repeat(40) + } else { + old.clone() + }; + let edit = GitVisibilityEdit::from_replacement_objects( + Some(evidence_old), + evidence_tip.clone(), + vec![evidence_tip], + ); + let mut edits = BTreeMap::from([("refs/heads/main".to_owned(), edit.clone())]); + if case == "extra" || case == "missing" { + edits.insert("refs/heads/feature".to_owned(), edit); + } + if case == "missing" { + edits.remove("refs/heads/main"); + } + let delta = CapsuleVisibilityDelta::new(edits).unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule.clone()).unwrap(); + let (_, controls) = run.verified_controls().unwrap(); + let refs = BTreeMap::from([("refs/heads/main".to_owned(), old.clone())]); + let index = + visibility_index(BTreeMap::from([("refs/heads/main".to_owned(), vec![old])])); + assert_eq!( + ( + capsule_is_ready(&capsule, &refs, &index).is_err(), + control_is_ready(&controls[0], &refs, &index).is_err(), + apply_capsule_visibility_index(&capsule, &mut index.clone()).is_err(), + apply_capsule_control_visibility_index(&controls[0], &mut index.clone()) + .is_err(), + ), + (true, true, true, true), + "{case}", + ); + } + } + #[test] fn ref_capsules_wait_for_cross_ref_visibility_dependencies() { let old_tip = "1".repeat(40); From 3ab79821ccc554a089be706348ec02d90d4a5890 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 07:29:46 -0700 Subject: [PATCH 023/127] feat(metadata): page capsule visibility edits --- .../docs/architecture/indexed-remote-reads.md | 50 +++ crab/src/core/error.rs | 1 + crates/crab-metadata/README.md | 24 +- .../crab-metadata/src/capsule_protocol/mod.rs | 2 + .../crab-metadata/src/capsule_protocol/run.rs | 151 ++++--- .../src/capsule_protocol/store/tests.rs | 143 +++++++ .../src/capsule_protocol/visibility.rs | 44 +- .../src/capsule_protocol/visibility/paged.rs | 388 ++++++++++++++++++ .../visibility/paged/reader.rs | 239 +++++++++++ crates/crab-metadata/src/error.rs | 6 + crates/crab-metadata/src/git_visibility.rs | 6 +- crates/crab-metadata/src/pages.rs | 7 +- crates/crab-remote/examples/support/mod.rs | 13 +- 13 files changed, 973 insertions(+), 101 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/visibility/paged.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 7597ec655..3c4551a91 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -869,3 +869,53 @@ failure. Path history still visits unrelated commits/trees and needs the planned change-point traversal; absent/deleted paths require explicit change records because the current path-state trie drops deleted children. Limits were not raised and no baseline or expected-failure file was changed. + + +## Paged visibility delta format and lookup work in progress + +Capsule visibility delta version 3 replaces flat JSON OID lists with a paged +ref directory and separately authenticated added/removed OID sets. The shared +pager uses explicit kind codes 14/15. Empty sets have no root, and identical +sets may share immutable pages. Existing visibility byte/ref/object bounds +admit the format; complete decoding also charges logical OID references so +shared roots cannot expand into unbounded repeated output. Complete inspection +authenticates the body, validates edit invariants and reconstructs canonical +bytes. Retired delta JSON has no serialization alias or fallback reader. + +Run footer version 3 commits each visibility descriptor. Visibility pages always +remain in their capsule section; the previous small-table footer copy is removed. +Full control readers compare descriptors against the original sections before +exposing edits. Catalog's existing bounded inline/detached policy is unchanged. +The stored visibility snapshot, released v1 proof formats, root pointers and +manifest/catalog contracts are unchanged. Upstream release refs still match the +recorded audit and none contains capsule-protocol code. Retired run prefixes +must be republished; no automatic migration is provided. + +`CapsuleRunControl::visibility_reader` captures the descriptor and embedded body +span for one authenticated capsule. Batched OID changes preserve duplicates and +order. Absent refs, absent changes and missing/corrupt pages remain distinct. +Replacement edits must admit their new tip; an OID in both loaded sets is an +error. Warm pages retain their encoded-work charge, and the smallest budget +fails before I/O. Visibility changes alone never authorize a want: prior closure +and ref reconciliation remain separate required proofs. + +Focused proof passes 86 payload-only capsule tests, 94 storage/remote-index +capsule tests, all 36 read-side capsule tests, strict metadata Clippy, both +qualifier builds, observer tests and HTTP/S3 consumer checks after the frontend +build. A 10k-OID lookup opens its footer below 4 KiB, stays below a 128 KiB +query budget, returns verified misses, reads zero warm backend bytes and rejects +corruption. Separate tests cover removed/unchanged OIDs, empty edits, shared +roots, wide ref directories, normalized edit properties and re-signed overlap. +Removing canonical byte comparison makes the exact authenticated-unclaimed-byte +negative control fail; restored source passes. The retired footer-version test +now rejects version 2 and allows the new version 3; the existing detached-control +assertion is unchanged. + +Final-source isolated `make install` passes for CLI/FUSE/NFS/cache-server and the +Git helper. Fresh-prefix RustFS qualification passes 63 checks and 45 commands: +real Crab pushes publish 100/1,000-commit fixtures, both index-maintenance processes +finish, and all 20 supported-query/native-Git comparisons pass. Source/binary +hashes match the receipts and remain external. One sample per query establishes +publication/read smoke, not the final 20-sample performance cells. Complete +repository opening still materializes visibility and catalog state. This format/lookup work does not yet satisfy bounded +opening, the remaining stage-4 gates, the 10k path-history gate or stages 5–8. diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index 23a14ad9d..fb8e468cc 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -2047,6 +2047,7 @@ impl From for CrabError { | crab_metadata::error::MetadataError::PathStateSize { .. } | crab_metadata::error::MetadataError::BrowseIndexRecord { .. } | crab_metadata::error::MetadataError::GitObjectIndexRecord { .. } + | crab_metadata::error::MetadataError::CapsuleVisibilityRecord { .. } | crab_metadata::error::MetadataError::PlanAlreadyAttempted { .. } | crab_metadata::error::MetadataError::RefJournalCommitUncertain { .. } | crab_metadata::error::MetadataError::ManifestCommitUncertain { .. }) => { diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 76151326c..ca91e8a60 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -169,7 +169,7 @@ capsules and encodes/authenticates the final run once. Run level is the `ceil(log2(count))` size class; the authenticated capsule count remains exact. Mixed-level carries retain canonical capsule bytes and member ordinals without encoding discarded intermediate runs; complete capsule verification remains -mandatory. The v2 cutover uses CRBRUN01 with footer version 2 and +mandatory. The v2 cutover uses CRBRUN01 with footer version 3 and root/history/ref-head versions 4/3/5; there is no reader fallback to earlier run or pointer semantics. Runs may contain byte-identical packs from different transactions. Their source @@ -192,7 +192,7 @@ suffix. Detached large visibility/catalog sections retain their separate bounded reads. The unshipped `CRBRUN04`/`CRBRUN05`/`CRBRUN06`/`CRBRUN07` formats are rejected, not read through a compatibility path; readers and writers must cut over together. -Run footer and admission version 2 replace their unshipped version-1 shapes. +Run footer version 3 rejects its retired versions 1/2; admission remains version 2. Exact admission uses authenticated OID-to-member pages and dense per-member count pages. Their roots are committed by the footer; complete readers verify both directories and canonical body bytes before exposing admission. Page kind @@ -206,6 +206,26 @@ order and duplicates; cached and fetched pages share one encoded-work budget. Complete inspection still scans both admission trees. Complete visibility and catalog decoding and source/ref capture remain separate opening costs. +Visibility delta version 3 replaces flat JSON OID lists with authenticated ref +headers and separate added/removed OID trees. Ref directories and object sets +page their internal routing. Empty sets have no root; shared immutable sets may +reuse pages. Complete delta decoding checks body identity, both sets, edit +invariants and canonical bytes. The existing visibility proof byte/ref/object +bounds also admit this encoding and prevent repeated roots from expanding into +unbounded logical work during complete inspection. + +Run footers commit each visibility descriptor, and full control readers compare +it against the capsule section. Visibility OID tables are always detached from +the footer. `CapsuleRunControl::visibility_reader` opens authenticated batched +changes for one captured capsule, preserving duplicates and input order. A +missing ref, an unchanged OID and a missing/corrupt page have distinct results. +Replacement edits must admit their new tip; queried OIDs cannot belong to both +change sets. These changes alone never authorize a ref or object want. +The capsule delta's retired JSON shape has no reader or serialization alias; +retired run prefixes require republishing. Released v1 visibility index and +manifest/catalog formats are unchanged. Complete repository reconciliation +still loads visibility; lazy closure reconciliation remains unfinished. + Payload modules cover manifests, segmented lists, pack metadata, commit-graph summaries, ref registries, chunk/file indexes, receipts, transactions, and canonical key/value codecs. Storage-backed helpers are feature-gated: diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 29e8c1b34..ff6d3b1d9 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -70,6 +70,8 @@ pub use transaction_record::{ CapsuleTransactionRecord, CapsuleTransactionStatus, MAX_CAPSULE_TRANSACTION_RECORD_BYTES, }; pub use visibility::{CapsuleVisibilityDelta, CapsuleVisibilitySnapshot}; +#[cfg(feature = "storage")] +pub use visibility::{CapsuleVisibilityReader, VisibilityChange}; fn valid_ref_name(name: &str) -> bool { gix_validate::reference::name_partial(name.as_bytes().as_bstr()).is_ok() diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index 5f774d508..69611d214 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -11,7 +11,7 @@ use crate::error::{MetadataError, Result}; use crate::validation::validate_content_hash; const RUN_MAGIC: &[u8; 8] = b"CRBRUN01"; -const RUN_VERSION: u32 = 2; +const RUN_VERSION: u32 = 3; const RUN_TRAILER_BYTES: usize = 8 + 32 + RUN_MAGIC.len(); pub(super) const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_INLINE_RUN_CONTROL_SECTION_BYTES: usize = 512 * 1024; @@ -54,8 +54,7 @@ struct RunSectionLocation { #[serde(deny_unknown_fields)] struct RunControlSections { transaction: Vec, - #[serde(default)] - visibility: Option>, + visibility_pages: Option, #[serde(default)] catalog: Option>, } @@ -117,7 +116,7 @@ pub struct CapsuleControlLocation { visibility: Option, catalog: Option, transaction_bytes: Bytes, - visibility_bytes: Option, + visibility_pages: Option, catalog_bytes: Option, } @@ -297,17 +296,20 @@ impl CapsuleRunControl { Some(&transaction_bytes), "transaction", )?; - let visibility_bytes = location.control.visibility.clone().map(Bytes::from); let visibility_range = location .visibility .as_ref() .map(|range| PackRange::from_parts(range.offset, range.length, range.hash.clone())) .transpose()?; - verify_embedded_control_section( - visibility_range.as_ref(), - visibility_bytes.as_ref(), - "visibility", - )?; + match (&visibility_range, &location.control.visibility_pages) { + (Some(range), Some(pages)) => pages.validate_region(range.length())?, + (None, None) => {} + _ => { + return Err(corrupt( + "visibility section and page descriptor presence differs", + )); + } + } let catalog_bytes = location.control.catalog.clone().map(Bytes::from); let catalog_range = location .catalog @@ -327,7 +329,7 @@ impl CapsuleRunControl { visibility: visibility_range, catalog: catalog_range, transaction_bytes, - visibility_bytes, + visibility_pages: location.control.visibility_pages.clone(), catalog_bytes, }); expected_offset = location @@ -435,6 +437,37 @@ impl CapsuleRunControl { .transpose() } + #[cfg(feature = "storage")] + /// Open visibility changes for one captured capsule without reading its OID table. + /// + /// An absent section returns `None`; a capsule absent from this run is an error. + pub fn visibility_reader( + &self, + layout: &crab_storage::StoreLayout, + capsule_hash: &str, + maximum: u64, + ) -> Result> { + let location = self + .capsules + .iter() + .find(|location| location.hash == capsule_hash) + .ok_or_else(|| corrupt("visibility capsule is absent from its run"))?; + match (&location.visibility, &location.visibility_pages) { + (Some(range), Some(pages)) => super::visibility::CapsuleVisibilityReader::new( + layout, + &self.hash, + range, + pages.clone(), + maximum, + ) + .map(Some), + (None, None) => Ok(None), + _ => Err(corrupt( + "visibility section and page descriptor presence differs", + )), + } + } + /// Return the run's authenticated capsule controls. #[must_use] pub fn capsule_locations(&self) -> &[CapsuleControlLocation] { @@ -497,11 +530,19 @@ impl CapsuleRunControl { Some(&location.transaction_bytes), "transaction", )?; - let visibility_bytes = location.visibility_bytes.clone().or_else(|| { - external_controls - .get(&(location.hash.clone(), CapsuleSectionKind::VisibilityDelta)) - .cloned() - }); + let visibility_bytes = external_controls + .get(&(location.hash.clone(), CapsuleSectionKind::VisibilityDelta)) + .cloned(); + if visibility_bytes + .as_ref() + .map(|bytes| super::visibility::VisibilityPages::decode(bytes)) + .transpose()? + != location.visibility_pages + { + return Err(corrupt( + "visibility page descriptor differs from its run footer", + )); + } let catalog_bytes = location.catalog_bytes.clone().or_else(|| { external_controls .get(&(location.hash.clone(), CapsuleSectionKind::CatalogDelta)) @@ -628,7 +669,6 @@ impl CapsuleControlLocation { ) -> impl Iterator + '_ { self.visibility .as_ref() - .filter(|_| self.visibility_bytes.is_none()) .map(|range| { ( self.hash.clone(), @@ -725,10 +765,12 @@ impl CapsuleRun { let visibility = section_location(capsule, offset, CapsuleSectionKind::VisibilityDelta)?; let catalog = section_location(capsule, offset, CapsuleSectionKind::CatalogDelta)?; - let inline_visibility = + // OID pages stay in the capsule section; only their authenticated + // descriptor belongs in the footer, independent of current table size. + let visibility_pages = section_bytes_for_kind(capsule, CapsuleSectionKind::VisibilityDelta)? - .filter(|bytes| bytes.len() <= MAX_INLINE_RUN_CONTROL_SECTION_BYTES) - .map(|bytes| bytes.to_vec()); + .map(|bytes| super::visibility::VisibilityPages::decode(&bytes)) + .transpose()?; let inline_catalog = section_bytes_for_kind(capsule, CapsuleSectionKind::CatalogDelta)? .filter(|bytes| bytes.len() <= MAX_INLINE_RUN_CONTROL_SECTION_BYTES) .map(|bytes| bytes.to_vec()); @@ -743,7 +785,7 @@ impl CapsuleRun { catalog, control: RunControlSections { transaction: capsule.section_bytes(0)?.to_vec(), - visibility: inline_visibility, + visibility_pages, catalog: inline_catalog, }, }); @@ -824,44 +866,27 @@ impl CapsuleRun { break footer_bytes; } - let mut largest = None; - for (index, location) in locations.iter().enumerate() { - for (kind, bytes) in [ - ( - CapsuleSectionKind::VisibilityDelta, - location.control.visibility.as_ref(), - ), - ( - CapsuleSectionKind::CatalogDelta, - location.control.catalog.as_ref(), - ), - ] { - if let Some(bytes) = bytes - && largest - .as_ref() - .is_none_or(|(_, _, length)| *length < bytes.len()) - { - largest = Some((index, kind, bytes.len())); - } - } - } - let Some((index, kind, _)) = largest else { + let largest = locations + .iter() + .enumerate() + .filter_map(|(index, location)| { + location + .control + .catalog + .as_ref() + .map(|bytes| (index, bytes.len())) + }) + .max_by_key(|(_, length)| *length); + let Some((index, _)) = largest else { return Err(contract_error(format!( "capsule run footer exceeds {MAX_RUN_FOOTER_BYTES} bytes" ))); }; - let location = locations + locations .get_mut(index) - .ok_or_else(|| contract_error("capsule run control disappeared"))?; - match kind { - CapsuleSectionKind::VisibilityDelta => location.control.visibility = None, - CapsuleSectionKind::CatalogDelta => location.control.catalog = None, - _ => { - return Err(contract_error( - "capsule run selected a non-detachable control section", - )); - } - } + .ok_or_else(|| contract_error("capsule run control disappeared"))? + .control + .catalog = None; }; let footer_length = u64::try_from(footer_bytes.len()) .map_err(|_| contract_error("capsule run footer length cannot be represented"))?; @@ -1325,12 +1350,16 @@ fn validate_control_bundle(capsule: &Capsule, location: &RunCapsuleLocation) -> .transpose()?; verify_control_section(visibility_range.as_ref(), visibility.as_ref(), "visibility")?; verify_control_section(catalog_range.as_ref(), catalog.as_ref(), "catalog")?; - let embedded_visibility = location.control.visibility.clone().map(Bytes::from); - verify_embedded_control_section( - visibility_range.as_ref(), - embedded_visibility.as_ref(), - "visibility", - )?; + if visibility + .as_ref() + .map(|bytes| super::visibility::VisibilityPages::decode(bytes)) + .transpose()? + != location.control.visibility_pages + { + return Err(corrupt( + "visibility page descriptor differs from its capsule", + )); + } let embedded_catalog = location.control.catalog.clone().map(Bytes::from); verify_embedded_control_section(catalog_range.as_ref(), embedded_catalog.as_ref(), "catalog")?; Ok(()) @@ -1735,7 +1764,7 @@ mod tests { #[test] fn unsupported_run_footer_versions_are_rejected() { let run = CapsuleRun::leaf(capsule('1', '2')).unwrap(); - for version in [0, 1, 3, 4, 5, 6, 7] { + for version in [0, 1, 2, 4, 5, 6, 7] { let mut footer = run.footer.clone(); footer.version = version; let footer = serde_json::to_vec(&footer).unwrap(); diff --git a/crates/crab-metadata/src/capsule_protocol/store/tests.rs b/crates/crab-metadata/src/capsule_protocol/store/tests.rs index 74528ce23..70560bb41 100644 --- a/crates/crab-metadata/src/capsule_protocol/store/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/store/tests.rs @@ -391,3 +391,146 @@ async fn git_control_keeps_oid_admission_cold_and_paged_queries_obey_budgets() { (true, true, true), ); } + +#[tokio::test] +async fn visibility_lookup_keeps_oid_pages_cold_until_requested() { + use crate::capsule_protocol::{ + CapsulePointer, CapsuleSection, CapsuleVisibilityDelta, VisibilityChange, + }; + use std::sync::atomic::{AtomicU64, Ordering}; + let read_bytes = Arc::new(AtomicU64::new(0)); + let observed = read_bytes.clone(); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_read_byte_observer(Arc::new(move |bytes| { + observed.fetch_add(bytes, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store, "repositories/paged-visibility".into()); + let oids = (0u32..10000) + .map(|index| { + let mut oid = [0; 20]; + oid[..4].copy_from_slice(&index.to_be_bytes()); + oid + }) + .collect::>(); + let objects = oids + .iter() + .map(crate::git_visibility::encode_oid) + .collect::>(); + let tip = objects[7000].clone(); + let transaction = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(tip.clone()), + None, + )], + ) + .unwrap(); + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/main".into(), + crate::git_visibility::GitVisibilityEdit::from_replacement_objects(None, tip, objects), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![], + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule.clone()).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let footer = layout + .store() + .range_get( + &layout.capsule_path(run.hash()), + pointer.control_offset()..pointer.size(), + ) + .await + .unwrap(); + let control = CapsuleRunControl::decode_footer(footer, &pointer).unwrap(); + assert!(read_bytes.swap(0, Ordering::Relaxed) < 4096); + let reader = control + .visibility_reader(&layout, capsule.hash(), 128 * 1024) + .unwrap() + .unwrap(); + let requested = [oids[9999], oids[0], oids[9999], [0xff; 20]]; + let expected = Some(vec![ + Some(VisibilityChange::Added), + Some(VisibilityChange::Added), + Some(VisibilityChange::Added), + None, + ]); + assert_eq!( + reader + .lookup(layout.store(), "refs/heads/main", &requested) + .await + .unwrap(), + expected + ); + assert!(read_bytes.swap(0, Ordering::Relaxed) < 128 * 1024); + assert_eq!( + reader + .lookup(layout.store(), "refs/heads/main", &requested) + .await + .unwrap(), + expected + ); + assert_eq!( + reader + .lookup(layout.store(), "refs/heads/missing", &requested) + .await + .unwrap(), + None + ); + assert_eq!(read_bytes.swap(0, Ordering::Relaxed), 0); + let limited = control + .visibility_reader(&layout, capsule.hash(), 1) + .unwrap() + .unwrap(); + assert!( + limited + .lookup(layout.store(), "refs/heads/main", &requested) + .await + .is_err() + ); + assert_eq!(read_bytes.load(Ordering::Relaxed), 0); + let mut damaged = run.bytes().to_vec(); + damaged[control.capsule_locations()[0] + .visibility() + .unwrap() + .offset() as usize] ^= 1; + layout + .store() + .put_overwrite(&layout.capsule_path(run.hash()), damaged.into()) + .await + .unwrap(); + let fresh = control + .visibility_reader(&layout, capsule.hash(), 128 * 1024) + .unwrap() + .unwrap(); + assert!( + fresh + .lookup(layout.store(), "refs/heads/main", &[oids[0]]) + .await + .is_err() + ); +} diff --git a/crates/crab-metadata/src/capsule_protocol/visibility.rs b/crates/crab-metadata/src/capsule_protocol/visibility.rs index 186e61420..0b4980f7f 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility.rs @@ -10,24 +10,24 @@ use crate::git_visibility::{ use super::valid_ref_name; -const VISIBILITY_DELTA_VERSION: u32 = 2; +const VISIBILITY_DELTA_VERSION: u32 = 3; + +mod paged; +pub(super) use paged::Descriptor as VisibilityPages; +#[cfg(feature = "storage")] +pub use paged::{CapsuleVisibilityReader, VisibilityChange}; const VISIBILITY_SNAPSHOT_VERSION: u32 = 3; /// Ref-keyed reachability changes authenticated by one publication capsule. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct CapsuleVisibilityDelta { - version: u32, edits: BTreeMap, } impl CapsuleVisibilityDelta { /// Build a canonical delta with exactly one reachability edit per changed live ref. pub fn new(edits: BTreeMap) -> Result { - let delta = Self { - version: VISIBILITY_DELTA_VERSION, - edits, - }; + let delta = Self { edits }; delta.validate()?; Ok(delta) } @@ -35,24 +35,11 @@ impl CapsuleVisibilityDelta { /// Encode this delta for an authenticated capsule section. pub fn encode(&self) -> Result { self.validate()?; - serde_json::to_vec(self).map(Bytes::from).map_err(|source| { - MetadataError::Internal(format!( - "capsule visibility delta serialization failed: {source}" - )) - }) + paged::encode(self) } pub(crate) fn decode(bytes: &[u8]) -> Result { - let delta: Self = - serde_json::from_slice(bytes).map_err(|source| MetadataError::CorruptObject { - path: "capsule-protocol visibility delta".to_owned(), - reason: format!("visibility delta is invalid JSON: {source}"), - })?; - delta.validate().map_err(as_corruption)?; - if delta.encode()?.as_ref() != bytes { - return Err(corrupt("visibility delta is not canonically encoded")); - } - Ok(delta) + paged::decode(bytes) } /// Return the ref-keyed reachability changes. @@ -62,11 +49,12 @@ impl CapsuleVisibilityDelta { } fn validate(&self) -> Result<()> { - if self.version != VISIBILITY_DELTA_VERSION { - return Err(contract_error("visibility delta version is unsupported")); - } - if self.edits.is_empty() { - return Err(contract_error("visibility delta must contain an edit")); + if self.edits.is_empty() + || self.edits.len() > crate::git_visibility::MAX_GIT_VISIBILITY_REFS + { + return Err(contract_error( + "visibility delta ref count is outside its bound", + )); } for (name, edit) in &self.edits { if !name.starts_with("refs/") || !valid_ref_name(name) { diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs new file mode 100644 index 000000000..6b972b6d1 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs @@ -0,0 +1,388 @@ +//! Canonical authenticated pages for one capsule visibility delta. +use super::*; +use crate::git_visibility::{ + GIT_VISIBILITY_EDIT_VERSION, MAX_GIT_VISIBILITY_INDEX_BYTES, decode_oid, encode_oid, +}; +use crate::pages::{Kind, PageRef, Writer}; + +#[cfg(feature = "storage")] +mod reader; +#[cfg(feature = "storage")] +pub use reader::{CapsuleVisibilityReader, VisibilityChange}; + +const MAGIC: &[u8; 8] = b"CRBVSD01"; +const TRAILER_BYTES: usize = 12; +const MAX_HEADER_BYTES: usize = 4096; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Descriptor { + version: u32, + body_bytes: u64, + body_hash: [u8; 32], + refs: PageRef, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Edit { + name: String, + old_oid: Option, + new_oid: String, + replaces: bool, + added: Option, + removed: Option, +} + +pub(super) fn encode(delta: &CapsuleVisibilityDelta) -> Result { + // Admit encoded OID work before allocating row buffers. Ref headers and page + // directories are checked against the same proof-byte ceiling after encoding. + let minimum = delta.edits.iter().try_fold(0u64, |total, (_, edit)| { + let rows = (edit.added.len() as u64).checked_add(edit.removed.len() as u64)?; + total.checked_add(rows.checked_mul(24)?) + }); + if minimum.is_none_or(|bytes| bytes > MAX_GIT_VISIBILITY_INDEX_BYTES) { + return Err(contract_error("visibility delta exceeds its byte bound")); + } + let mut writer = Writer::default(); + let mut refs = BTreeMap::new(); + for (name, edit) in &delta.edits { + let added = objects(&mut writer, &edit.added)?; + let removed = objects(&mut writer, &edit.removed)?; + let row = Edit { + name: name.clone(), + old_oid: edit.old_oid.clone(), + new_oid: edit.new_oid.clone(), + replaces: edit.replaces, + added, + removed, + }; + let key = blake3::hash(name.as_bytes()).as_bytes().to_vec(); + let value = serde_json::to_vec(&row) + .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; + if refs.insert(key, value).is_some() { + return Err(contract_error("visibility ref-name hash collision")); + } + } + let refs = writer.tree(Kind::VisibilityRefs, refs.into_iter().collect())?; + let descriptor = Descriptor { + version: VISIBILITY_DELTA_VERSION, + body_bytes: writer.bytes.len() as u64, + body_hash: *blake3::hash(&writer.bytes).as_bytes(), + refs, + }; + let header = serde_json::to_vec(&descriptor) + .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; + if header.len() > MAX_HEADER_BYTES { + return Err(contract_error( + "visibility descriptor exceeds its byte bound", + )); + } + writer.bytes.extend_from_slice(&header); + crate::pages::put_u32(&mut writer.bytes, header.len())?; + writer.bytes.extend_from_slice(MAGIC); + if writer.bytes.len() as u64 > MAX_GIT_VISIBILITY_INDEX_BYTES { + return Err(contract_error("visibility delta exceeds its byte bound")); + } + Ok(writer.bytes.into()) +} + +pub(super) fn decode(bytes: &[u8]) -> Result { + let descriptor = Descriptor::decode(bytes)?; + let body = &bytes[..descriptor.body_bytes as usize]; + if blake3::hash(body).as_bytes() != &descriptor.body_hash { + return Err(corrupt("visibility page body hash differs")); + } + let mut visited = BTreeMap::new(); + let refs = crate::pages::scan( + body, + &descriptor.refs, + Kind::VisibilityRefs, + "capsule visibility refs", + &mut visited, + )?; + let mut edits = BTreeMap::new(); + let mut change_bytes = 0u64; + for (key, value) in refs { + let row = Edit::decode(&key, &value, descriptor.body_bytes)?; + let count = row.added.as_ref().map_or(0, |root| u64::from(root.count)) + + row.removed.as_ref().map_or(0, |root| u64::from(root.count)); + change_bytes = change_bytes + .checked_add(count * 24) + .filter(|bytes| *bytes <= MAX_GIT_VISIBILITY_INDEX_BYTES) + .ok_or_else(|| corrupt("visibility decoded changes exceed their byte bound"))?; + let mut read_objects = |root: Option<&PageRef>| -> Result> { + root.map(|root| { + crate::pages::scan( + body, + root, + Kind::VisibilityChanges, + "capsule visibility objects", + &mut visited, + ) + }) + .transpose()? + .unwrap_or_default() + .into_iter() + .map(|(oid, value)| { + if !value.is_empty() { + return Err(corrupt("visibility object row has unexpected payload")); + } + Ok(encode_oid(&oid.try_into().map_err(|_| { + corrupt("visibility OID has invalid length") + })?)) + }) + .collect() + }; + let added = read_objects(row.added.as_ref())?; + let removed = read_objects(row.removed.as_ref())?; + let edit = GitVisibilityEdit { + version: GIT_VISIBILITY_EDIT_VERSION, + old_oid: row.old_oid, + new_oid: row.new_oid, + replaces: row.replaces, + added, + removed, + }; + if edits.insert(row.name, edit).is_some() { + return Err(corrupt("visibility ref repeats")); + } + } + let delta = CapsuleVisibilityDelta::new(edits).map_err(as_corruption)?; + // Rebuilding verifies canonical page placement and rejects unclaimed bytes, + // aliases with conflicting counts, and semantically re-signed malformed data. + if delta.encode()?.as_ref() != bytes { + return Err(corrupt("visibility pages are not canonical")); + } + Ok(delta) +} + +fn objects(writer: &mut Writer, oids: &[String]) -> Result> { + if oids.is_empty() { + return Ok(None); + } + let rows = oids + .iter() + .map(|oid| Ok((decode_oid(oid)?.to_vec(), Vec::new()))) + .collect::>>()?; + writer.tree(Kind::VisibilityChanges, rows).map(Some) +} + +impl Descriptor { + pub(crate) fn decode(bytes: &[u8]) -> Result { + if bytes.len() < TRAILER_BYTES + || bytes.len() as u64 > MAX_GIT_VISIBILITY_INDEX_BYTES + || &bytes[bytes.len() - MAGIC.len()..] != MAGIC + { + return Err(corrupt("visibility page trailer or size is invalid")); + } + let end = bytes.len() - TRAILER_BYTES; + let length = u32::from_le_bytes( + bytes[end..end + 4] + .try_into() + .map_err(|_| corrupt("visibility descriptor length is truncated"))?, + ) as usize; + if length == 0 || length > MAX_HEADER_BYTES { + return Err(corrupt("visibility descriptor length exceeds its bound")); + } + let start = end + .checked_sub(length) + .ok_or_else(|| corrupt("visibility descriptor range is invalid"))?; + let descriptor: Self = serde_json::from_slice(&bytes[start..end]) + .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; + if descriptor.version != VISIBILITY_DELTA_VERSION || descriptor.body_bytes != start as u64 { + return Err(corrupt( + "visibility descriptor version or body span is invalid", + )); + } + descriptor.validate_region(bytes.len() as u64)?; + Ok(descriptor) + } + + pub(crate) fn validate_region(&self, length: u64) -> Result<()> { + let header = serde_json::to_vec(self) + .map_err(|source| MetadataError::CapsuleVisibilityRecord { source })?; + if self.version != VISIBILITY_DELTA_VERSION + || header.len() > MAX_HEADER_BYTES + || self + .body_bytes + .checked_add(header.len() as u64) + .and_then(|n| n.checked_add(TRAILER_BYTES as u64)) + != Some(length) + || length > MAX_GIT_VISIBILITY_INDEX_BYTES + { + return Err(corrupt( + "visibility descriptor differs from its section span", + )); + } + self.refs.validate(Kind::VisibilityRefs, self.body_bytes)?; + if self.refs.count as usize > crate::git_visibility::MAX_GIT_VISIBILITY_REFS { + return Err(corrupt("visibility delta contains too many refs")); + } + Ok(()) + } +} + +impl Edit { + fn decode(key: &[u8], bytes: &[u8], body_bytes: u64) -> Result { + let row: Self = serde_json::from_slice(bytes) + .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; + if !row.name.starts_with("refs/") + || !valid_ref_name(&row.name) + || blake3::hash(row.name.as_bytes()).as_bytes() != key + { + return Err(corrupt("visibility ref key differs from its name")); + } + if let Some(old) = &row.old_oid { + decode_oid(old)?; + } + decode_oid(&row.new_oid)?; + let count = row.added.as_ref().map_or(0, |root| u64::from(root.count)) + + row.removed.as_ref().map_or(0, |root| u64::from(root.count)); + if count > crate::git_visibility::MAX_GIT_VISIBILITY_OBJECTS + || (row.replaces && (row.removed.is_some() || row.added.is_none())) + { + return Err(corrupt("visibility edit counts are invalid")); + } + for root in [&row.added, &row.removed].into_iter().flatten() { + root.validate(Kind::VisibilityChanges, body_bytes)?; + } + Ok(row) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn delta() -> CapsuleVisibilityDelta { + CapsuleVisibilityDelta::new(BTreeMap::from([ + ( + "refs/heads/main".into(), + GitVisibilityEdit::from_delta_objects( + Some("1".repeat(40)), + "2".repeat(40), + vec!["2".repeat(40)], + vec!["3".repeat(40)], + ), + ), + ( + "refs/heads/alias".into(), + GitVisibilityEdit::from_delta_objects( + Some("2".repeat(40)), + "2".repeat(40), + vec![], + vec![], + ), + ), + ])) + .unwrap() + } + + fn wire(body: Vec, mut descriptor: Descriptor) -> Vec { + descriptor.body_bytes = body.len() as u64; + descriptor.body_hash = *blake3::hash(&body).as_bytes(); + let header = serde_json::to_vec(&descriptor).unwrap(); + let mut bytes = body; + bytes.extend_from_slice(&header); + bytes.extend_from_slice(&(header.len() as u32).to_le_bytes()); + bytes.extend_from_slice(MAGIC); + bytes + } + + #[test] + fn paged_visibility_preserves_added_removed_and_empty_edits() { + let expected = delta(); + assert_eq!(decode(&encode(&expected).unwrap()).unwrap(), expected); + } + + #[test] + fn visibility_rejects_authenticated_unclaimed_bytes() { + let bytes = encode(&delta()).unwrap(); + let descriptor = Descriptor::decode(&bytes).unwrap(); + let mut body = bytes[..descriptor.body_bytes as usize].to_vec(); + body.push(0); + assert!(decode(&wire(body, descriptor)).is_err()); + } + + #[test] + fn visibility_pages_reject_corruption_and_retired_encodings() { + let bytes = encode(&delta()).unwrap(); + let mut damaged = bytes.to_vec(); + damaged[0] ^= 1; + let old = + serde_json::to_vec(&serde_json::json!({"version":2,"edits":delta().edits()})).unwrap(); + assert_eq!( + ( + decode(&damaged).is_err(), + decode(&old).is_err(), + decode(&bytes[..bytes.len() - 1]).is_err() + ), + (true, true, true) + ); + } + + #[test] + fn re_signed_visibility_cannot_add_and_remove_the_same_oid() { + let mut writer = Writer::default(); + let changes = writer + .tree(Kind::VisibilityChanges, vec![(vec![0x22; 20], vec![])]) + .unwrap(); + let row = Edit { + name: "refs/heads/main".into(), + old_oid: Some("1".repeat(40)), + new_oid: "2".repeat(40), + replaces: false, + added: Some(changes.clone()), + removed: Some(changes), + }; + let refs = writer + .tree( + Kind::VisibilityRefs, + vec![( + blake3::hash(row.name.as_bytes()).as_bytes().to_vec(), + serde_json::to_vec(&row).unwrap(), + )], + ) + .unwrap(); + let descriptor = Descriptor { + version: VISIBILITY_DELTA_VERSION, + body_bytes: 0, + body_hash: [0; 32], + refs, + }; + assert!(decode(&wire(writer.bytes, descriptor)).is_err()); + } + + #[test] + fn wide_ref_directory_pages_internal_routing_and_shared_changes() { + let edit = + GitVisibilityEdit::from_replacement_objects(None, "2".repeat(40), vec!["2".repeat(40)]); + let expected = CapsuleVisibilityDelta::new( + (0..2000) + .map(|i| (format!("refs/heads/branch-{i:04}"), edit.clone())) + .collect(), + ) + .unwrap(); + let bytes = encode(&expected).unwrap(); + let descriptor = Descriptor::decode(&bytes).unwrap(); + assert_eq!( + (descriptor.refs.height > 0, decode(&bytes).unwrap()), + (true, expected) + ); + } + + proptest::proptest! { + #[test] + fn visibility_change_pages_match_normalized_edits( + added in proptest::collection::vec(2u8..64,0..100), + removed in proptest::collection::vec(64u8..128,0..100), + ) { + let edit = GitVisibilityEdit::from_delta_objects(Some("1".repeat(40)), "2".repeat(40), + added.into_iter().map(|n|format!("{n:040x}")).collect(), + removed.into_iter().map(|n|format!("{n:040x}")).collect()); + let expected = CapsuleVisibilityDelta::new(BTreeMap::from([("refs/heads/main".into(),edit)])).unwrap(); + proptest::prop_assert_eq!(decode(&encode(&expected).unwrap()).unwrap(),expected); + } + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs new file mode 100644 index 000000000..16ef46d08 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs @@ -0,0 +1,239 @@ +//! Requested visibility changes from one authenticated embedded section. +use super::*; +use crate::pages::{PageCache, PageWork}; +use crab_storage::{Store, StoreLayout}; +use std::collections::BTreeSet; + +/// Change to an object's membership in one ref closure; never standalone authorization. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VisibilityChange { + Added, + Removed, +} + +/// Bounded lookups captured from a capsule run's authenticated visibility descriptor. +#[derive(Debug)] +pub struct CapsuleVisibilityReader { + path: object_store::path::Path, + offset: u64, + descriptor: Descriptor, + maximum: u64, + cache: PageCache, +} + +impl CapsuleVisibilityReader { + pub(crate) fn new( + layout: &StoreLayout, + run_hash: &str, + range: &super::super::super::PackRange, + descriptor: Descriptor, + maximum: u64, + ) -> Result { + descriptor.validate_region(range.length())?; + range + .offset() + .checked_add(range.length()) + .ok_or_else(|| corrupt("visibility source span overflows"))?; + Ok(Self { + path: layout.capsule_path(run_hash), + offset: range.offset(), + descriptor, + maximum, + cache: PageCache::default(), + }) + } + + /// Return requested changes in input order, or an authenticated absent-ref result. + /// + /// OID misses remain distinct from missing refs. The caller admits input/output + /// cardinality; cached and fetched pages share one encoded-work budget. + pub async fn lookup( + &self, + store: &Store, + ref_name: &str, + oids: &[[u8; 20]], + ) -> Result>>> { + if !ref_name.starts_with("refs/") || !valid_ref_name(ref_name) { + return Err(contract_error("visibility lookup ref name is invalid")); + } + let mut work = PageWork { + maximum: self.maximum, + bytes: 0, + seen: BTreeSet::new(), + }; + let key = blake3::hash(ref_name.as_bytes()).as_bytes().to_vec(); + let mut rows = self + .rows( + store, + &self.descriptor.refs, + Kind::VisibilityRefs, + std::slice::from_ref(&key), + &mut work, + ) + .await?; + let Some(value) = rows.remove(0) else { + return Ok(None); + }; + let row = Edit::decode(&key, &value, self.descriptor.body_bytes)?; + if row.name != ref_name { + return Err(corrupt("visibility lookup ref hash collides")); + } + let mut keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + if row.replaces { + keys.push(decode_oid(&row.new_oid)?.to_vec()); + } + // Separate authenticated sets give each direction an exact count. A loaded + // OID present in both is malformed evidence, never an implicit precedence. + let added = match &row.added { + Some(root) => { + self.rows(store, root, Kind::VisibilityChanges, &keys, &mut work) + .await? + } + None => vec![None; keys.len()], + }; + let removed = match &row.removed { + Some(root) => { + self.rows(store, root, Kind::VisibilityChanges, &keys, &mut work) + .await? + } + None => vec![None; keys.len()], + }; + let changes = added + .into_iter() + .zip(removed) + .map(|(added, removed)| match (added, removed) { + (None, None) => Ok(None), + (Some(value), None) if value.is_empty() => Ok(Some(VisibilityChange::Added)), + (None, Some(value)) if value.is_empty() => Ok(Some(VisibilityChange::Removed)), + _ => Err(corrupt( + "visibility object has conflicting sets or unexpected payload", + )), + }) + .collect::>>()?; + if row.replaces && changes.last() != Some(&Some(VisibilityChange::Added)) { + return Err(corrupt("replacement visibility does not admit its new tip")); + } + Ok(Some(changes.into_iter().take(oids.len()).collect())) + } + + async fn rows( + &self, + store: &Store, + root: &PageRef, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + let end = self + .offset + .checked_add(self.descriptor.body_bytes) + .ok_or_else(|| corrupt("visibility body span overflows"))?; + self.cache + .lookup( + store, + &self.path, + 0, + self.offset..end, + root, + kind, + keys, + work, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capsule_protocol::{ + Capsule, CapsuleRefEdit, CapsuleRun, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, + }; + use std::sync::Arc; + + #[tokio::test] + async fn visibility_lookup_distinguishes_removed_unchanged_and_absent_refs() { + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([ + ( + "refs/heads/main".into(), + GitVisibilityEdit::from_delta_objects( + Some("1".repeat(40)), + "2".repeat(40), + vec!["2".repeat(40)], + vec!["3".repeat(40)], + ), + ), + ( + "refs/heads/alias".into(), + GitVisibilityEdit::from_delta_objects( + Some("2".repeat(40)), + "2".repeat(40), + vec![], + vec![], + ), + ), + ])) + .unwrap(); + let transaction = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + Some("1".repeat(40)), + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![], + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule.clone()).unwrap(); + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "repositories/visibility".into(), + ); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let (control, _) = run.verified_controls().unwrap(); + let reader = control + .visibility_reader(&layout, capsule.hash(), 128 * 1024) + .unwrap() + .unwrap(); + let oids = [[0x33; 20], [0x22; 20], [0x44; 20], [0x33; 20]]; + assert_eq!( + ( + reader + .lookup(layout.store(), "refs/heads/main", &oids) + .await + .unwrap(), + reader + .lookup(layout.store(), "refs/heads/alias", &oids) + .await + .unwrap(), + reader + .lookup(layout.store(), "refs/heads/missing", &oids) + .await + .unwrap() + ), + ( + Some(vec![ + Some(VisibilityChange::Removed), + Some(VisibilityChange::Added), + None, + Some(VisibilityChange::Removed) + ]), + Some(vec![None; 4]), + None + ), + ); + } +} diff --git a/crates/crab-metadata/src/error.rs b/crates/crab-metadata/src/error.rs index 7943a3e45..e3fb0b52d 100644 --- a/crates/crab-metadata/src/error.rs +++ b/crates/crab-metadata/src/error.rs @@ -26,6 +26,12 @@ pub enum MetadataError { #[source] source: serde_json::Error, }, + /// A capsule visibility page descriptor or edit record could not be encoded or decoded. + #[error("capsule visibility record encoding is invalid")] + CapsuleVisibilityRecord { + #[source] + source: serde_json::Error, + }, /// A file lookup could not acquire process-wide execution capacity. #[cfg(feature = "file-index-reader")] #[error("file lookup admission closed")] diff --git a/crates/crab-metadata/src/git_visibility.rs b/crates/crab-metadata/src/git_visibility.rs index c5974438c..872282715 100644 --- a/crates/crab-metadata/src/git_visibility.rs +++ b/crates/crab-metadata/src/git_visibility.rs @@ -18,7 +18,7 @@ pub const GIT_VISIBILITY_INDEX_VERSION: u32 = 1; /// Maximum serialized proof accepted from object storage. pub const MAX_GIT_VISIBILITY_INDEX_BYTES: u64 = 128 * 1024 * 1024; -const MAX_GIT_VISIBILITY_REFS: usize = 100_000; +pub(crate) const MAX_GIT_VISIBILITY_REFS: usize = 100_000; /// Maximum number of distinct Git objects in one visibility proof dictionary. pub const MAX_GIT_VISIBILITY_OBJECTS: u64 = 10_000_000; @@ -2019,7 +2019,7 @@ fn validate_oid(value: &str) -> Result<()> { Ok(()) } -fn decode_oid(value: &str) -> Result { +pub(crate) fn decode_oid(value: &str) -> Result { validate_oid(value)?; let mut decoded = [0u8; 20]; let (pairs, remainder) = value.as_bytes().as_chunks::<2>(); @@ -2040,7 +2040,7 @@ fn hex_value(value: u8) -> u8 { } } -fn encode_oid(oid: &GitVisibilityOid) -> String { +pub(crate) fn encode_oid(oid: &GitVisibilityOid) -> String { const HEX: &[u8; 16] = b"0123456789abcdef"; let mut encoded = String::with_capacity(40); for byte in oid { diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 9c733711c..5047dbd62 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -57,6 +57,8 @@ pub(crate) enum Kind { PackSlots = 11, RunAdmission = 12, RunAdmissionCounts = 13, + VisibilityRefs = 14, + VisibilityChanges = 15, } impl Kind { @@ -75,8 +77,8 @@ impl Kind { pub(crate) fn key_bytes(self) -> usize { match self { - Self::Names | Self::CommitAncestryNodes => 32, - Self::CommitOids | Self::RunAdmission => 20, + Self::Names | Self::CommitAncestryNodes | Self::VisibilityRefs => 32, + Self::CommitOids | Self::RunAdmission | Self::VisibilityChanges => 20, #[cfg(feature = "remote-index")] Self::PackInventory => 32, #[cfg(feature = "remote-index")] @@ -342,6 +344,7 @@ pub(crate) fn decode( #[cfg(feature = "remote-index")] Kind::ObjectLocators | Kind::PackOffsets | Kind::PackInventory | Kind::PackSlots => "git_object_index", Kind::RunAdmission | Kind::RunAdmissionCounts => "capsule_admission", + Kind::VisibilityRefs | Kind::VisibilityChanges => "capsule_visibility", _ => "path_state", }, decoded_bytes=bytes.len() as u64, "index page decoded"); diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index d34332dd1..5edeed14b 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -25,8 +25,8 @@ pub struct Measurements { read_bytes: AtomicU64, written_bytes: AtomicU64, active: AtomicU64, - decoded_blocks: [AtomicU64; 4], - decoded_bytes: [AtomicU64; 4], + decoded_blocks: [AtomicU64; 5], + decoded_bytes: [AtomicU64; 5], } #[derive(Clone, Copy)] @@ -35,8 +35,8 @@ struct Snapshot { outcomes: [u64; 9], read_bytes: u64, written_bytes: u64, - decoded_blocks: [u64; 4], - decoded_bytes: [u64; 4], + decoded_blocks: [u64; 5], + decoded_bytes: [u64; 5], } impl StorageObserver for Measurements { @@ -72,6 +72,7 @@ impl tracing::field::Visit for DecodeEvent { "path_state" => Some(1), "git_object_index" => Some(2), "capsule_admission" => Some(3), + "capsule_visibility" => Some(4), _ => None, }; } @@ -166,6 +167,7 @@ impl Measurements { "path_state", "git_object_index", "capsule_admission", + "capsule_visibility", ] .into_iter() .enumerate() @@ -242,12 +244,13 @@ mod tests { tracing::debug!(target:"crab_metadata::index_decode", index_format="path_state", decoded_bytes=123_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="git_object_index", decoded_bytes=67_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_admission", decoded_bytes=29_u64); + tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_visibility", decoded_bytes=31_u64); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), - ([1, 1, 1, 1], [41, 123, 67, 29]) + ([1, 1, 1, 1, 1], [41, 123, 67, 29, 31]) ); } From d54d6eef5c268befeddb4c44db769b970023184b Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 08:43:32 -0700 Subject: [PATCH 024/127] perf(read): reconcile paged visibility lazily Keep captured ref closures authoritative while deferring full OID sets. Bulk visibility APIs become async and authenticate the captured source. --- .../docs/architecture/indexed-remote-reads.md | 63 ++ crab/src/cmd/metadb.rs | 4 +- crab/src/git/remote_helper.rs | 10 +- crab/src/git/upload_pack_wire.rs | 4 +- .../src/receive/git_workspace.rs | 2 +- crates/crab-auth-server/src/view.rs | 1 + crates/crab-http-server/src/git.rs | 2 +- .../crab-http-server/src/receive/publish.rs | 2 +- .../src/receive_fault_tests.rs | 2 +- crates/crab-metadata/README.md | 9 +- .../crab-metadata/src/capsule_protocol/mod.rs | 2 +- .../crab-metadata/src/capsule_protocol/run.rs | 134 +++- .../src/capsule_protocol/store.rs | 8 +- .../src/capsule_protocol/visibility.rs | 2 +- .../src/capsule_protocol/visibility/paged.rs | 49 +- .../visibility/paged/reader.rs | 155 +++- crates/crab-metadata/src/pages/cache.rs | 70 +- crates/crab-read/README.md | 19 +- crates/crab-read/src/capsule_protocol.rs | 174 +++-- .../src/capsule_protocol/visibility.rs | 669 ++++++++++++++++++ crates/crab-remote/src/checkpoint.rs | 2 +- crates/crab-remote/tests/checkpoint.rs | 35 +- .../tests/checkpoint/frontier_admission.rs | 1 + 23 files changed, 1277 insertions(+), 142 deletions(-) create mode 100644 crates/crab-read/src/capsule_protocol/visibility.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 3c4551a91..108591800 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -919,3 +919,66 @@ hashes match the receipts and remain external. One sample per query establishes publication/read smoke, not the final 20-sample performance cells. Complete repository opening still materializes visibility and catalog state. This format/lookup work does not yet satisfy bounded opening, the remaining stage-4 gates, the 10k path-history gate or stages 5–8. + + +## Lazy frontier visibility reconciliation + +Git control capture now keeps explicit paged visibility evidence. Scalar ref edit +headers are authenticated in batches and matched to the unique transaction edits; +unloaded evidence is never treated as absence. Complete and paged controls share +one ref-ordering loop, retaining expected-old links, activation and predecessor +ordering, exact final refs and peeled refs, and uncommitted/deleted-ref checks. + +The read owner represents closures as immutable parent/edit layers. A new ref +selects the same first visible inherited base as complete reconciliation. Moving +the source ref later cannot change that captured parent. Prior tips and removals +are checked through membership lookup; removed OIDs cannot also be added. The +historical dictionary retains removed objects without making them visible. +An initial authenticated added root supplies its exact dictionary count; equal +roots add no objects. Differing actual additions are read and checked against +the retained dictionary, preserving its existing maximum. Actual edit work can +still be large; unrelated historical closures remain cold in the initial case. + +Bulk `git_visibility_index`, `candidate_git_visibility` and `tip_bound_transitions` +are asynchronous and materialize the same captured controls only on request. +Immutable section hashes, descriptors and canonical bytes are checked before +bulk visibility is exposed. Tip-bound planning retains its complete captured +transition source, independent of the controls still needed for current refs. +CLI, HTTP, protected-service and checkpoint callers await that proof. Diagnostic +formatting excludes the captured credential-bearing transport. The wire format +is unchanged from the preceding visibility-format slice. + +Focused proof passes 46 capsule-read tests, 38 checkpoint integration tests, +86 payload-only and 94 storage/remote-index capsule tests, strict metadata/read +Clippy and CLI/HTTP/S3/protected-service consumer checks after a frontend build. +The 10k-OID regression opens below 64 KiB while leaving complete visibility +unloaded. Layer tests match complete edits, preserve inherited bases and reject +removals outside the prior closure. Replacing ref membership with dictionary +presence makes the exact retained-invisible-object test fail; restored source +passes. Final-source RustFS replay passes 41 qualifier processes: all ten queries +at 100 commits and 20 cold/warm resolve samples per 100/1,000-commit history. +Warm resolve backend bytes remain zero, and receipts match the qualified source. +Opening bytes change from 31,888/121,857 to 31,292/79,065 at 100/1,000 commits. +These are local debug observations, not a representative-provider latency target. + +Final-source isolated `make install` passes for CLI/FUSE/NFS/cache-server and the +Git helper. A fresh-prefix smoke using that CLI passes 39 checks and 26 commands: +real Crab push, index maintenance and all ten supported-query/native-Git checks +at 100 commits. Shared-consumer tests pass 21 path-state, 27 graph and 78 object- +locator tests (one existing ignored test), plus 27 cross-surface paged tests. + +The fresh 10,000-commit corpus publishes successfully, but its full query run +fails path-history admission: 100,010 entries exceed the 100,000-entry maximum. +That failed report is retained. Separate final-source resolve and attribution +qualification passes 20 samples each against the same corpus; opening reads +81,743 bytes and warm query backend bytes remain zero. Source hashes match both +replay receipts. These targeted results do not turn the full-query failure into +a pass. Receipts remain on the mounted qualification volume, outside the PR. + +Parent PR #547 CI remains failed after attempt 2. The original effect-delivery +cancellation reservation failure passes on retry; an activity-lease-expiry test +then leaves 74,168 local-disk bytes reserved after shutdown. The cleanup cause +remains unresolved. No test, baseline or expected-failure assertion is weakened. +Checkpoint visibility/catalog decoding, ref/source capture and broader frontier +amplification remain open. The 10k path-history budget failure, stages 5–8 and +unified staging remain unresolved; this slice does not complete the initiative. diff --git a/crab/src/cmd/metadb.rs b/crab/src/cmd/metadb.rs index 2efb6f16d..fc7f1f0f6 100644 --- a/crab/src/cmd/metadb.rs +++ b/crab/src/cmd/metadb.rs @@ -1738,7 +1738,7 @@ async fn diagnose_capsule( ) .await?; let catalog = view.pointer_catalog()?; - view.git_visibility_index()?; + view.git_visibility_index().await?; // Verification must read layered sources and prove reachable content, // not rebuild a complete pack or accept catalog membership as file proof. let dependencies = crab_read::capsule_protocol::verify_reachable_dependencies( @@ -2663,7 +2663,7 @@ async fn run_capsule_rebuild_in( ) .await?; let catalog = view.pointer_catalog()?; - view.git_visibility_index()?; + view.git_visibility_index().await?; // Authenticate the complete closure before either a no-op success or a new // checkpoint; immutable publication alone does not verify reachable files. crab_read::capsule_protocol::verify_reachable_dependencies( diff --git a/crab/src/git/remote_helper.rs b/crab/src/git/remote_helper.rs index 04f710e11..b95344ac3 100644 --- a/crab/src/git/remote_helper.rs +++ b/crab/src/git/remote_helper.rs @@ -3129,14 +3129,14 @@ async fn fetch_capsule_incremental_packs( &repository, &visible_ref_names, &request, - Some(view.tip_bound_transitions()), + Some(&view.tip_bound_transitions().await?), cancel, ) .await } else { crab_read::plan_upload_pack( &repository, - &view.git_visibility_index()?, + &view.git_visibility_index().await?, &visible_ref_names, &request, cancel, @@ -3582,7 +3582,7 @@ async fn fetch_capsule_constrained_pack( ..Default::default() }; let repository = capsule_git_repository(&view, store, router, config, runtime, cancel).await?; - let visibility = view.git_visibility_index()?; + let visibility = view.git_visibility_index().await?; let plan = crab_read::plan_upload_pack(&repository, &visibility, &visible_refs, &request, cancel) .await?; @@ -3735,7 +3735,7 @@ async fn fetch_capsule_promisor_objects( filtered_promisor, )?; let repository = capsule_git_repository(&view, store, router, config, runtime, cancel).await?; - let visibility = view.git_visibility_index()?; + let visibility = view.git_visibility_index().await?; let request = crab_read::UploadPackRequest { wants, filter: crab_read::UploadPackFilter::None, @@ -5400,7 +5400,7 @@ mod tests { ) .await .expect("open bounded capsule reader"); - let visibility = view.git_visibility_index().expect("visibility proof"); + let visibility = view.git_visibility_index().await.expect("visibility proof"); let refs = ["refs/heads/main".to_owned()]; let plan = crab_read::plan_upload_pack( &repository, diff --git a/crab/src/git/upload_pack_wire.rs b/crab/src/git/upload_pack_wire.rs index 9f5f7777d..bcc427c81 100644 --- a/crab/src/git/upload_pack_wire.rs +++ b/crab/src/git/upload_pack_wire.rs @@ -1441,10 +1441,10 @@ async fn open_capsule_repository_from_root( let identity = RepositoryIdentity::new(provider, prefix.to_owned(), 1).map_err(remote_error)?; let proof = if tip_bound { UploadPackVisibilityProof::TipBound { - transitions: Arc::new(view.tip_bound_transitions().clone()), + transitions: Arc::new(view.tip_bound_transitions().await?), } } else { - UploadPackVisibilityProof::Materialized(view.git_visibility_index()?) + UploadPackVisibilityProof::Materialized(view.git_visibility_index().await?) }; let repository = view .git_repository_from_store( diff --git a/crates/crab-auth-server/src/receive/git_workspace.rs b/crates/crab-auth-server/src/receive/git_workspace.rs index 600a65511..6a6036163 100644 --- a/crates/crab-auth-server/src/receive/git_workspace.rs +++ b/crates/crab-auth-server/src/receive/git_workspace.rs @@ -138,7 +138,7 @@ pub(super) async fn verify_capsule_git_candidate( .into_iter() .map(|(name, reachable)| (name, reachable_object_ids(&reachable))) .collect::>(); - let declared = view.candidate_git_visibility(candidate)?; + let declared = view.candidate_git_visibility(candidate).await?; if closures != declared { return Err(invalid( "candidate capsule Git visibility differs from its reachable object closure", diff --git a/crates/crab-auth-server/src/view.rs b/crates/crab-auth-server/src/view.rs index ce89b3499..7d32ebe92 100644 --- a/crates/crab-auth-server/src/view.rs +++ b/crates/crab-auth-server/src/view.rs @@ -1355,6 +1355,7 @@ mod tests { ); assert!( view.git_visibility_index() + .await .unwrap() .contains_hex_in_ref("refs/heads/main", expected.trim()) ); diff --git a/crates/crab-http-server/src/git.rs b/crates/crab-http-server/src/git.rs index 6f8c59669..44915d6d9 100644 --- a/crates/crab-http-server/src/git.rs +++ b/crates/crab-http-server/src/git.rs @@ -278,7 +278,7 @@ pub(crate) async fn upload_pack( } "fetch" => { let request = wire::parse_fetch(&request.args)?; - let visibility = view.git_visibility_index()?; + let visibility = view.git_visibility_index().await?; let refs = repository .refs() .entries diff --git a/crates/crab-http-server/src/receive/publish.rs b/crates/crab-http-server/src/receive/publish.rs index 1c1e52d29..8f63fc224 100644 --- a/crates/crab-http-server/src/receive/publish.rs +++ b/crates/crab-http-server/src/receive/publish.rs @@ -424,7 +424,7 @@ async fn publish_attempt( .into()); } let refs = view.refs().clone(); - let visibility = view.git_visibility_index()?; + let visibility = view.git_visibility_index().await?; let has_branch = refs.keys().any(|name| name.starts_with("refs/heads/")); let actor = principal.identity().ok_or(ReceiveError::Forbidden)?; let initial_head = (!has_branch) diff --git a/crates/crab-http-server/src/receive_fault_tests.rs b/crates/crab-http-server/src/receive_fault_tests.rs index ff99a3f98..1413da2c4 100644 --- a/crates/crab-http-server/src/receive_fault_tests.rs +++ b/crates/crab-http-server/src/receive_fault_tests.rs @@ -210,7 +210,7 @@ async fn prepared_artifacts_preserve_plan_attribution_through_compaction() { &cancel, move |_holders, cancel| async move { let view = leased_repo.open_view().await?; - let visibility = view.git_visibility_index()?; + let visibility = view.git_visibility_index().await?; let repository = view .git_repository( leased_repo.identity.clone(), diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index ca91e8a60..0cba3ca65 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -223,8 +223,13 @@ Replacement edits must admit their new tip; queried OIDs cannot belong to both change sets. These changes alone never authorize a ref or object want. The capsule delta's retired JSON shape has no reader or serialization alias; retired run prefixes require republishing. Released v1 visibility index and -manifest/catalog formats are unchanged. Complete repository reconciliation -still loads visibility; lazy closure reconciliation remains unfinished. +manifest/catalog formats are unchanged. Git control capture retains explicit +paged evidence; `visibility_delta` returns an error for unloaded data rather +than treating it as absence. Scalar header reads and actual-set inspection share +the pager's encoded-work admission. Complete materialization verifies the captured +section hash and descriptor, checks canonical delta bytes, and retains immutable +shared results. `crab-read` owns lazy closure reconciliation; complete metadata +capture remains available for maintenance and publication. Payload modules cover manifests, segmented lists, pack metadata, commit-graph summaries, ref registries, chunk/file indexes, receipts, transactions, and diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index ff6d3b1d9..680b807e9 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -71,7 +71,7 @@ pub use transaction_record::{ }; pub use visibility::{CapsuleVisibilityDelta, CapsuleVisibilitySnapshot}; #[cfg(feature = "storage")] -pub use visibility::{CapsuleVisibilityReader, VisibilityChange}; +pub use visibility::{CapsuleVisibilityReader, VisibilityChange, VisibilityEditHeader}; fn valid_ref_name(name: &str) -> bool { gix_validate::reference::name_partial(name.as_bytes().as_bstr()).is_ok() diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index 69611d214..f9f6f9d59 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -128,11 +128,18 @@ pub struct CapsuleControl { transaction_id: String, base_root_digest: String, transaction: CapsuleTransaction, - visibility: Option, + visibility: Option, catalog: Option, git_packs: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +enum ControlVisibility { + Complete(std::sync::Arc), + #[cfg(feature = "storage")] + Paged(std::sync::Arc), +} + impl CapsuleRunControl { /// Decode complete control and admission bytes from the authenticated suffix. pub fn decode_suffix( @@ -521,6 +528,57 @@ impl CapsuleRunControl { pub(crate) fn materialize_capsules_with_external_controls( &self, external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, + ) -> Result> { + self.materialize_controls(external_controls, |location| { + let visibility_bytes = external_controls + .get(&(location.hash.clone(), CapsuleSectionKind::VisibilityDelta)) + .cloned(); + if visibility_bytes + .as_ref() + .map(|bytes| super::visibility::VisibilityPages::decode(bytes)) + .transpose()? + != location.visibility_pages + { + return Err(corrupt( + "visibility page descriptor differs from its run footer", + )); + } + let delta = visibility_bytes + .as_ref() + .map(|bytes| CapsuleVisibilityDelta::decode(bytes)) + .transpose()?; + verify_control_section( + location.visibility.as_ref(), + visibility_bytes.as_ref(), + "visibility", + )?; + Ok(delta.map(|delta| ControlVisibility::Complete(std::sync::Arc::new(delta)))) + }) + } + + #[cfg(feature = "storage")] + pub(crate) fn materialize_capsules_with_paged_visibility( + &self, + layout: &crab_storage::StoreLayout, + external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, + ) -> Result> { + self.materialize_controls(external_controls, |location| { + self.visibility_reader( + layout, + &location.hash, + crate::git_visibility::MAX_GIT_VISIBILITY_INDEX_BYTES, + ) + .map(|reader| { + reader.map(|reader| ControlVisibility::Paged(std::sync::Arc::new(reader))) + }) + }) + } + + #[cfg(any(feature = "storage", test))] + fn materialize_controls( + &self, + external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, + visibility: impl Fn(&CapsuleControlLocation) -> Result>, ) -> Result> { self.capsules .iter() @@ -530,29 +588,12 @@ impl CapsuleRunControl { Some(&location.transaction_bytes), "transaction", )?; - let visibility_bytes = external_controls - .get(&(location.hash.clone(), CapsuleSectionKind::VisibilityDelta)) - .cloned(); - if visibility_bytes - .as_ref() - .map(|bytes| super::visibility::VisibilityPages::decode(bytes)) - .transpose()? - != location.visibility_pages - { - return Err(corrupt( - "visibility page descriptor differs from its run footer", - )); - } + let visibility = visibility(location)?; let catalog_bytes = location.catalog_bytes.clone().or_else(|| { external_controls .get(&(location.hash.clone(), CapsuleSectionKind::CatalogDelta)) .cloned() }); - verify_control_section( - location.visibility.as_ref(), - visibility_bytes.as_ref(), - "visibility", - )?; verify_control_section( location.catalog.as_ref(), catalog_bytes.as_ref(), @@ -564,10 +605,6 @@ impl CapsuleRunControl { { return Err(corrupt("capsule run transaction does not match its footer")); } - let visibility = visibility_bytes - .as_ref() - .map(|bytes| CapsuleVisibilityDelta::decode(bytes)) - .transpose()?; let catalog = catalog_bytes .as_ref() .map(|bytes| PointerCatalog::decode_delta(bytes)) @@ -610,9 +647,44 @@ impl CapsuleControl { &self.transaction } - /// Return the visibility delta, when present. - pub fn visibility_delta(&self) -> Option<&CapsuleVisibilityDelta> { - self.visibility.as_ref() + /// Return already materialized visibility, distinguishing cold data from absence. + pub fn visibility_delta(&self) -> Result> { + match &self.visibility { + None => Ok(None), + Some(ControlVisibility::Complete(delta)) => Ok(Some(delta)), + #[cfg(feature = "storage")] + Some(ControlVisibility::Paged(_)) => Err(corrupt( + "visibility is paged; complete inspection must materialize it", + )), + } + } + + /// Whether this control advertises visibility evidence, loaded or paged. + pub fn has_visibility(&self) -> bool { + self.visibility.is_some() + } + + #[cfg(feature = "storage")] + /// Return captured paged evidence without materializing its OID sets. + pub fn paged_visibility( + &self, + ) -> Option<&std::sync::Arc> { + match &self.visibility { + Some(ControlVisibility::Paged(reader)) => Some(reader), + _ => None, + } + } + + #[cfg(feature = "storage")] + /// Materialize captured visibility for bulk operations without recapturing ref state. + pub async fn materialize_visibility(&self, store: &crab_storage::Store) -> Result { + let mut control = self.clone(); + if let Some(ControlVisibility::Paged(reader)) = &self.visibility { + control.visibility = Some(ControlVisibility::Complete( + reader.materialize(store).await?, + )); + } + Ok(control) } /// Return the pointer catalog delta, when present. @@ -2001,7 +2073,15 @@ mod tests { section, )])) .unwrap(); - assert_eq!(controls[0].visibility_delta().unwrap().edits().len(), 1); + assert_eq!( + controls[0] + .visibility_delta() + .unwrap() + .unwrap() + .edits() + .len(), + 1 + ); assert_eq!(run.verified_controls().unwrap().1, controls); } } diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index 5ba2b5a99..5f6725c38 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -393,7 +393,7 @@ pub async fn load_capsule_run_control( "capsule pointer footer hash does not match the authenticated run suffix", )); } - let detached = load_detached_control_sections(router, &path, &control).await?; + let detached = load_detached_control_sections(router, &path, &control, true).await?; let capsules = control.materialize_capsules_with_external_controls(&detached)?; Ok((control, capsules)) } @@ -434,8 +434,8 @@ pub async fn load_capsule_run_git_control( )); } } - let detached = load_detached_control_sections(router, &path, &control).await?; - let capsules = control.materialize_capsules_with_external_controls(&detached)?; + let detached = load_detached_control_sections(router, &path, &control, false).await?; + let capsules = control.materialize_capsules_with_paged_visibility(router, &detached)?; Ok((control, capsules)) } @@ -443,11 +443,13 @@ async fn load_detached_control_sections( router: &StoreLayout, path: &object_store::path::Path, control: &CapsuleRunControl, + complete_visibility: bool, ) -> Result> { let ranges = control .capsule_locations() .iter() .flat_map(|location| location.detached_controls()) + .filter(|(_, kind, _)| complete_visibility || *kind != CapsuleSectionKind::VisibilityDelta) .collect::>(); futures_util::stream::iter(ranges.into_iter().map(|(hash, kind, range)| async move { let end = range diff --git a/crates/crab-metadata/src/capsule_protocol/visibility.rs b/crates/crab-metadata/src/capsule_protocol/visibility.rs index 0b4980f7f..b588213e4 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility.rs @@ -15,7 +15,7 @@ const VISIBILITY_DELTA_VERSION: u32 = 3; mod paged; pub(super) use paged::Descriptor as VisibilityPages; #[cfg(feature = "storage")] -pub use paged::{CapsuleVisibilityReader, VisibilityChange}; +pub use paged::{CapsuleVisibilityReader, VisibilityChange, VisibilityEditHeader}; const VISIBILITY_SNAPSHOT_VERSION: u32 = 3; /// Ref-keyed reachability changes authenticated by one publication capsule. diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs index 6b972b6d1..3ae87ee33 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs @@ -23,9 +23,10 @@ pub(crate) struct Descriptor { refs: PageRef, } -#[derive(Debug, Serialize, Deserialize)] +/// Authenticated scalar edit metadata and opaque added/removed set roots. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct Edit { +pub struct VisibilityEditHeader { name: String, old_oid: Option, new_oid: String, @@ -49,7 +50,7 @@ pub(super) fn encode(delta: &CapsuleVisibilityDelta) -> Result { for (name, edit) in &delta.edits { let added = objects(&mut writer, &edit.added)?; let removed = objects(&mut writer, &edit.removed)?; - let row = Edit { + let row = VisibilityEditHeader { name: name.clone(), old_oid: edit.old_oid.clone(), new_oid: edit.new_oid.clone(), @@ -104,7 +105,7 @@ pub(super) fn decode(bytes: &[u8]) -> Result { let mut edits = BTreeMap::new(); let mut change_bytes = 0u64; for (key, value) in refs { - let row = Edit::decode(&key, &value, descriptor.body_bytes)?; + let row = VisibilityEditHeader::decode(&key, &value, descriptor.body_bytes)?; let count = row.added.as_ref().map_or(0, |root| u64::from(root.count)) + row.removed.as_ref().map_or(0, |root| u64::from(root.count)); change_bytes = change_bytes @@ -223,7 +224,43 @@ impl Descriptor { } } -impl Edit { +impl VisibilityEditHeader { + /// Return the prior tip used by the edit. + pub fn old_oid(&self) -> Option<&str> { + self.old_oid.as_deref() + } + /// Return the edit's new tip. + pub fn new_oid(&self) -> &str { + &self.new_oid + } + /// Whether the added set replaces the prior closure. + pub fn replaces(&self) -> bool { + self.replaces + } + /// Return the exact authenticated added-set count. + pub fn added_count(&self) -> u32 { + self.added.as_ref().map_or(0, |root| root.count) + } + /// Return the exact authenticated removed-set count. + pub fn removed_count(&self) -> u32 { + self.removed.as_ref().map_or(0, |root| root.count) + } + /// Compare rooted addition identities without treating locations as identities. + pub fn same_additions(&self, other: &Self) -> bool { + match (&self.added, &other.added) { + (None, None) => true, + (Some(a), Some(b)) => { + a.hash == b.hash + && a.bytes == b.bytes + && a.height == b.height + && a.count == b.count + && a.first == b.first + && a.last == b.last + } + _ => false, + } + } + fn decode(key: &[u8], bytes: &[u8], body_bytes: u64) -> Result { let row: Self = serde_json::from_slice(bytes) .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; @@ -328,7 +365,7 @@ mod tests { let changes = writer .tree(Kind::VisibilityChanges, vec![(vec![0x22; 20], vec![])]) .unwrap(); - let row = Edit { + let row = VisibilityEditHeader { name: "refs/heads/main".into(), old_oid: Some("1".repeat(40)), new_oid: "2".repeat(40), diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs index 16ef46d08..faff57b6d 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs @@ -18,6 +18,9 @@ pub struct CapsuleVisibilityReader { offset: u64, descriptor: Descriptor, maximum: u64, + length: u64, + hash: String, + complete: std::sync::OnceLock>, cache: PageCache, } @@ -39,10 +42,149 @@ impl CapsuleVisibilityReader { offset: range.offset(), descriptor, maximum, + length: range.length(), + hash: range.blake3().to_owned(), + complete: std::sync::OnceLock::new(), cache: PageCache::default(), }) } + /// Return the exact number of ref headers advertised by this captured descriptor. + pub fn ref_count(&self) -> u32 { + self.descriptor.refs.count + } + + /// Read only scalar edit metadata, preserving requested ref order and authenticated misses. + pub async fn headers( + &self, + store: &Store, + names: &[&str], + ) -> Result>> { + let mut work = PageWork { + maximum: self.maximum, + bytes: 0, + seen: BTreeSet::new(), + }; + self.read_headers(store, names, &mut work).await + } + + async fn read_headers( + &self, + store: &Store, + names: &[&str], + work: &mut PageWork, + ) -> Result>> { + if names + .iter() + .any(|name| !name.starts_with("refs/") || !valid_ref_name(name)) + { + return Err(contract_error("visibility lookup ref name is invalid")); + } + let keys = names + .iter() + .map(|name| blake3::hash(name.as_bytes()).as_bytes().to_vec()) + .collect::>(); + self.rows( + store, + &self.descriptor.refs, + Kind::VisibilityRefs, + &keys, + work, + ) + .await? + .into_iter() + .zip(keys) + .zip(names) + .map(|((value, key), name)| { + let row = value + .map(|bytes| VisibilityEditHeader::decode(&key, &bytes, self.descriptor.body_bytes)) + .transpose()?; + if row.as_ref().is_some_and(|row| row.name != *name) { + return Err(corrupt("visibility lookup ref hash collides")); + } + Ok(row) + }) + .collect() + } + + /// Inspect one actual edit set through bounded authenticated pages. + pub async fn objects( + &self, + store: &Store, + name: &str, + direction: VisibilityChange, + ) -> Result> { + let mut work = PageWork { + maximum: self.maximum, + bytes: 0, + seen: BTreeSet::new(), + }; + let row = self + .read_headers(store, &[name], &mut work) + .await? + .pop() + .flatten() + .ok_or_else(|| corrupt("required visibility ref is absent"))?; + let root = match direction { + VisibilityChange::Added => row.added, + VisibilityChange::Removed => row.removed, + }; + let Some(root) = root else { + return Ok(Vec::new()); + }; + let end = self + .offset + .checked_add(self.descriptor.body_bytes) + .ok_or_else(|| corrupt("visibility body span overflows"))?; + self.cache + .scan( + store, + &self.path, + 0, + self.offset..end, + &root, + Kind::VisibilityChanges, + &mut work, + ) + .await? + .into_iter() + .map(|(oid, value)| { + if !value.is_empty() { + return Err(corrupt("visibility object row has unexpected payload")); + } + oid.try_into() + .map_err(|_| corrupt("visibility OID has invalid length")) + }) + .collect() + } + + /// Authenticate and materialize the complete captured delta for bulk inspection. + pub async fn materialize( + &self, + store: &Store, + ) -> Result> { + if self.length > self.maximum { + return Err(corrupt("complete visibility exceeds its byte limit")); + } + if let Some(delta) = self.complete.get() { + return Ok(delta.clone()); + } + let end = self + .offset + .checked_add(self.length) + .ok_or_else(|| corrupt("visibility source span overflows"))?; + let bytes = store.range_get(&self.path, self.offset..end).await?; + if blake3::hash(&bytes).to_hex().as_str() != self.hash + || Descriptor::decode(&bytes)? != self.descriptor + { + return Err(corrupt( + "visibility section differs from its captured descriptor", + )); + } + let delta = std::sync::Arc::new(CapsuleVisibilityDelta::decode(&bytes)?); + Ok(self.complete.get_or_init(|| delta).clone()) + } + /// Return requested changes in input order, or an authenticated absent-ref result. /// /// OID misses remain distinct from missing refs. The caller admits input/output @@ -74,7 +216,7 @@ impl CapsuleVisibilityReader { let Some(value) = rows.remove(0) else { return Ok(None); }; - let row = Edit::decode(&key, &value, self.descriptor.body_bytes)?; + let row = VisibilityEditHeader::decode(&key, &value, self.descriptor.body_bytes)?; if row.name != ref_name { return Err(corrupt("visibility lookup ref hash collides")); } @@ -143,6 +285,17 @@ impl CapsuleVisibilityReader { } } +impl PartialEq for CapsuleVisibilityReader { + fn eq(&self, other: &Self) -> bool { + self.path == other.path + && self.offset == other.offset + && self.length == other.length + && self.hash == other.hash + && self.descriptor == other.descriptor + } +} +impl Eq for CapsuleVisibilityReader {} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs index 74a37f0f2..1138d86cf 100644 --- a/crates/crab-metadata/src/pages/cache.rs +++ b/crates/crab-metadata/src/pages/cache.rs @@ -24,6 +24,21 @@ pub(crate) struct PageWork { pub(crate) seen: BTreeSet<(usize, u64)>, } +impl PageWork { + fn admit(&mut self, layer: usize, offset: u64, reference: &PageRef) -> Result<()> { + if self.seen.insert((layer, offset)) { + self.bytes = self + .bytes + .checked_add(u64::from(reference.bytes)) + .ok_or_else(|| corruption("query byte count overflows"))?; + if self.bytes > self.maximum { + return Err(corruption("metadata query exceeds its byte limit")); + } + } + Ok(()) + } +} + impl PageCache { #[expect( clippy::too_many_arguments, @@ -55,15 +70,7 @@ impl PageCache { while !pending.is_empty() { for (reference, _) in &pending { reference.validate(kind, source_bytes)?; - if work.seen.insert((layer, source.start + reference.offset)) { - work.bytes = work - .bytes - .checked_add(u64::from(reference.bytes)) - .ok_or_else(|| corruption("query byte count overflows"))?; - if work.bytes > work.maximum { - return Err(corruption("metadata query exceeds its byte limit")); - } - } + work.admit(layer, source.start + reference.offset, reference)?; } // Charge the entire wave before starting I/O, including warm pages. let pages = futures_util::stream::iter(std::mem::take(&mut pending)) @@ -114,6 +121,51 @@ impl PageCache { Ok(results) } + pub(crate) async fn scan( + &self, + store: &Store, + path: &object_store::path::Path, + layer: usize, + source: std::ops::Range, + root: &PageRef, + kind: Kind, + work: &mut PageWork, + ) -> Result { + let source_bytes = source + .end + .checked_sub(source.start) + .ok_or_else(|| corruption("invalid page source range"))?; + let mut pending = vec![root.clone()]; + let mut rows = Vec::new(); + while !pending.is_empty() { + for reference in &pending { + reference.validate(kind, source_bytes)?; + work.admit(layer, source.start + reference.offset, reference)?; + } + let pages = futures_util::stream::iter(std::mem::take(&mut pending)) + .map(|reference| { + let source = source.clone(); + async move { + self.fetch(store, path, layer, source, &reference, kind) + .await + } + }) + .buffered(4) + .try_collect::>() + .await?; + for page in pages { + match page.as_ref() { + Page::Leaf(values) => rows.extend(values.iter().cloned()), + Page::Branch(children) => pending.extend(children.iter().cloned()), + } + } + } + if rows.len() != root.count as usize || rows.windows(2).any(|pair| pair[0].0 >= pair[1].0) { + return Err(corruption("scanned tree coverage is invalid")); + } + Ok(rows) + } + // Guards protect local cache work only, never origin I/O. fn state(&self) -> Result> { self.0 diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 0f82acf70..29aa1425f 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -37,17 +37,24 @@ Replica selection reports readiness, generations, fallbacks, and routing choices so callers can distinguish an unavailable replica from a corrupt object. -`capsule_protocol::open_git_view_from_root` captures complete refs, visibility -and pointer catalogs without materializing run OID admission. It validates +`capsule_protocol::open_git_view_from_root` captures complete refs and pointer +catalogs while keeping frontier visibility and run OID admission paged. It validates authenticated member counts and omits placement-hint dictionaries. CLI reads, HTTP browse/projection and S3 gateway reads use this view. Physical lookup uses the captured index; unindexed reads retain their existing pack-index path. Maintenance and writers requiring exact admission sets continue using the -complete metadata entry point. Visibility, catalog and source/ref enumeration -are still eager; this entry point does not establish bounded repository opening. +complete metadata entry point. Checkpoint visibility, catalog and source/ref +enumeration are still eager; total bounded opening remains unqualified. Full-capsule and control-only views share borrowed visibility readiness/application checks. Canonical transaction decoding guarantees unique ref names; matched-row counts reject extra evidence without copying the per-object edit lists. +Full and paged controls use one ref-ordering loop. Paged closure layers preserve +the inherited base when its source ref later moves, and dictionary retention +never supplies visibility for a new ref. Initial rooted additions use their +authenticated count; differing actual delta additions/removals remain admitted +work. `git_visibility_index`, `candidate_git_visibility` and +`tip_bound_transitions` asynchronously materialize captured controls only when +bulk inspection needs them. They do not recapture refs or switch origins. `ShardHydrator` provides memory, file, and half-open byte-range reconstruction. Before full-file reconstruction it checks that the pointer's shard terms are @@ -237,8 +244,8 @@ checkpoint for consumers that need embedded pack bytes. These entry points diffe requirements, not storage-format compatibility. `open_view_from_root_with_metadata` captures the complete visibility and pointer catalog controls while leaving Git pack bodies cold, including before the first -checkpoint. Browsing and background index builders use this path with the -origin-backed Git reader. Metadata capture still verifies transaction ordering, +checkpoint. Background index builders use this complete path; browsing uses the +Git view with the origin-backed Git reader. Metadata capture still verifies transaction ordering, activations and exact resulting refs; it is not footer-only fetch admission. Pack installation from metadata views also requires the origin layout and consumes authenticated source ranges. The embedded-pack reader and candidate installers diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 606fe3a04..fe6b1e2a8 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -18,6 +18,8 @@ use tokio_util::sync::CancellationToken; use crate::{ReadError, Result}; +mod visibility; + const LAYERED_SIDECAR_COALESCE_GAP_BYTES: u64 = 64 * 1024; const LAYERED_SIDECAR_MAX_EXTRA_BYTES: u64 = 4 * 1024 * 1024; const LAYERED_SIDECAR_MAX_WINDOW_BYTES: u64 = 16 * 1024 * 1024; @@ -69,7 +71,7 @@ pub struct CapsuleDependencyProof { } /// One authenticated repository root and every post-checkpoint capsule it names. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct CapsuleRepositoryView { root: crab_metadata::capsule_protocol::RootSnapshot, browse_indexes: Option, @@ -77,6 +79,8 @@ pub struct CapsuleRepositoryView { capsules: Vec, capsule_controls: Vec, tip_bound_transitions: CapsuleTipBoundTransitions, + visibility_store: Option, + tip_bound_controls: Vec, refs: BTreeMap, peeled_refs: BTreeMap, visible_ref_transactions: BTreeMap, @@ -89,6 +93,18 @@ pub struct CapsuleRepositoryView { frontier_object_admission: BTreeMap<[u8; 20], Vec>, } +impl std::fmt::Debug for CapsuleRepositoryView { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Captured transports may contain credentials; diagnostics expose identity only. + formatter + .debug_struct("CapsuleRepositoryView") + .field("root_digest", &self.root.record().digest()) + .field("ref_count", &self.refs.len()) + .field("frontier_count", &self.capsule_run_pointers.len()) + .finish_non_exhaustive() + } +} + /// One complete layered Git pack that can be copied directly to a clone. /// /// This is admitted only for a cold, unfiltered clone whose authenticated view @@ -347,9 +363,20 @@ impl CapsuleRepositoryView { /// This is used only to avoid re-walking a known ref update. The returned /// transitions remain bound to the loaded root and controls; a consumer /// must not use a partial chain as an authorization proof. - #[must_use] - pub fn tip_bound_transitions(&self) -> &CapsuleTipBoundTransitions { - &self.tip_bound_transitions + pub async fn tip_bound_transitions(&self) -> Result { + let mut transitions = self.tip_bound_transitions.clone(); + if let Some(store) = &self.visibility_store { + let controls = try_join_all( + self.tip_bound_controls + .iter() + .map(|control| control.materialize_visibility(store)), + ) + .await?; + for (name, values) in build_tip_bound_transitions(&[], &controls)? { + transitions.entry(name).or_default().extend(values); + } + } + Ok(transitions) } /// Admit the complete layered cold-clone pack set for this exact view. @@ -1206,7 +1233,7 @@ impl CapsuleRepositoryView { } /// Materialize the complete view-bound Git visibility proof. - pub fn git_visibility_index( + pub async fn git_visibility_index( &self, ) -> Result { let mut index = visibility_index_for_checkpoint(self.checkpoint.as_ref())?; @@ -1215,7 +1242,11 @@ impl CapsuleRepositoryView { apply_capsule_visibility_index(capsule, &mut index)?; } for capsule in &self.capsule_controls { - apply_capsule_control_visibility_index(capsule, &mut index)?; + let complete = match &self.visibility_store { + Some(store) => capsule.materialize_visibility(store).await?, + None => capsule.clone(), + }; + apply_capsule_control_visibility_index(&complete, &mut index)?; } let packs = self.git_pack_manifest_entries()?; @@ -1245,11 +1276,11 @@ impl CapsuleRepositoryView { } /// Materialize visibility after one candidate capsule without publishing it. - pub fn candidate_git_visibility( + pub async fn candidate_git_visibility( &self, candidate: &Capsule, ) -> Result>> { - let mut index = self.git_visibility_index()?; + let mut index = self.git_visibility_index().await?; if !capsule_is_ready(candidate, &self.refs, &index)? { return Err(corrupt_path( "candidate capsule Git visibility", @@ -1473,7 +1504,7 @@ fn build_tip_bound_transitions( if !seen_transactions.insert(capsule.transaction_id().to_owned()) { continue; } - if let Some(delta) = capsule.visibility_delta() { + if let Some(delta) = capsule.visibility_delta()? { append_tip_bound_transitions(&mut transitions, capsule.transaction(), delta)?; } } @@ -4224,6 +4255,8 @@ pub fn compacted_view_from_checkpoint( capsules: Vec::new(), capsule_controls: Vec::new(), tip_bound_transitions, + visibility_store: None, + tip_bound_controls: Vec::new(), visible_ref_transactions: BTreeMap::new(), ref_capsule_counts: BTreeMap::new(), capsule_run_pointers: Vec::new(), @@ -4534,6 +4567,8 @@ async fn assemble_view( capsules: root_capsules, capsule_controls: Vec::new(), tip_bound_transitions, + visibility_store: None, + tip_bound_controls: Vec::new(), refs, peeled_refs, visible_ref_transactions, @@ -4624,11 +4659,13 @@ async fn assemble_layered_control_view( checkpoint.visibility_transitions(), )?; } - for (ref_name, transitions) in build_tip_bound_transitions(&[], &all_capsule_controls)? { - tip_bound_transitions - .entry(ref_name) - .or_default() - .extend(transitions); + if !matches!(read, ControlRead::Git) { + for (ref_name, transitions) in build_tip_bound_transitions(&[], &all_capsule_controls)? { + tip_bound_transitions + .entry(ref_name) + .or_default() + .extend(transitions); + } } let mut capsule_run_sources = Vec::new(); let mut capsule_run_indexes = BTreeMap::new(); @@ -4668,7 +4705,7 @@ async fn assemble_layered_control_view( } for capsule in capsules { if !matches!(read, ControlRead::Git) - && let Some(visibility) = capsule.visibility_delta() + && let Some(visibility) = capsule.visibility_delta()? { extend_frontier_object_admission( &mut frontier_object_admission, @@ -4778,6 +4815,8 @@ async fn assemble_layered_control_view( capsules: Vec::new(), capsule_controls: Vec::new(), tip_bound_transitions, + visibility_store: None, + tip_bound_controls: Vec::new(), refs: expected_refs, peeled_refs: expected_peeled, visible_ref_transactions, @@ -4812,19 +4851,27 @@ async fn assemble_layered_control_view( } } } - let mut visibility_index = visibility_index_for_checkpoint(checkpoint.as_ref())?; + let initial = visibility_index_for_checkpoint(checkpoint.as_ref())?; + let mut visibility = if matches!(read, ControlRead::Git) { + visibility::Reconciliation::Paged(Box::new(visibility::Visibility::new( + router.store().clone(), + initial, + snapshot.record().root().refs().keys().cloned(), + ))) + } else { + visibility::Reconciliation::Complete(initial) + }; for capsule in &root_capsules { - if capsule.visibility_delta().is_some() { - apply_capsule_control_visibility_index(capsule, &mut visibility_index)?; - } + visibility.apply(capsule).await?; } let ordered = order_ref_controls( snapshot.record().root().refs(), snapshot.record().root().peeled_refs(), - visibility_index, + visibility, &transaction_predecessors, journal_capsules, - )?; + ) + .await?; for capsule in &ordered { apply_control_refs(capsule, &mut refs, &mut peeled_refs)?; } @@ -4842,6 +4889,12 @@ async fn assemble_layered_control_view( capsules: Vec::new(), capsule_controls: root_capsules, tip_bound_transitions, + visibility_store: matches!(read, ControlRead::Git).then(|| router.store().clone()), + tip_bound_controls: if matches!(read, ControlRead::Git) { + all_capsule_controls + } else { + Vec::new() + }, refs, peeled_refs, visible_ref_transactions, @@ -5288,10 +5341,10 @@ fn order_ref_capsules( Ok(ordered) } -fn order_ref_controls( +async fn order_ref_controls( base_refs: &BTreeMap, base_peeled: &BTreeMap, - mut visibility: crab_metadata::git_visibility::GitVisibilityIndex, + mut visibility: visibility::Reconciliation, predecessors: &BTreeMap>, mut pending: BTreeMap, ) -> Result> { @@ -5301,39 +5354,34 @@ fn order_ref_controls( let mut applied = BTreeSet::new(); let mut ordered = Vec::with_capacity(pending.len()); while !pending.is_empty() { - let ready = pending - .iter() - .find_map(|(id, capsule)| { - if predecessors - .get(capsule.transaction_id()) - .is_some_and(|required| { - required.iter().any(|transaction_id| { - tracked.contains(transaction_id) && !applied.contains(transaction_id) - }) - }) - { - return None; - } - match control_is_ready(capsule, &refs, &visibility) { - Ok(true) => Some(Ok(id.clone())), - Ok(false) => None, - Err(error) => Some(Err(error)), - } - }) - .transpose()?; - let Some(ready) = ready else { - return Err(corrupt_path( + let mut ready = None; + for (id, capsule) in &pending { + if predecessors + .get(capsule.transaction_id()) + .is_some_and(|required| { + required + .iter() + .any(|id| tracked.contains(id) && !applied.contains(id)) + }) + { + continue; + } + if visibility.ready(capsule, &refs).await? { + ready = Some(id.clone()); + break; + } + } + let id = ready.ok_or_else(|| { + corrupt_path( "capsule-protocol ref heads", "capsule ref history is cyclic or does not extend the compacted root", - )); - }; + ) + })?; let capsule = pending - .remove(&ready) + .remove(&id) .ok_or_else(|| ReadError::internal("ready capsule control disappeared"))?; apply_control_refs(&capsule, &mut refs, &mut peeled)?; - if capsule.visibility_delta().is_some() { - apply_capsule_control_visibility_index(&capsule, &mut visibility)?; - } + visibility.apply(&capsule).await?; applied.insert(capsule.transaction_id().to_owned()); ordered.push(capsule); } @@ -5347,7 +5395,7 @@ fn control_is_ready( ) -> Result { visibility_is_ready( capsule.transaction(), - capsule.visibility_delta(), + capsule.visibility_delta()?, refs, visibility_index, ) @@ -5476,7 +5524,7 @@ fn apply_capsule_control_visibility_index( capsule: &CapsuleControl, index: &mut crab_metadata::git_visibility::GitVisibilityIndex, ) -> Result<()> { - apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta(), index) + apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta()?, index) } fn apply_transaction_visibility_index( @@ -6404,12 +6452,12 @@ mod tests { assert_eq!( ( git.git_snapshot().unwrap(), - git.git_visibility_index().unwrap(), + git.git_visibility_index().await.unwrap(), git.pointer_catalog().unwrap() ), ( complete.git_snapshot().unwrap(), - complete.git_visibility_index().unwrap(), + complete.git_visibility_index().await.unwrap(), complete.pointer_catalog().unwrap() ) ); @@ -6435,8 +6483,8 @@ mod tests { complete.git_snapshot().unwrap() ); assert_eq!( - metadata.git_visibility_index().unwrap(), - complete.git_visibility_index().unwrap() + metadata.git_visibility_index().await.unwrap(), + complete.git_visibility_index().await.unwrap() ); } } @@ -6666,7 +6714,15 @@ mod tests { .is_some() ); assert_eq!(capsules.len(), 1); - assert_eq!(capsules[0].visibility_delta().unwrap().edits().len(), 1); + assert_eq!( + capsules[0] + .visibility_delta() + .unwrap() + .unwrap() + .edits() + .len(), + 1 + ); let operations = observer .observations .lock() @@ -7127,7 +7183,7 @@ mod tests { ) .unwrap(); - let actual = view.candidate_git_visibility(&candidate).unwrap(); + let actual = view.candidate_git_visibility(&candidate).await.unwrap(); assert_eq!( actual, diff --git a/crates/crab-read/src/capsule_protocol/visibility.rs b/crates/crab-read/src/capsule_protocol/visibility.rs new file mode 100644 index 000000000..cd8089dfc --- /dev/null +++ b/crates/crab-read/src/capsule_protocol/visibility.rs @@ -0,0 +1,669 @@ +//! Captured visibility closure layers; physical presence never grants membership. +use super::*; +use crab_metadata::capsule_protocol::{ + CapsuleVisibilityReader, VisibilityChange, VisibilityEditHeader, +}; +use crab_metadata::git_visibility::{GitVisibilityIndex, MAX_GIT_VISIBILITY_OBJECTS}; + +pub(super) enum Reconciliation { + Complete(GitVisibilityIndex), + Paged(Box), +} + +impl Reconciliation { + pub(super) async fn ready( + &self, + control: &CapsuleControl, + refs: &BTreeMap, + ) -> Result { + match self { + Self::Complete(index) => control_is_ready(control, refs, index), + Self::Paged(index) => index.ready(control, refs).await, + } + } + pub(super) async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { + match self { + Self::Complete(index) if control.has_visibility() => { + apply_capsule_control_visibility_index(control, index) + } + Self::Complete(_) => Ok(()), + Self::Paged(index) => index.apply(control).await, + } + } +} + +struct Layer { + parent: Option, + initial_ref: Option, + edit: Option<(Arc, String, VisibilityEditHeader)>, +} + +pub(super) struct Visibility { + store: Store, + initial: GitVisibilityIndex, + layers: Vec, + refs: BTreeMap, + dictionary: BTreeSet<[u8; 20]>, + dictionary_roots: Vec<(Arc, String, VisibilityEditHeader)>, + dictionary_count: u64, +} + +fn oid(value: &str) -> Result<[u8; 20]> { + ObjectId::from_hex(value.as_bytes()) + .map_err(|_| corrupt_path("capsule Git visibility", "invalid object identity"))? + .as_bytes() + .try_into() + .map_err(|_| corrupt_path("capsule Git visibility", "object format is unsupported")) +} + +impl Visibility { + pub(super) fn new( + store: Store, + initial: GitVisibilityIndex, + names: impl Iterator, + ) -> Self { + let mut layers = Vec::new(); + let mut refs = BTreeMap::new(); + for name in names.filter(|name| initial.contains_ref(name)) { + refs.insert(name.clone(), layers.len()); + layers.push(Layer { + parent: None, + initial_ref: Some(name), + edit: None, + }); + } + let dictionary_count = initial.objects().len() as u64; + Self { + store, + initial, + layers, + refs, + dictionary: BTreeSet::new(), + dictionary_roots: Vec::new(), + dictionary_count, + } + } + + async fn contains(&self, layer: Option, oids: &[[u8; 20]]) -> Result> { + let mut result = vec![false; oids.len()]; + let mut pending = (0..oids.len()).collect::>(); + let mut next = layer; + while let Some(index) = next { + let layer = self.layers.get(index).ok_or_else(|| { + corrupt_path("capsule Git visibility", "closure parent is invalid") + })?; + if let Some(name) = &layer.initial_ref { + for position in pending { + result[position] = self.initial.contains_in_ref(name, &oids[position]); + } + break; + } + if let Some((reader, name, header)) = &layer.edit { + let keys = pending + .iter() + .map(|position| oids[*position]) + .collect::>(); + let changes = reader + .lookup(&self.store, name, &keys) + .await? + .ok_or_else(|| { + corrupt_path("capsule Git visibility", "required edit is absent") + })?; + let mut unresolved = Vec::new(); + for (position, change) in pending.into_iter().zip(changes) { + match change { + Some(VisibilityChange::Added) => result[position] = true, + Some(VisibilityChange::Removed) => {} + None if !header.replaces() => unresolved.push(position), + None => {} + } + } + pending = unresolved; + } + if pending.is_empty() { + break; + } + next = layer.parent; + } + Ok(result) + } + + async fn dictionary_contains(&self, oids: &[[u8; 20]]) -> Result> { + let mut result = oids + .iter() + .map(|oid| { + self.initial.objects().binary_search(oid).is_ok() || self.dictionary.contains(oid) + }) + .collect::>(); + for (reader, name, _) in &self.dictionary_roots { + let positions = result + .iter() + .enumerate() + .filter_map(|(index, present)| (!present).then_some(index)) + .collect::>(); + if positions.is_empty() { + break; + } + let keys = positions + .iter() + .map(|index| oids[*index]) + .collect::>(); + let changes = reader + .lookup(&self.store, name, &keys) + .await? + .ok_or_else(|| { + corrupt_path( + "capsule Git visibility", + "dictionary addition source is absent", + ) + })?; + for (index, change) in positions.into_iter().zip(changes) { + result[index] = change == Some(VisibilityChange::Added); + } + } + Ok(result) + } + + async fn first_ref(&self, object: [u8; 20]) -> Result> { + for index in self.refs.values() { + if self.contains(Some(*index), &[object]).await?[0] { + return Ok(Some(*index)); + } + } + Ok(None) + } + + async fn headers( + &self, + control: &CapsuleControl, + ) -> Result< + Option<( + Arc, + Vec>, + )>, + > { + let Some(reader) = control.paged_visibility() else { + if control.has_visibility() { + return Err(corrupt_path( + "capsule Git visibility", + "Git control is not paged", + )); + } + return Ok(None); + }; + let edits = control.transaction().edits(); + let live = edits.iter().filter(|edit| edit.new_oid().is_some()).count(); + if reader.ref_count() as usize != live { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence has missing or uncommitted refs", + )); + } + let names = edits.iter().map(|edit| edit.ref_name()).collect::>(); + let headers = reader.headers(&self.store, &names).await?; + for (edit, header) in edits.iter().zip(&headers) { + match (edit.new_oid(), header) { + (None, None) => {} + (Some(new), Some(header)) + if header.new_oid() == new + && edit + .expected_old() + .is_none_or(|old| header.old_oid() == Some(old)) => {} + _ => { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence differs from its ref edit", + )); + } + } + } + Ok(Some((reader.clone(), headers))) + } + + pub(super) async fn ready( + &self, + control: &CapsuleControl, + refs: &BTreeMap, + ) -> Result { + if control + .transaction() + .edits() + .iter() + .any(|edit| refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old()) + { + return Ok(false); + } + let Some((_, headers)) = self.headers(control).await? else { + return Ok(true); + }; + for (edit, header) in control.transaction().edits().iter().zip(headers) { + let Some(header) = header else { + continue; + }; + if edit.expected_old().is_some() && !self.refs.contains_key(edit.ref_name()) { + return Ok(false); + } + if edit.expected_old().is_none() + && !header.replaces() + && let Some(old) = header.old_oid() + && self.first_ref(oid(old)?).await?.is_none() + { + return Ok(false); + } + } + Ok(true) + } + + pub(super) async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { + let Some((reader, headers)) = self.headers(control).await? else { + return Ok(()); + }; + for (edit, header) in control.transaction().edits().iter().zip(headers) { + let Some(header) = header else { + self.refs.remove(edit.ref_name()); + continue; + }; + let mut parent = self.refs.get(edit.ref_name()).copied(); + if parent.is_none() + && !header.replaces() + && let Some(old) = header.old_oid() + { + parent = self.first_ref(oid(old)?).await?; + } + if let Some(old) = header.old_oid() + && !self.contains(parent, &[oid(old)?]).await?[0] + { + return Err(corrupt_path( + "capsule Git visibility", + "visibility delta prior closure does not contain its old ref tip", + )); + } + let new = oid(header.new_oid())?; + let change = reader + .lookup(&self.store, edit.ref_name(), &[new]) + .await? + .ok_or_else(|| corrupt_path("capsule Git visibility", "required edit is absent"))?; + let visible = match change[0] { + Some(VisibilityChange::Added) => true, + Some(VisibilityChange::Removed) => false, + None if !header.replaces() => self.contains(parent, &[new]).await?[0], + None => false, + }; + if !visible { + return Err(corrupt_path( + "capsule Git visibility", + "visibility edit result does not contain its new ref tip", + )); + } + let removed = reader + .objects(&self.store, edit.ref_name(), VisibilityChange::Removed) + .await?; + if !removed.is_empty() { + reader + .lookup(&self.store, edit.ref_name(), &removed) + .await? + .ok_or_else(|| { + corrupt_path("capsule Git visibility", "required edit is absent") + })?; + } + if self.contains(parent, &removed).await?.contains(&false) { + return Err(corrupt_path( + "capsule Git visibility", + "visibility edit removes an object outside the prior closure", + )); + } + // A first rooted set supplies its exact dictionary count. Later equal + // roots add no objects; actual differing deltas admit their additions. + let shared = self + .dictionary_roots + .iter() + .any(|(_, _, prior)| header.same_additions(prior)); + if header.added_count() > 0 && self.dictionary_count == 0 { + self.dictionary_count = u64::from(header.added_count()); + self.dictionary_roots.push(( + reader.clone(), + edit.ref_name().to_owned(), + header.clone(), + )); + } else if !shared { + let added = reader + .objects(&self.store, edit.ref_name(), VisibilityChange::Added) + .await?; + let present = self.dictionary_contains(&added).await?; + for (object, present) in added.into_iter().zip(present) { + if !present { + self.dictionary.insert(object); + self.dictionary_count += 1; + } + } + } + if self.dictionary_count > MAX_GIT_VISIBILITY_OBJECTS { + return Err(corrupt_path( + "capsule Git visibility", + "visibility object dictionary is too large", + )); + } + let index = self.layers.len(); + self.layers.push(Layer { + parent, + initial_ref: None, + edit: Some((reader.clone(), edit.ref_name().to_owned(), header)), + }); + self.refs.insert(edit.ref_name().to_owned(), index); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crab_metadata::capsule_protocol::{ + CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, + CapsuleVisibilityDelta, + }; + use crab_metadata::git_visibility::GitVisibilityEdit; + use object_store::memory::InMemory; + + fn hex(value: u8) -> String { + format!("{value:040x}") + } + fn object(value: u8) -> [u8; 20] { + let mut oid = [0; 20]; + oid[19] = value; + oid + } + async fn control( + layout: &StoreLayout, + name: &str, + expected: Option, + edit: GitVisibilityEdit, + ) -> CapsuleControl { + let transaction = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + name, + expected.map(hex), + Some(edit.new_oid.clone()), + None, + )], + ) + .unwrap(); + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([(name.to_owned(), edit)])).unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let (_, mut controls) = + crab_metadata::capsule_protocol::load_capsule_run_git_control(layout, &pointer) + .await + .unwrap(); + controls.remove(0) + } + fn initial() -> GitVisibilityIndex { + GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new()).unwrap() + } + + #[tokio::test] + async fn initial_replacement_opens_without_materializing_its_oid_set() { + use crab_metadata::capsule_protocol::{RepositoryRoot, RootRecord}; + use std::sync::atomic::{AtomicU64, Ordering}; + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = + Store::new(Arc::new(InMemory::new())).with_read_byte_observer(Arc::new(move |size| { + observed.fetch_add(size, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store, "repositories/large-root".into()); + let initial = RootRecord::encode( + RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(), + ) + .unwrap(); + crab_metadata::capsule_protocol::create_root(&layout, initial.clone()) + .await + .unwrap(); + let objects = (0u32..10000) + .map(|index| format!("{index:040x}")) + .collect::>(); + let tip = objects[9000].clone(); + let transaction = CapsuleTransaction::new( + initial.digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(tip.clone()), + None, + )], + ) + .unwrap(); + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/main".into(), + GitVisibilityEdit::from_replacement_objects(None, tip.clone(), objects), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let next = RootRecord::encode( + initial + .root() + .advance( + initial.digest(), + BTreeMap::from([("refs/heads/main".into(), tip)]), + BTreeMap::new(), + vec![pointer], + transaction.id().unwrap().as_str(), + ) + .unwrap(), + ) + .unwrap(); + layout + .store() + .put_overwrite(&layout.capsule_root_path(), next.bytes().clone()) + .await + .unwrap(); + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + bytes.store(0, Ordering::Relaxed); + let view = open_git_view_from_root( + &layout, + root, + CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 1024 * 1024, + }, + ) + .await + .unwrap(); + assert!( + bytes.load(Ordering::Relaxed) < 64 * 1024 + && view.capsule_controls[0].visibility_delta().is_err() + ); + } + + #[tokio::test] + async fn layered_closures_match_complete_edits_and_preserve_captured_bases() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/layers".into(), + ); + let mut full = initial(); + let mut lazy = Visibility::new(layout.store().clone(), full.clone(), std::iter::empty()); + let edits = [ + ( + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects( + None, + hex(1), + vec![hex(1), hex(2), hex(3)], + ), + ), + ( + "refs/heads/feature", + None, + GitVisibilityEdit::from_delta_objects( + Some(hex(1)), + hex(2), + vec![hex(4)], + vec![hex(3)], + ), + ), + ( + "refs/heads/main", + Some(1), + GitVisibilityEdit::from_delta_objects( + Some(hex(1)), + hex(5), + vec![hex(5)], + vec![hex(3)], + ), + ), + ]; + for (name, expected, edit) in edits { + let control = control(&layout, name, expected, edit.clone()).await; + full.apply_ref_edit(name.to_owned(), &edit).unwrap(); + lazy.apply(&control).await.unwrap(); + } + let requested = (1..7).map(object).collect::>(); + let mut actual = BTreeMap::new(); + for (name, index) in &lazy.refs { + actual.insert( + name.clone(), + lazy.contains(Some(*index), &requested).await.unwrap(), + ); + } + let expected = lazy + .refs + .keys() + .map(|name| { + ( + name.clone(), + requested + .iter() + .map(|oid| full.contains_in_ref(name, oid)) + .collect::>(), + ) + }) + .collect::>(); + assert_eq!( + (actual, lazy.dictionary_count), + (expected, full.objects().len() as u64) + ); + } + + #[tokio::test] + async fn removed_invisible_objects_do_not_supply_new_ref_bases() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/invisible".into(), + ); + let mut lazy = Visibility::new(layout.store().clone(), initial(), std::iter::empty()); + let create = control( + &layout, + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1), hex(3)]), + ) + .await; + lazy.apply(&create).await.unwrap(); + let remove = control( + &layout, + "refs/heads/main", + Some(1), + GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(2)], vec![hex(3)]), + ) + .await; + lazy.apply(&remove).await.unwrap(); + let branch = control( + &layout, + "refs/heads/feature", + None, + GitVisibilityEdit::from_delta_objects(Some(hex(3)), hex(4), vec![hex(4)], vec![]), + ) + .await; + assert_eq!( + ( + lazy.dictionary_contains(&[object(3)]).await.unwrap(), + lazy.ready( + &branch, + &BTreeMap::from([("refs/heads/main".into(), hex(2))]) + ) + .await + .unwrap() + ), + (vec![true], false) + ); + } + + #[tokio::test] + async fn removal_outside_captured_prior_closure_is_rejected() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/bad-removal".into(), + ); + let mut lazy = Visibility::new(layout.store().clone(), initial(), std::iter::empty()); + let create = control( + &layout, + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1)]), + ) + .await; + lazy.apply(&create).await.unwrap(); + let bad = control( + &layout, + "refs/heads/main", + Some(1), + GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(2)], vec![hex(9)]), + ) + .await; + assert!(lazy.apply(&bad).await.is_err()); + } +} diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index 8591808fb..fcdac69ee 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -531,7 +531,7 @@ async fn publish_layered_sources( CheckpointError, > { let catalog_digest = source_catalog_digest(&sources)?; - let visibility_index = view.git_visibility_index()?; + let visibility_index = view.git_visibility_index().await?; let member_admission = layered_member_admission( &visibility_index, &sources, diff --git a/crates/crab-remote/tests/checkpoint.rs b/crates/crab-remote/tests/checkpoint.rs index 19f0a3768..bec518f3f 100644 --- a/crates/crab-remote/tests/checkpoint.rs +++ b/crates/crab-remote/tests/checkpoint.rs @@ -76,8 +76,8 @@ async fn maintenance_reuses_committed_checkpoint_without_reloading_authority() { let after = view(&layout).await; assert_eq!(after.refs(), before.refs()); assert_eq!( - after.git_visibility_index().unwrap().ref_closures(), - before.git_visibility_index().unwrap().ref_closures() + after.git_visibility_index().await.unwrap().ref_closures(), + before.git_visibility_index().await.unwrap().ref_closures() ); assert_eq!(after.git_pack_count(), 1); assert_eq!( @@ -898,8 +898,8 @@ async fn logical_checkpoint_does_not_read_or_replace_pack_sources() { ); assert_eq!(after.refs(), before.refs()); assert_eq!( - after.git_visibility_index().unwrap().ref_closures(), - before.git_visibility_index().unwrap().ref_closures() + after.git_visibility_index().await.unwrap().ref_closures(), + before.git_visibility_index().await.unwrap().ref_closures() ); } @@ -920,7 +920,12 @@ async fn first_checkpoint_opens_its_frontier_from_authenticated_controls() { let after = view(&layout).await; assert_eq!(after.layered_checkpoint().unwrap().sources().len(), 2); assert_eq!( - after.git_visibility_index().unwrap().ref_closures().len(), + after + .git_visibility_index() + .await + .unwrap() + .ref_closures() + .len(), 2 ); } @@ -1131,8 +1136,8 @@ async fn first_checkpoint_rolls_up_only_the_suffix_required_by_the_source_limit( ); assert_eq!(after.refs(), before.refs()); assert_eq!( - after.git_visibility_index().unwrap().ref_closures(), - before.git_visibility_index().unwrap().ref_closures() + after.git_visibility_index().await.unwrap().ref_closures(), + before.git_visibility_index().await.unwrap().ref_closures() ); } @@ -1257,8 +1262,8 @@ async fn duplicate_suffix_pack_does_not_change_stable_member_positions() { let after = view(&layout).await; assert_eq!(after.layered_checkpoint().unwrap().sources()[0], prefix); assert_eq!( - after.git_visibility_index().unwrap().ref_closures(), - before.git_visibility_index().unwrap().ref_closures() + after.git_visibility_index().await.unwrap().ref_closures(), + before.git_visibility_index().await.unwrap().ref_closures() ); verify_git_blobs( &layout, @@ -1296,8 +1301,8 @@ async fn duplicate_only_suffix_can_be_consolidated() { assert_eq!(after.layered_checkpoint().unwrap().sources().len(), 1); assert_eq!(after.refs(), before.refs()); assert_eq!( - after.git_visibility_index().unwrap().ref_closures(), - before.git_visibility_index().unwrap().ref_closures() + after.git_visibility_index().await.unwrap().ref_closures(), + before.git_visibility_index().await.unwrap().ref_closures() ); verify_git_blobs( &layout, @@ -1356,7 +1361,7 @@ async fn layered_inventory_includes_new_frontier_in_complete_and_control_views() let old_packs = checkpointed.git_pack_count(); let old_bytes = checkpointed.git_pack_bytes().unwrap(); let old_objects = checkpointed.git_object_count().unwrap(); - let old_visibility = checkpointed.git_visibility_index().unwrap(); + let old_visibility = checkpointed.git_visibility_index().await.unwrap(); let (_, new_pack) = blob_pack(b"late"); publish_blob(&layout, "late").await; let root = crab_write::capsule_protocol::open_root(&layout) @@ -1375,7 +1380,11 @@ async fn layered_inventory_includes_new_frontier_in_complete_and_control_views() ); assert_eq!(current.git_object_count().unwrap(), old_objects + 1); assert_ne!( - current.git_visibility_index().unwrap().pack_index_hash, + current + .git_visibility_index() + .await + .unwrap() + .pack_index_hash, old_visibility.pack_index_hash ); } diff --git a/crates/crab-remote/tests/checkpoint/frontier_admission.rs b/crates/crab-remote/tests/checkpoint/frontier_admission.rs index ac2a6ced7..5c3fd9f28 100644 --- a/crates/crab-remote/tests/checkpoint/frontier_admission.rs +++ b/crates/crab-remote/tests/checkpoint/frontier_admission.rs @@ -101,6 +101,7 @@ async fn frontier_delta_base_placement_does_not_probe_unrelated_member_indexes() assert!( complete .git_visibility_index() + .await .unwrap() .ref_closures() .values() From da3495d8f84fcc66cdd8cedb90dd2fe1371ec8c1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 09:45:52 -0700 Subject: [PATCH 025/127] perf(read): follow path-state change points Retain deletion and type-reset markers in unreleased path-state v6. Verify candidate changes against raw Git while skipping unrelated commits. --- .../docs/architecture/indexed-remote-reads.md | 64 +++++ .../e2e/run_remote_read_scaling_rustfs.py | 17 +- crates/crab-http-server/src/projection.rs | 6 +- crates/crab-metadata/README.md | 13 +- crates/crab-metadata/src/path_state.rs | 39 ++- crates/crab-metadata/src/path_state/codec.rs | 11 +- crates/crab-metadata/src/path_state/reader.rs | 125 +++++++-- .../src/path_state/reader_tests.rs | 88 +++++++ crates/crab-metadata/src/path_state/scan.rs | 15 +- crates/crab-remote-git/README.md | 9 + crates/crab-remote-git/src/path_state.rs | 32 +++ crates/crab-remote-git/src/snapshot.rs | 144 ++++++++--- .../tests/remote_repository.rs | 243 ++++++++++++++++++ 13 files changed, 719 insertions(+), 87 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 108591800..dcc227164 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -982,3 +982,67 @@ remains unresolved. No test, baseline or expected-failure assertion is weakened. Checkpoint visibility/catalog decoding, ref/source capture and broader frontier amplification remain open. The 10k path-history budget failure, stages 5–8 and unified staging remain unresolved; this slice does not complete the initiative. + + +## Path-history and blame change points in progress + +The 10,000-commit path-history failure comes from walking every first-parent +commit despite available path-state attribution. The entry maximum remains +100,000. First-parent path history now looks up candidate changes, verifies raw +candidate/parent commits and tree entries, and resumes pagination at the parent. +Blame uses the same lookup to skip unchanged versions while retaining existing +line comparisons, mode-only behavior and deletion boundaries. Historical paths +below a blob ancestor are absent; direct entry lookup retains its component-kind +error. Root-path and all-parent history retain their traversal policy. + +Path-state descriptor version 6 retains deletion/type-reset ordinals and old +trie descendants. Reset markers mask older visible values without rewriting old +roots or expanding every deleted subtree. Missing descendants can have covering +reset candidates, so raw entry comparison remains required before reporting a +change. The paged reader shares visible attribution and candidate lookup through +one interpreter. A missing/corrupt advertised page remains an error. Complete +loads, compaction, resumable prefixes and maintenance scans retain the markers. +HTTP projection keeps visible-node attribution: its sole caller supplies paths +from the captured raw directory listing (`crates/crab-http-server/src/api.rs`). + +Upstream release tags were rechecked at their peeled commits: v1.0.1 and v1.2.0 +through v1.2.4 contain no path-state source. Unreleased version 5 is rejected; +existing generation/browse maintenance handles its corrupt/retired descriptor by +rebuilding, without a second reader codec. Broader lifecycle proof remains open. +The read-scaling harness now captures source and manifests for metadata, read, +remote, remote-Git, storage and write owners, alongside binary and fixture proofs. + +Proof passes 23 storage/remote-index path-state and seven payload-only tests, +native-Git history/continuation comparisons, cold indexed/unindexed blame +comparisons with separate runtimes, every native blame line for mode/content/ +unchanged-file cases, ordinary semantic and four blame regressions, and strict +metadata/remote-Git Clippy. Four harness tests, frontend build and CLI/HTTP/S3/ +protected-service consumer checks pass. The separate runtimes prevent a shared +result cache from substituting for indexed execution. + +Fresh-prefix RustFS publication, maintenance and all ten queries pass at 100, +1,000 and 10,000 commits: 87 checks / 64 commands. The original failed 10k report +remains retained. Final-source replay passes 123 processes: all ten queries per +history and twenty cold/warm path-history plus twenty blame samples per history. +Receipts match the current Rust source hashes, native Git results and completed +backend operations. Warm query origin reads remain zero. Cold read bytes are: + +| Commits | Path history | Blame | +| --- | ---: | ---: | +| 100 | 133,757 | 133,772 | +| 1,000 | 728,954 | 728,968 | +| 10,000 | 4,691,860 | 4,708,972 | + +The raw commit walk is eliminated for indexed exact paths. Cold page/range +amplification remains visible in these measurements and needs further tuning; +these are local debug/RustFS observations, not representative-provider targets. +CLI link build passes before publication. This is not complete stage-5 +proof: comparison caching and the complete parity matrix remain outstanding, +alongside stage-4 opening/lifecycle/provider gaps and later stages. First-parent +blame remains a bounded supported subset; Git's automatic whole-file rename +following is a separate requirement ([Git blame documentation](https://git-scm.com/docs/git-blame)). + +PR #549 CI now also reports a container Compose recovery/cluster qualification +failure. The failed log is retained; cause remains unresolved and no assertion +or baseline has been weakened. This work has not landed and the goal remains +active. diff --git a/crab/scripts/e2e/run_remote_read_scaling_rustfs.py b/crab/scripts/e2e/run_remote_read_scaling_rustfs.py index 5a18f23df..9889a733f 100644 --- a/crab/scripts/e2e/run_remote_read_scaling_rustfs.py +++ b/crab/scripts/e2e/run_remote_read_scaling_rustfs.py @@ -76,16 +76,19 @@ def qualify(self) -> None: self.report.artifacts["qualifier_sha256"] = sha256_file(self.args.qualifier) source_root = self.artifacts / "qualified-source" source_root.mkdir() - inputs = ( - "Cargo.lock", "crates/crab-remote/Cargo.toml", - "crates/crab-remote/examples/qualify_remote.rs", - "crates/crab-remote/examples/support/mod.rs", - "crates/crab-metadata/src/path_state/codec.rs", - "crates/crab-metadata/src/split_commit_graph.rs", + source_prefixes = ( + "crates/crab-metadata/", "crates/crab-read/", "crates/crab-remote/", + "crates/crab-remote-git/", "crates/crab-storage/", "crates/crab-write/", + ) + tracked = subprocess.check_output(["git", "ls-files", "--", *source_prefixes], + cwd=REPO_ROOT, text=True).splitlines() + inputs = [name for name in tracked if name.endswith((".rs", "/Cargo.toml"))] + inputs.extend(( + "Cargo.lock", "Cargo.toml", "crab/scripts/e2e/run_remote_read_scaling_rustfs.py", "crab/scripts/e2e/run_concurrent_push_smoke.py", "crab/scripts/e2e/verify_remote_browse.py", - ) + )) hashes = {} for relative in inputs: target = source_root / relative diff --git a/crates/crab-http-server/src/projection.rs b/crates/crab-http-server/src/projection.rs index e033400c2..ee034536f 100644 --- a/crates/crab-http-server/src/projection.rs +++ b/crates/crab-http-server/src/projection.rs @@ -501,7 +501,11 @@ async fn project_attribution( }; check_cancelled(cancellation)?; let (nodes, edges, roots) = match item { - PathStateItem::Node { reference, value } => ( + // Projection attributes actual visible entries. Retained deletion + // history is used by canonical history reads, not visible listing rows. + PathStateItem::Node { + reference, value, .. + } => ( Some(projection::AttributionNodeRow { node_hash: node_hash(source, reference.layer, reference.index), last_change_ordinal: value, diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 0cba3ca65..66cc207d6 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -103,7 +103,7 @@ cargo build -p crab-metadata --example qualify_graph_build --release --locked /usr/bin/time -l "$CARGO_TARGET_DIR/release/examples/qualify_graph_build" branches 8000 ``` -Path-state descriptor version 5 requires exact-name tree comparison semantics +Path-state descriptor version 6 requires exact-name tree comparison semantics and authenticated pages; retired descriptors are rejected. Maintenance rebuilds completed indexes and resumable checkpoints through the existing conditional publication path. Record, node, component and child pages @@ -414,9 +414,18 @@ responsibility. ### Paged path attribution -Unreleased path-state descriptor version 5 authenticates bounded record, node, +Unreleased path-state descriptor version 6 authenticates bounded record, node, component and child pages inside each immutable layer object. Canonical full loads validate the complete physical closure; `PathStateReader` resolves selected paths through verified ranges and a bounded parsed-page cache. Graph and ordinal bindings are retained. GC retains the layer object and therefore every embedded page. See the [format and qualification contract](../../crab/docs/architecture/indexed-remote-reads.md#paged-path-state-format). + +Path-state nodes retain deletion/type-reset ordinals and historical descendants. +A reset masks older visible values; it does not mutate prior commit roots. +`last_change` locates candidate changes for absent paths as well as visible ones. +A covering reset may nominate a never-existing descendant; raw tree comparison +must decide whether that candidate actually changed the requested path. Full +loads, resumable construction, compaction and maintenance scans retain markers. +The retired unreleased version 5 is rebuilt by existing maintenance, never read +through an alternate codec. diff --git a/crates/crab-metadata/src/path_state.rs b/crates/crab-metadata/src/path_state.rs index 250a8a49d..a1445c757 100644 --- a/crates/crab-metadata/src/path_state.rs +++ b/crates/crab-metadata/src/path_state.rs @@ -41,7 +41,7 @@ pub use storage::{ // Whole-layer predecessors are unreleased derived data. Rebuild them before // publishing paged readiness, retaining the exact tree-comparison semantics. -const DESCRIPTOR_VERSION: u32 = 5; +const DESCRIPTOR_VERSION: u32 = 6; const MAX_DESCRIPTOR_BYTES: u64 = 128 * 1024; const MAX_AUTHOR_BYTES: usize = 16 * 1024; const MAX_MESSAGE_BYTES: usize = 1024 * 1024; @@ -60,7 +60,7 @@ pub struct PathStateMutation { pub path: Vec, /// The path exists after this commit and receives this commit's ordinal. pub present: bool, - /// Existing descendants are discarded before applying this mutation. + /// Existing descendants become absent unless recreated by this commit. pub reset: bool, } @@ -106,6 +106,8 @@ pub struct PathStateRecord { #[derive(Debug, Clone, PartialEq, Eq)] pub struct PathStateNode { pub value: Option, + /// Last deletion or type reset covering this path and its descendants. + pub reset: Option, pub children: BTreeMap, PathStateNodeRef>, } @@ -301,14 +303,19 @@ impl PathStateIndex { fn lookup(&self, root: PathStateNodeRef, path: &[u8]) -> Result> { let mut current = root; + let mut reset = None; for component in path.split(|byte| *byte == b'/') { let node = self.node(current)?; + reset = reset.max(node.reset); let Some(next) = node.children.get(component) else { return Ok(None); }; current = *next; } - Ok(self.node(current)?.value) + Ok(self + .node(current)? + .value + .filter(|value| reset.is_none_or(|reset| *value >= reset))) } fn node(&self, reference: PathStateNodeRef) -> Result<&PathStateNode> { @@ -540,6 +547,7 @@ fn copy_compacted_node( }; nodes.push(PathStateNode { value: node.value, + reset: node.reset, children, }); remapped.insert(source, reference); @@ -578,6 +586,7 @@ fn node_from_parts<'a>( #[derive(Debug, Default)] struct WorkingNode { value: Option, + reset: Option, children: BTreeMap, WorkingChild>, } @@ -591,6 +600,7 @@ impl WorkingNode { fn from_node(node: &PathStateNode) -> Self { Self { value: node.value, + reset: node.reset, children: node .children .iter() @@ -610,7 +620,7 @@ fn apply_mutation( ) -> Result<()> { if mutation.path.is_empty() { if mutation.reset { - root.children.clear(); + root.reset = Some(ordinal); } root.value = mutation.present.then_some(ordinal); return Ok(()); @@ -643,16 +653,14 @@ fn apply_components( .first() .ok_or_else(|| corruption("path-state mutation has no component"))?; if components.len() == 1 { - if !mutation.present { - node.children.remove(*name); - return Ok(()); + let mut child = + take_working_child(node.children.remove(*name), base, nodes, layer_ordinal)?; + // Retain deletion/reset history without walking every old descendant. + // The reset ordinal masks old visible values, while old roots stay immutable. + if mutation.reset || !mutation.present { + child.reset = Some(ordinal); } - let mut child = if mutation.reset { - WorkingNode::default() - } else { - take_working_child(node.children.remove(*name), base, nodes, layer_ordinal)? - }; - child.value = Some(ordinal); + child.value = mutation.present.then_some(ordinal); node.children .insert(Arc::from(*name), WorkingChild::Owned(Box::new(child))); return Ok(()); @@ -711,6 +719,7 @@ fn freeze_node( .map_err(|_| corruption("path-state layer has too many nodes"))?; nodes.push(PathStateNode { value: node.value, + reset: node.reset, children, }); Ok(PathStateNodeRef { @@ -777,7 +786,9 @@ fn validate_descriptor(descriptor: &PathStateDescriptor, layers: &[PathStateLaye for (node_index, node) in layer.nodes.iter().enumerate() { if node .value - .is_some_and(|value| value >= descriptor.commit_count) + .into_iter() + .chain(node.reset) + .any(|value| value >= descriptor.commit_count) || node.children.len() > MAX_CHILDREN_PER_NODE { return corrupt("path-state node exceeds its limits"); diff --git a/crates/crab-metadata/src/path_state/codec.rs b/crates/crab-metadata/src/path_state/codec.rs index 9ef1545d0..bee1b054c 100644 --- a/crates/crab-metadata/src/path_state/codec.rs +++ b/crates/crab-metadata/src/path_state/codec.rs @@ -58,6 +58,7 @@ pub(super) fn encode_layer(layer: &PathStateLayer) -> Result { }) .collect::>>()?; let mut value = node.value.unwrap_or(u32::MAX).to_le_bytes().to_vec(); + value.extend_from_slice(&node.reset.unwrap_or(u32::MAX).to_le_bytes()); if rows.is_empty() { value.push(0); } else { @@ -166,7 +167,7 @@ pub(super) fn decode_layer( path, &mut visited, )? { - let (value, children) = decode_node(&value, reference.bytes, path)?; + let (value, reset, children) = decode_node(&value, reference.bytes, path)?; let mut entries = BTreeMap::new(); if let Some(root) = children { for (key, value) in paged::scan(bytes, &root, Kind::Children, path, &mut visited)? { @@ -185,6 +186,7 @@ pub(super) fn decode_layer( } nodes.push(PathStateNode { value, + reset, children: entries, }); } @@ -248,9 +250,10 @@ pub(super) fn decode_node( bytes: &[u8], source_bytes: u64, path: &str, -) -> Result<(Option, Option)> { +) -> Result<(Option, Option, Option)> { let mut input = Input::new(bytes, path); let value = optional(input.u32()?); + let reset = optional(input.u32()?); let children = match input.array::<1>()?[0] { 0 => None, 1 => { @@ -264,7 +267,7 @@ pub(super) fn decode_node( _ => return corrupt_at(path, "invalid child-root tag"), }; input.finish()?; - Ok((value, children)) + Ok((value, reset, children)) } #[cfg(any(feature = "storage", test))] @@ -303,10 +306,12 @@ mod tests { let name: Arc<[u8]> = Arc::from(b"repeated-component-\xff".as_slice()); let mut nodes = vec![PathStateNode { value: Some(0), + reset: None, children: BTreeMap::new(), }]; nodes.extend((0..1024).map(|_| PathStateNode { value: None, + reset: None, children: BTreeMap::from([(name.clone(), PathStateNodeRef { layer: 0, index: 0 })]), })); PathStateLayer { diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index c84f78953..169a7a6b5 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -72,6 +72,37 @@ impl PathStateReader { start: &[u8; 20], paths: &[&[u8]], ) -> Result> { + self.summaries(store, graph, start, paths, true) + .await? + .into_iter() + .map(|summary| summary.ok_or_else(|| corruption("visible path has no attribution"))) + .collect() + } + + /// Locate last path changes, including deletion and covering type resets. + pub async fn last_change( + &self, + store: &Store, + graph: CommitGraphView<'_>, + start: &[u8; 20], + paths: &[&[u8]], + ) -> Result>> { + Ok(self + .summaries(store, graph, start, paths, false) + .await? + .into_iter() + .map(|summary| summary.map(|summary| summary.oid)) + .collect()) + } + + async fn summaries( + &self, + store: &Store, + graph: CommitGraphView<'_>, + start: &[u8; 20], + paths: &[&[u8]], + visible: bool, + ) -> Result>> { // Scalars can match two rebuilt graphs with different ordinals. The // immutable layer identities pin the ordering verified during load. if !matches_graph(&self.descriptor, graph) || self.graph_layers != graph.descriptor().layers @@ -99,6 +130,8 @@ impl PathStateReader { .await? .remove(0); let mut nodes = vec![start.root; paths.len()]; + let mut resets = vec![None; paths.len()]; + let mut found = vec![true; paths.len()]; let mut remaining = paths .iter() .map(|path| path.split(|byte| *byte == b'/')) @@ -108,12 +141,14 @@ impl PathStateReader { let states = self.nodes(store, &nodes, &pending, &mut work).await?; let mut components = BTreeMap::>::new(); for (&position, state) in pending.iter().zip(states) { + resets[position] = resets[position].max(state.1); let Some(component) = remaining[position].next() else { continue; }; - let children = state - .1 - .ok_or_else(|| corruption("visible path is absent from its trie"))?; + let Some(children) = state.2 else { + found[position] = false; + continue; + }; components .entry(nodes[position].layer as usize) .or_default() @@ -146,7 +181,10 @@ impl PathStateReader { for ((position, component, root), (key, value)) in requests.into_iter().zip(keys.into_iter().zip(values)) { - let value = value.ok_or_else(|| corruption("visible component is absent"))?; + let Some(value) = value else { + found[position] = false; + continue; + }; let (ordinal, name) = codec::decode_name(&key, &value, self.paths[layer].as_ref())?; if name != component || ordinal >= names.count { @@ -186,7 +224,10 @@ impl PathStateReader { ) .await?; for ((position, _), value) in batch.requests.into_iter().zip(values) { - let value = value.ok_or_else(|| corruption("visible child is absent"))?; + let Some(value) = value else { + found[position] = false; + continue; + }; let child = codec::decode_child(&value, self.paths[layer].as_ref())?; self.validate_node(child, Some(nodes[position]))?; nodes[position] = child; @@ -194,29 +235,59 @@ impl PathStateReader { } } } - let positions = (0..paths.len()).collect::>(); + let positions = (0..paths.len()) + .filter(|position| found[*position]) + .collect::>(); let states = self.nodes(store, &nodes, &positions, &mut work).await?; - let ordinals = states - .into_iter() - .map(|(value, _)| { - value - .filter(|ordinal| *ordinal <= start_ordinal) - .ok_or_else(|| { - corruption("path attribution is absent or points to a future commit") - }) - }) - .collect::>>()?; - Ok(self - .records(store, graph, &ordinals, &mut work) + let mut ordinals = resets.clone(); + for (position, (value, reset, _)) in positions.into_iter().zip(states) { + let reset = resets[position].max(reset); + ordinals[position] = if visible { + value.filter(|value| reset.is_none_or(|reset| *value >= reset)) + } else { + value.max(reset) + }; + } + if visible { + for (ordinal, found) in ordinals.iter_mut().zip(found) { + if !found { + *ordinal = None; + } + if ordinal.is_none() { + return corrupt("visible path is absent from its trie"); + } + } + } + if ordinals + .iter() + .flatten() + .any(|ordinal| *ordinal > start_ordinal) + { + return corrupt("path attribution points to a future commit"); + } + let requested = ordinals.iter().flatten().copied().collect::>(); + let mut records = self + .records(store, graph, &requested, &mut work) .await? + .into_iter(); + ordinals .into_iter() - .map(|record| PathStateSummary { - oid: record.oid, - author: record.author, - author_seconds: record.author_seconds, - message: record.message, + .map(|ordinal| { + ordinal + .map(|_| { + let record = records + .next() + .ok_or_else(|| corruption("path summary is missing"))?; + Ok(PathStateSummary { + oid: record.oid, + author: record.author, + author_seconds: record.author_seconds, + message: record.message, + }) + }) + .transpose() }) - .collect()) + .collect() } async fn records( @@ -291,7 +362,7 @@ impl PathStateReader { nodes: &[PathStateNodeRef], positions: &[usize], work: &mut PageWork, - ) -> Result, Option)>> { + ) -> Result, Option, Option)>> { let mut grouped = BTreeMap::>::new(); for (output, &position) in positions.iter().enumerate() { let node = nodes[position]; @@ -326,7 +397,9 @@ impl PathStateReader { let node = codec::decode_node(&value, reference.bytes, self.paths[layer].as_ref())?; if node .0 - .is_some_and(|ordinal| ordinal >= self.descriptor.commit_count) + .into_iter() + .chain(node.1) + .any(|ordinal| ordinal >= self.descriptor.commit_count) { return corrupt("node attribution is out of bounds"); } diff --git a/crates/crab-metadata/src/path_state/reader_tests.rs b/crates/crab-metadata/src/path_state/reader_tests.rs index 7625b1316..42f913186 100644 --- a/crates/crab-metadata/src/path_state/reader_tests.rs +++ b/crates/crab-metadata/src/path_state/reader_tests.rs @@ -338,3 +338,91 @@ async fn paged_graph_binding_matches_materialized_attribution_without_layer_load expected ); } + +#[tokio::test] +async fn deletion_and_reset_candidates_survive_unrelated_commits_and_old_roots() { + let graph = super::super::tests::graph_with_commits(6); + let mutation = |path: &[u8], present, reset| PathStateMutation { + path: path.to_vec(), + present, + reset, + }; + let inputs = (1..=6) + .map(|n| PathStateInput { + oid: [n; 20], + first_parent: (n > 1).then_some([n - 1; 20]), + author: Vec::new(), + author_seconds: i64::from(n), + message: Vec::new(), + mutations: match n { + 1 => vec![mutation(b"", false, true), mutation(b"foo/a", true, false)], + 2 => vec![mutation(b"foo", false, false)], + 3 => vec![ + mutation(b"foo", true, true), + mutation(b"foo/b", true, false), + ], + 4 => vec![mutation(b"unrelated", true, false)], + 5 => vec![mutation(b"foo/b", false, false)], + _ => vec![mutation(b"foo", true, true)], + }, + }) + .collect(); + let write = append_path_state(None, &graph, inputs).unwrap(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "change-points".to_owned()); + upload_path_state(&store, &layout, &write, DEFAULT_MAX_PATH_STATE_BYTES) + .await + .unwrap(); + let reader = PathStateReader::load( + &store, + &layout, + &write.descriptor_hash, + CommitGraphView::Materialized(&graph), + DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + for (commit, expected) in [ + (1, [1, 1]), + (2, [2, 2]), + (3, [3, 3]), + (4, [3, 3]), + (5, [3, 5]), + (6, [6, 6]), + ] { + let actual = reader + .last_change( + &store, + CommitGraphView::Materialized(&graph), + &[commit; 20], + &[b"foo/a", b"foo/b"], + ) + .await + .unwrap(); + assert_eq!( + actual.into_iter().map(Option::unwrap).collect::>(), + expected.map(|value| [value; 20]) + ); + } + let visible = reader + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[4; 20], + &[b"foo/b"], + ) + .await + .unwrap(); + assert_eq!(visible[0].oid, [3; 20]); + assert!( + reader + .latest( + &store, + CommitGraphView::Materialized(&graph), + &[4; 20], + &[b"foo/a"] + ) + .await + .is_err() + ); +} diff --git a/crates/crab-metadata/src/path_state/scan.rs b/crates/crab-metadata/src/path_state/scan.rs index 1fc8f6c3c..88effaf9a 100644 --- a/crates/crab-metadata/src/path_state/scan.rs +++ b/crates/crab-metadata/src/path_state/scan.rs @@ -11,6 +11,7 @@ pub enum PathStateItem { Node { reference: PathStateNodeRef, value: Option, + reset: Option, }, Edge { parent: PathStateNodeRef, @@ -200,10 +201,12 @@ impl PathStateScan<'_> { index: integer(&key)?, }; self.reader.validate_node(node, None)?; - let (value, children) = + let (value, reset, children) = codec::decode_node(&value, reference.bytes, path.as_ref())?; if value - .is_some_and(|ordinal| ordinal >= self.reader.descriptor.commit_count) + .into_iter() + .chain(reset) + .any(|ordinal| ordinal >= self.reader.descriptor.commit_count) { return corrupt("node attribution is out of bounds"); } @@ -211,6 +214,7 @@ impl PathStateScan<'_> { return Ok(Some(PathStateItem::Node { reference: node, value, + reset, })); } self.phase = Phase::Records; @@ -409,13 +413,18 @@ mod tests { .collect::>(); while let Some(item) = scan.next().await.unwrap() { match item { - PathStateItem::Node { reference, value } => { + PathStateItem::Node { + reference, + value, + reset, + } => { assert_eq!( reference.index as usize, layers[reference.layer as usize].nodes.len() ); layers[reference.layer as usize].nodes.push(PathStateNode { value, + reset, children: BTreeMap::new(), }); } diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index bf693af45..67b54e417 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -238,3 +238,12 @@ duplicates. Each verified index probes the smaller side, avoiding a complete request-set scan per small frontier member or an index scan per point read. Self-contained matches remain preferred over external deltas. This bounds CPU lookup work; it does not change source-range admission or reduce origin requests. + +First-parent path history and blame use captured path-state change points when +an index is available. History verifies each candidate against raw first-parent +commit/tree entries, including deletion, mode changes and covering type resets. +Continuations resume from the candidate's parent without rewalking unrelated +commits. Blame preserves line comparison and deletion boundaries while jumping +between relevant versions. Root-path and all-parent history retain their existing +raw traversal; index absence retains the ordinary Git read path. Missing or +corrupt advertised pages are errors, not a reason to switch traversal. diff --git a/crates/crab-remote-git/src/path_state.rs b/crates/crab-remote-git/src/path_state.rs index 865d27b74..3bae28ba9 100644 --- a/crates/crab-remote-git/src/path_state.rs +++ b/crates/crab-remote-git/src/path_state.rs @@ -19,6 +19,38 @@ pub(crate) struct PathStateIndex { } impl PathStateIndex { + pub(crate) async fn last_change( + &self, + commit: ObjectId, + path: &GitPath, + operation: &OperationContext, + ) -> Result> { + operation.ensure_active()?; + let ObjectId::Sha1(commit) = commit else { + return Err(Error::UnsupportedObjectFormat); + }; + operation.charge(BudgetDimension::LogicalObjects, 1).await?; + operation.charge(BudgetDimension::Entries, 1).await?; + operation + .charge(BudgetDimension::Depth, path.components().count() as u64) + .await?; + let store = operation.metadata_store(); + let summary = operation + .read_metadata(self.index.last_change( + &store, + self.commit_graph.view(), + &commit, + &[path.as_bytes()], + )) + .await + .map_err(|error| match error { + Error::Metadata(source) => metadata_error(source), + error => error, + })?; + operation.ensure_active()?; + Ok(summary.into_iter().next().flatten().map(ObjectId::Sha1)) + } + pub(crate) async fn load( store: &Store, layout: &StoreLayout, diff --git a/crates/crab-remote-git/src/snapshot.rs b/crates/crab-remote-git/src/snapshot.rs index 9ca8c17f8..30a3357a7 100644 --- a/crates/crab-remote-git/src/snapshot.rs +++ b/crates/crab-remote-git/src/snapshot.rs @@ -270,6 +270,13 @@ impl RemoteGitSnapshot { page: &PageRequest, operation: &OperationContext, ) -> Result> { + if !path.is_root() + && let Some(index) = self.repository.path_state(operation).await? + { + return self + .indexed_first_parent_path_history(start, path, page, operation, index) + .await; + } if let Some(commit_graph) = self.commit_graph.as_deref() { return self .commit_graph_path_history(start, path, page, operation, commit_graph) @@ -279,6 +286,58 @@ impl RemoteGitSnapshot { .await } + async fn indexed_first_parent_path_history( + &self, + start: ObjectId, + path: &GitPath, + page: &PageRequest, + operation: &OperationContext, + index: &crate::path_state::PathStateIndex, + ) -> Result> { + let mut items = Vec::new(); + let mut next = Some(start); + let mut visited = HashSet::new(); + while let Some(start) = next { + let Some(changed) = index.last_change(start, path, operation).await? else { + break; + }; + if !visited.insert(changed) { + return Err(Error::Corrupt { + stage: CorruptionStage::PathState, + }); + } + let commit = path_history_commit(changed, operation).await?; + let mut commits = vec![commit.clone()]; + next = commit.parents.first().copied(); + if let Some(parent) = next { + commits.push(path_history_commit(parent, operation).await?); + } + if let Some(graph) = self.commit_graph.as_deref() + && !graph.parents_match(&commits, operation).await? + { + return Err(Error::Corrupt { + stage: CorruptionStage::PathState, + }); + } + // Reset markers also cover never-existing descendants. Raw entries + // decide whether this candidate actually changed the requested path. + let entries = entries_at_trees(&commits, path, operation).await?; + if append_path_history_change( + &mut items, + commit, + entries[0].as_ref(), + &entries[1..], + page, + operation, + ) + .await? + { + return self.first_parent_path_page(path, items, next); + } + } + Ok(Page { items, next: None }) + } + async fn commit_graph_path_history( &self, start: ObjectId, @@ -512,7 +571,7 @@ impl RemoteGitSnapshot { } let mut filled_page = false; if traversed >= skip - && let Some(kind) = path_change(self, &commit, path, traversal, operation).await? + && let Some(kind) = path_change(&commit, path, traversal, operation).await? { operation .charge( @@ -677,8 +736,31 @@ impl RemoteGitSnapshot { return Ok(cached.value.as_ref().clone()); } let budget_before = operation.budget_usage().await; - let head_commit = operation.read_commit(self.commit_oid).await?; let head_blob = read_blame_blob(self, path, operation).await?; + let path_index = self.repository.path_state(operation).await?; + let head_oid = match &path_index { + Some(index) => index + .last_change(self.commit_oid, path, operation) + .await? + .ok_or(Error::Corrupt { + stage: CorruptionStage::PathState, + })?, + None => self.commit_oid, + }; + let head_commit = if path_index.is_some() { + path_history_commit(head_oid, operation).await? + } else { + operation.read_commit(head_oid).await? + }; + if path_index.is_some() { + let entry = + entries_at_trees(std::slice::from_ref(&head_commit), path, operation).await?; + if entry[0].as_ref().map(|entry| entry.oid) != Some(head_blob.metadata.oid) { + return Err(Error::Corrupt { + stage: CorruptionStage::PathState, + }); + } + } let mut current_oid = head_blob.metadata.oid; let mut current_bytes = head_blob.bytes; let current_line_count = line_count(¤t_bytes, operation.cancellation())?; @@ -697,7 +779,7 @@ impl RemoteGitSnapshot { .saturating_mul(mem::size_of::()), source, })?; - origins.resize(current_lines.len(), self.commit_oid); + origins.resize(current_lines.len(), head_commit.oid); let mut tracked = Vec::new(); tracked .try_reserve_exact(current_lines.len()) @@ -715,8 +797,27 @@ impl RemoteGitSnapshot { visited.insert(current_commit.oid); 'history: while let Some(parent_oid) = current_commit.parents.first().copied() { - let parent_commits = - blame_parent_commits(parent_oid, self.commit_graph.as_deref(), operation).await?; + let parent_commits = match &path_index { + Some(index) => { + let Some(changed) = index.last_change(parent_oid, path, operation).await? + else { + break; + }; + let commits = vec![path_history_commit(changed, operation).await?]; + if let Some(graph) = self.commit_graph.as_deref() + && !graph.parents_match(&commits, operation).await? + { + return Err(Error::Corrupt { + stage: CorruptionStage::PathState, + }); + } + commits + } + None => { + blame_parent_commits(parent_oid, self.commit_graph.as_deref(), operation) + .await? + } + }; for parent_commit in &parent_commits { if !visited.insert(parent_commit.oid) { return Err(Error::Corrupt { @@ -1743,23 +1844,21 @@ fn blame_ranges( } async fn path_change( - snapshot: &RemoteGitSnapshot, commit: &Commit, path: &GitPath, traversal: HistoryTraversal, operation: &OperationContext, ) -> Result> { - let current = entry_at_tree(snapshot, commit.tree, commit.oid, path, operation).await?; let parent_oids: Vec = match traversal { HistoryTraversal::FirstParent => commit.parents.first().copied().into_iter().collect(), HistoryTraversal::AllParents => commit.parents.clone(), }; - let mut parents = Vec::new(); + let mut commits = vec![commit.clone()]; for oid in parent_oids { - let parent = operation.read_commit(oid).await?; - parents.push(entry_at_tree(snapshot, parent.tree, parent.oid, path, operation).await?); + commits.push(operation.read_commit(oid).await?); } - Ok(path_change_kind(current.as_ref(), &parents)) + let entries = entries_at_trees(&commits, path, operation).await?; + Ok(path_change_kind(entries[0].as_ref(), &entries[1..])) } async fn path_history_commit(oid: ObjectId, operation: &OperationContext) -> Result { @@ -1879,8 +1978,9 @@ async fn entries_at_trees( for ((index, _), entry) in active.into_iter().zip(entries) { trees[index] = match entry { Some(entry) if entry.kind == EntryKind::Tree => Some(entry.oid), - Some(entry) => return Err(Error::PathComponentNotTree { actual: entry.kind }), - None => None, + // Historical descendants are absent when an ancestor was a blob. + // Direct entry reads keep their stricter component-kind error. + Some(_) | None => None, }; } parent = parent.join(component)?; @@ -2019,24 +2119,6 @@ async fn queue_all_parents( Ok(()) } -async fn entry_at_tree( - source: &RemoteGitSnapshot, - tree: ObjectId, - commit: ObjectId, - path: &GitPath, - operation: &OperationContext, -) -> Result> { - let snapshot = RemoteGitSnapshot { - generation: source.generation, - pack_index_hash: source.pack_index_hash, - commit_oid: commit, - root_tree_oid: tree, - repository: Arc::clone(&source.repository), - commit_graph: source.commit_graph.clone(), - }; - snapshot.entry(path, operation).await -} - const CPU_CANCELLATION_INTERVAL: usize = 4 * 1024; fn classify_diff( diff --git a/crates/crab-remote-git/tests/remote_repository.rs b/crates/crab-remote-git/tests/remote_repository.rs index 6054833da..143bb62ae 100644 --- a/crates/crab-remote-git/tests/remote_repository.rs +++ b/crates/crab-remote-git/tests/remote_repository.rs @@ -5679,3 +5679,246 @@ async fn storage_telemetry_counts_shared_reads_once_and_excludes_warm_hits() { assert_eq!(reads.0.load(Ordering::SeqCst), observed); fixture.runtime.shutdown().await; } + +#[tokio::test(flavor = "multi_thread")] +async fn indexed_path_history_matches_raw_git_changes_and_continuations() { + use crab_metadata::path_state::{ + PathStateInput, PathStateMutation, append_path_state, upload_path_state, + }; + use crab_metadata::split_commit_graph::load_split_commit_graph; + let fixture = publish(DeltaKind::Ref, false, RepositoryOptions::default()).await; + crab_metadata::layout_descriptor::ensure_canonical_layout(&fixture.store, &fixture.layout) + .await + .unwrap(); + let hash = publish_complete_graph(&fixture, true).await; + let graph = load_split_commit_graph( + &fixture.store, + &fixture.layout, + &hash, + ObjectLimits::default().max_commit_graph_bytes, + ) + .await + .unwrap(); + let tree = |oid: &str| { + let bytes = git( + &[ + "--git-dir", + path(&fixture.source_git_dir), + "ls-tree", + "-r", + "-t", + "-z", + oid, + ], + None, + ); + bytes + .split(|byte| *byte == 0) + .filter(|row| !row.is_empty()) + .map(|row| { + let tab = row.iter().position(|byte| *byte == b'\t').unwrap(); + let fields = std::str::from_utf8(&row[..tab]) + .unwrap() + .split(' ') + .collect::>(); + ( + row[tab + 1..].to_vec(), + ( + u32::from_str_radix(fields[0], 8).unwrap(), + fields[2].to_owned(), + ), + ) + }) + .collect::>() + }; + let inputs = (0..graph.descriptor.commit_count) + .map(|ordinal| { + let record = graph.record(ordinal).unwrap(); + let oid = gix_hash::ObjectId::Sha1(record.oid); + let parent = record + .parents + .first() + .map(|parent| graph.record(*parent).unwrap().oid); + let before = parent + .map(|parent| tree(&gix_hash::ObjectId::Sha1(parent).to_string())) + .unwrap_or_default(); + let after = tree(&oid.to_string()); + let paths = before + .keys() + .chain(after.keys()) + .cloned() + .collect::>(); + let mut mutations = paths + .into_iter() + .filter(|name| before.get(name) != after.get(name)) + .map(|name| { + let reset = before + .get(&name) + .zip(after.get(&name)) + .is_some_and(|(old, new)| old.0 & 0o170000 != new.0 & 0o170000); + PathStateMutation { + present: after.contains_key(&name), + path: name, + reset, + } + }) + .collect::>(); + if parent.is_none() { + mutations.push(PathStateMutation { + path: Vec::new(), + present: false, + reset: true, + }); + } + PathStateInput { + oid: record.oid, + first_parent: parent, + author: Vec::new(), + author_seconds: 0, + message: Vec::new(), + mutations, + } + }) + .collect(); + let write = append_path_state(None, &graph, inputs).unwrap(); + upload_path_state( + &fixture.store, + &fixture.layout, + &write, + crab_metadata::path_state::DEFAULT_MAX_PATH_STATE_BYTES, + ) + .await + .unwrap(); + let mut captured = + crab_metadata::manifest_store::read_repository_snapshot(&fixture.store, &fixture.layout) + .await + .unwrap(); + captured.manifest.commit_graph_hash = Some(hash); + captured.manifest.path_state_hash = Some(write.descriptor_hash); + let cancel = CancellationToken::new(); + let indexed_runtime = Arc::new(RemoteGitRuntime::default()); + let indexed = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &captured, + fixture.repository.identity().clone(), + indexed_runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + for (reference, name) in [ + ("semantic-head", "mode.txt"), + ("semantic-head", "old-name.txt"), + ("semantic-head", "new-name.txt"), + ("semantic-head", "type/never"), + ("semantic-head", "never-existed"), + ] { + let path_value = GitPath::new(name.as_bytes().to_vec()).unwrap(); + let mut results = Vec::new(); + for repo in [&fixture.repository, &indexed] { + let operation = repo + .operation(OperationKind::PathHistory, &cancel) + .await + .unwrap(); + let result = async { + let snapshot = repo + .snapshot(&Revision::Reference(reference.to_owned()), &operation) + .await?; + let mut cursor = None; + let mut rows = Vec::new(); + loop { + let page = snapshot + .path_history( + &path_value, + HistoryTraversal::FirstParent, + &PageRequest::new(1, cursor)?, + &operation, + ) + .await?; + rows.extend( + page.items + .into_iter() + .map(|entry| (entry.commit.oid, entry.kind)), + ); + cursor = page.next; + if cursor.is_none() { + break; + } + } + Ok(rows) + } + .await; + results.push(operation.finish(result).await.unwrap()); + } + assert_eq!(results[0], results[1], "{reference}:{name}"); + let native = native_oid_lines( + &fixture.source_git_dir, + &[ + "log", + "--first-parent", + "--format=%H", + "--no-renames", + &fixture.semantic_head_commit.to_string(), + "--", + name, + ], + ); + assert_eq!( + results[1].iter().map(|(oid, _)| *oid).collect::>(), + native, + "{reference}:{name}" + ); + } + for name in ["mode.txt", "text.txt", "type.extra"] { + let path_value = GitPath::new(name.as_bytes().to_vec()).unwrap(); + let mut results = Vec::new(); + for repo in [&fixture.repository, &indexed] { + let operation = repo.operation(OperationKind::Blame, &cancel).await.unwrap(); + let result = async { + let snapshot = repo + .snapshot(&Revision::Reference("semantic-head".to_owned()), &operation) + .await?; + snapshot.blame(&path_value, &operation).await + } + .await; + results.push(operation.finish(result).await.unwrap()); + } + assert_eq!(results[0], results[1], "blame:{name}"); + let native = git( + &[ + "--git-dir", + path(&fixture.source_git_dir), + "blame", + "--first-parent", + "--root", + "--line-porcelain", + &fixture.semantic_head_commit.to_string(), + "--", + name, + ], + None, + ); + let expected = native + .split(|byte| *byte == b'\n') + .filter(|row| !row.starts_with(b"\t")) + .filter_map(|row| { + let fields = row + .split(|byte| byte.is_ascii_whitespace()) + .collect::>(); + (fields.len() >= 3 + && fields[0].len() == 40 + && fields[0].iter().all(u8::is_ascii_hexdigit)) + .then(|| parse_oid(fields[0])) + }) + .collect::>(); + let actual = results[1] + .ranges + .iter() + .flat_map(|range| std::iter::repeat_n(range.commit.oid, range.lines as usize)) + .collect::>(); + assert_eq!(actual, expected, "native blame:{name}"); + } + indexed_runtime.shutdown().await; + fixture.runtime.shutdown().await; +} From 612e273c1b1059dbec82d1dff26db4240c0aa772 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 10:34:44 -0700 Subject: [PATCH 026/127] fix(read): normalize scoped cancellation errors --- crates/crab-remote-git/src/operation.rs | 50 ++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/crab-remote-git/src/operation.rs b/crates/crab-remote-git/src/operation.rs index 9614bca76..b18e1dd24 100644 --- a/crates/crab-remote-git/src/operation.rs +++ b/crates/crab-remote-git/src/operation.rs @@ -412,12 +412,15 @@ impl OperationContext { /// ``` pub async fn finish(mut self, operation: Result) -> Result { let timed_out = tokio::time::Instant::now() >= self.deadline; + let interrupted = timed_out || self.cancellation.is_cancelled(); self.deadline_stop.cancel(); let operation = match operation { Ok(_) if timed_out => Err(Error::Timeout { operation: "repository operation", }), - Err(error) if timed_out => Err(error.after_interruption(true)), + // Admission may return wrapped cancellation before a read's outer + // select observes it. Preserve real failures and report scope reason. + Err(error) if interrupted => Err(error.after_interruption(timed_out)), result => result, }; let close = match self.session.take() { @@ -1453,6 +1456,51 @@ mod tests { use super::*; + #[tokio::test] + async fn finish_normalizes_admission_cancellation_after_read_returns() { + let store = Store::new(Arc::new(InMemory::new())); + let runtime = Arc::new(crate::RemoteGitRuntime::default()); + let state = Arc::new(crate::state::RepositoryState { + store: store.clone(), + layout: StoreLayout::new(store, "cancel-finish".to_owned()), + runtime: runtime.clone(), + identity: crate::RepositoryIdentity::new("memory", "cancel-finish", 1).unwrap(), + options: crate::RepositoryOptions::default(), + generation: 1, + pack_index_hash: Arc::from("a".repeat(64)), + git_validation_digest: Arc::from("b".repeat(64)), + manifest_etag: "etag".to_owned(), + shard_index_hash: Arc::from(""), + catalog_identity: None, + lookup_catalog_identity: None, + inventory: HashMap::new(), + refs: crate::RepositoryRefs::default(), + reader: None, + commit_graph: None, + path_state_hash: None, + path_state: tokio::sync::OnceCell::new(), + shallow_closure: None, + }); + let cancel = CancellationToken::new(); + let operation = + OperationContext::open_pack_transfer(state, &cancel, crate::OperationLimits::default()) + .await + .unwrap(); + let returned: Result<()> = Err(Error::Metadata( + crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::ReadRejected { + source: Box::new(Error::Cancelled), + }, + }, + )); + cancel.cancel(); + assert!(matches!( + operation.finish(returned).await, + Err(Error::Cancelled) + )); + runtime.shutdown().await; + } + #[tokio::test] async fn pack_transfer_operation_does_not_open_the_catalog() { let object_store: Arc = Arc::new(InMemory::new()); From b919314f9cae7252adc177efd52fa301e68428b8 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 10:53:03 -0700 Subject: [PATCH 027/127] fix(read): require captured graph readiness --- .../docs/architecture/indexed-remote-reads.md | 30 +++++ crates/crab-remote-git/README.md | 7 ++ crates/crab-remote-git/src/repository.rs | 17 ++- .../src/repository/tests/snapshot_indexes.rs | 109 +++++++++++------- crates/crab-remote/tests/checkpoint.rs | 85 ++++++++++++++ 5 files changed, 204 insertions(+), 44 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index dcc227164..64e6fb29f 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1046,3 +1046,33 @@ PR #549 CI now also reports a container Compose recovery/cluster qualification failure. The failed log is retained; cause remains unresolved and no assertion or baseline has been weakened. This work has not landed and the goal remains active. + + +## Captured graph failures and rejected page-size experiment + +Captured snapshot graph readiness is required when supplied. Previously the +snapshot opener inherited the manifest opener's suppression of graph failures, +which could silently replace indexed reads with raw traversal. All snapshot +constructors now share required graph loading; missing, corrupt or retired +advertised metadata fails opening. Absent readiness still permits raw reads. + +The manifest opener retains its released optional-graph policy. Upstream v1.2.4 +(`b4d5fd361f9a1a43fdd468dc604901d2c61006f0`) suppresses optional graph errors +while propagating cancellation and admission rejection. Released snapshot +constructors did not load graph indexes. Explicit graph attachment remains +required. Separate tests cover graph opening failure and lazy path corruption; +maintenance tests prove rejected retired graph/physical descriptors are rebuilt +from authoritative sources before derived reads resume. + +A 4 KiB derived-leaf experiment completed 186 source-matched RustFS processes +across 100/1,000/10,000 commits, including twenty cold/warm samples each for +resolve, history and blame. Native Git comparisons passed, operations drained, +and warm queries reread no origin bytes. At 10k, history query bytes fell from +4,691,860 to 1,909,743, but requests rose from 696 to 958. Median cold history +latency was 4926.67 ms versus the prior 1,779.36 ms; blame was 4832.20 ms versus +3,036.64 ms. These local debug timings are not representative-S3 targets. +The page-size policy is rejected: no experimental codec versions or leaf +limits are published. Receipts remain in the dedicated qualification volume. + +Checkpoint visibility/catalog opening, comparison caching, provider/lifecycle +proof and later stages remain incomplete. The initiative remains active. diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index 67b54e417..9237d4cca 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -247,3 +247,10 @@ commits. Blame preserves line comparison and deletion boundaries while jumping between relevant versions. Root-path and all-parent history retain their existing raw traversal; index absence retains the ordinary Git read path. Missing or corrupt advertised pages are errors, not a reason to switch traversal. + +Snapshot constructors treat supplied graph readiness as required metadata: a +missing, corrupt or retired advertised graph fails opening. Index absence still +permits ordinary raw reads. The manifest-based `open` retains its release-tagged +optional-graph policy; cancellation and admission failures are never suppressed. +Operation finalization normalizes wrapped cancellation only when its own scope +was interrupted, preserving real failures and typed locator-close errors. diff --git a/crates/crab-remote-git/src/repository.rs b/crates/crab-remote-git/src/repository.rs index 65b6927ad..5cea00b7a 100644 --- a/crates/crab-remote-git/src/repository.rs +++ b/crates/crab-remote-git/src/repository.rs @@ -480,7 +480,14 @@ impl RemoteGitRepository { }), generated_pack_lease_provider: None, } - .with_optional_commit_graph(manifest.commit_graph_hash.as_deref(), cancellation, None) + // Snapshot callers supplied captured readiness. Ignoring its failure + // would silently replace indexed reads with an unrelated raw traversal. + .with_graph( + manifest.commit_graph_hash.as_deref(), + cancellation, + None, + false, + ) .await } @@ -732,10 +739,11 @@ impl RemoteGitRepository { state: Arc::new(state), generated_pack_lease_provider: None, } - .with_optional_commit_graph( + .with_graph( manifest.commit_graph_hash.as_deref(), cancellation, Some(budget), + true, ) .await; } @@ -1243,16 +1251,17 @@ impl RemoteGitRepository { }) } - async fn with_optional_commit_graph( + async fn with_graph( mut self, content_hash: Option<&str>, cancellation: &CancellationToken, budget: Option, + optional: bool, ) -> Result { let Some(content_hash) = content_hash else { return Ok(self); }; - let graph = load_commit_graph(&self, content_hash, cancellation, budget, true).await?; + let graph = load_commit_graph(&self, content_hash, cancellation, budget, optional).await?; Arc::get_mut(&mut self.state) .ok_or(Error::InternalInvariant { invariant: "new repository state was shared before graph attachment", diff --git a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs index c08f3250a..e1817d7c7 100644 --- a/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs +++ b/crates/crab-remote-git/src/repository/tests/snapshot_indexes.rs @@ -236,49 +236,78 @@ async fn snapshot_read_indexes_do_not_hide_exhausted_request_or_byte_budgets() { } #[tokio::test] -async fn snapshot_read_indexes_fail_attribution_when_declared_metadata_is_corrupt() { - for kind in ["commit-graph", "path-state"] { - let fixture = open_fixture(1, None).await; - let snapshot = snapshot_with_read_indexes(&fixture, true).await; - let hash = match kind { - "commit-graph" => snapshot.manifest.commit_graph_hash.as_deref().unwrap(), - _ => snapshot.manifest.path_state_hash.as_deref().unwrap(), - }; - // Bypass immutable-write protection only to inject corrupt origin bytes. - fixture - .backend - .inner - .put( - &fixture.layout.bulk_manifest_path(kind, hash), - Bytes::from_static(b"corrupt").into(), - ) - .await - .unwrap(); - let runtime = Arc::new(RemoteGitRuntime::default()); - let cancel = CancellationToken::new(); - let repository = RemoteGitRepository::from_snapshot( - fixture.layout.clone(), - &snapshot, - RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), - runtime.clone(), - RepositoryOptions::default(), - &cancel, +async fn captured_corrupt_graph_fails_opening_instead_of_raw_traversal() { + let fixture = open_fixture(1, None).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; + let hash = snapshot.manifest.commit_graph_hash.as_deref().unwrap(); + fixture + .backend + .inner + .put( + &fixture.layout.bulk_manifest_path("commit-graph", hash), + Bytes::from_static(b"corrupt").into(), ) .await .unwrap(); - let operation = repository - .operation(crate::OperationKind::History, &cancel) - .await - .unwrap(); - let result = repository.state.path_state(&operation).await.map(|_| ()); - assert!(matches!( - operation.finish(result).await, - Err(Error::Corrupt { - stage: crate::CorruptionStage::PathState - }) - )); - runtime.shutdown().await; - } + let runtime = Arc::new(RemoteGitRuntime::default()); + let result = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &CancellationToken::new(), + ) + .await; + assert!(matches!( + result, + Err(Error::Metadata( + crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::CorruptObject { .. } + } + )) + )); + runtime.shutdown().await; +} + +#[tokio::test] +async fn captured_corrupt_path_state_fails_attribution() { + let fixture = open_fixture(1, None).await; + let snapshot = snapshot_with_read_indexes(&fixture, true).await; + let hash = snapshot.manifest.path_state_hash.as_deref().unwrap(); + fixture + .backend + .inner + .put( + &fixture.layout.bulk_manifest_path("path-state", hash), + Bytes::from_static(b"corrupt").into(), + ) + .await + .unwrap(); + let runtime = Arc::new(RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let repository = RemoteGitRepository::from_snapshot( + fixture.layout.clone(), + &snapshot, + RepositoryIdentity::new("memory", "org/repo", 1).unwrap(), + runtime.clone(), + RepositoryOptions::default(), + &cancel, + ) + .await + .unwrap(); + let operation = repository + .operation(crate::OperationKind::History, &cancel) + .await + .unwrap(); + let result = repository.state.path_state(&operation).await.map(|_| ()); + assert!(matches!( + operation.finish(result).await, + Err(Error::Corrupt { + stage: crate::CorruptionStage::PathState + }) + )); + runtime.shutdown().await; } #[tokio::test(flavor = "multi_thread")] diff --git a/crates/crab-remote/tests/checkpoint.rs b/crates/crab-remote/tests/checkpoint.rs index bec518f3f..ce24a3c17 100644 --- a/crates/crab-remote/tests/checkpoint.rs +++ b/crates/crab-remote/tests/checkpoint.rs @@ -1825,3 +1825,88 @@ async fn incremental_install_enforces_byte_budget_before_pack_payload_reads() { assert_eq!(actual.stdout, expected); } } + +#[tokio::test] +async fn maintenance_rebuilds_retired_advertised_indexes_before_opening_derived_reads() { + use crab_metadata::capsule_protocol::load_browse_indexes; + let (layout, _) = fixture().await; + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let identity = crab_remote_git::RepositoryIdentity::new("memory", "retirement", 1).unwrap(); + let cancel = CancellationToken::new(); + let build = || { + crab_remote::browse_indexes::ensure( + &layout, + &identity, + runtime.clone(), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel, + ) + }; + let original = build().await.unwrap().unwrap(); + for (field, family, hash, version) in [ + ( + "commit_graph_hash", + "commit-graph", + original.commit_graph_hash(), + 2, + ), + ( + "object_index_hash", + "git-object-index", + original.object_index_hash().unwrap(), + 1, + ), + ] { + let (bytes, _) = layout + .store() + .get_with_etag_bounded(&layout.bulk_manifest_path(family, hash), LIMIT) + .await + .unwrap(); + let mut descriptor: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + descriptor["version"] = serde_json::json!(version); + let bytes = serde_json::to_vec(&descriptor).unwrap(); + let retired_hash = blake3::hash(&bytes).to_hex().to_string(); + layout + .store() + .put_overwrite( + &layout.bulk_manifest_path(family, &retired_hash), + bytes.into(), + ) + .await + .unwrap(); + let mut advertised: serde_json::Value = + serde_json::from_slice(&original.encode().unwrap()).unwrap(); + advertised[field] = serde_json::json!(retired_hash); + layout + .store() + .put_overwrite( + &layout.capsule_browse_indexes_path(), + serde_json::to_vec(&advertised).unwrap().into(), + ) + .await + .unwrap(); + let captured = view(&layout) + .await + .with_browse_indexes(load_browse_indexes(&layout).await.unwrap()); + assert!( + captured + .git_repository_from_store( + layout.clone(), + identity.clone(), + runtime.clone(), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel + ) + .await + .is_err() + ); + assert_eq!(build().await.unwrap().unwrap(), original); + assert_eq!( + load_browse_indexes(&layout).await.unwrap(), + Some(original.clone()) + ); + } + runtime.shutdown().await; +} From f41f76564048153593fd46ae3b1bf62c41b07f45 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 11:42:04 -0700 Subject: [PATCH 028/127] perf(read): validate checkpoint admission once --- .../docs/architecture/indexed-remote-reads.md | 50 +++++++++++++++ crates/crab-metadata/README.md | 4 ++ .../src/capsule_protocol/layered.rs | 63 ++++++++++++++++++- crates/crab-read/src/capsule_protocol.rs | 13 ++-- crates/crab-remote/README.md | 15 +++-- crates/crab-remote/examples/qualify_remote.rs | 30 ++++++++- crates/crab-remote/examples/support/mod.rs | 13 ++-- 7 files changed, 170 insertions(+), 18 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 64e6fb29f..7fca50f43 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1076,3 +1076,53 @@ limits are published. Receipts remain in the dedicated qualification volume. Checkpoint visibility/catalog opening, comparison caching, provider/lifecycle proof and later stages remain incomplete. The initiative remains active. + + +## Checkpoint admission ownership and measured duplicate decoding + +The qualifier now exposes `checkpoint`, which uses the real logical publisher +and verifies the advertised pointer and footer afterward. It reports publication +work independently of reads. A successful `metadb compact` no-op is not used +as checkpoint evidence. `checkpoint_visibility` records complete ordinal-proof +decodes separately from graph/path/physical and frontier visibility pages. + +The dedicated RustFS fixtures were checkpointed at 100/1,000/10,000 commits, +then their indexes rebuilt. A 189-process baseline passes native Git comparisons +and twenty cold/warm samples each for resolve, path history and blame per cell. +It exposes two full visibility decodes during every indexed opening. + +Source/member ordinal bounds now belong to the checkpoint metadata decoder, +which checks its own source inventory before frontier sources are appended. +Previously an out-of-range checkpoint source could alias a newer frontier +source in the combined inventory. The new regression fails against the prior +decoder, then passes with the source fix. Complete view capture still checks +the source-catalog digest and closure proof. With verified physical readiness, +repository opening skips the second decode used only to construct placement +hints. Unindexed placement retains that needed admission map; ordinary fetch +controls still leave the body cold. No format, dependency or config changes. + +A source/binary-matched 183-process replay passes the same native oracles and +twenty cold/warm samples per operation/history. Operations drain and warm query +origin bytes remain zero. The measured change is decode work, not I/O: + +| Commits | Opening origin bytes | Proof bytes decoded before | After | +| --- | ---: | ---: | ---: | +| 100 | 29,255 | 20,492 | 10,246 | +| 1,000 | 119,259 | 150,768 | 75,384 | +| 10,000 | 771,540 | 1,453,518 | 726,759 | + +Proof: 15 checkpoint codec tests with storage and 15 payload-only tests; 46 +capsule-read tests; qualifier instrumentation tests; strict selected-owner +Clippy (`--no-deps`, all targets); qualifier and CLI link builds. A broader +dependency-inclusive Clippy probe still fails six pre-existing coordination +lint expectations; this is not a green workspace-lint claim. These expectations +are present on this stack and absent on current `origin/main`; no main-only +unrelated-failure claim is made. CI remains a separate gate. + +Opening still reads and decodes the entire checkpoint proof once. The next +implementation must page the retained OID dictionary, source/member bindings, +ref closures and transition histories through authenticated footer roots, then +wire lazy initial reconciliation and explicit bulk materialization. Retained +but invisible OIDs must stay separate from ref membership. Footer/source/ref +inventory costs, historical Xet catalog paging, comparison caching, lifecycle +and provider proof and fuller blame semantics remain incomplete. diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 66cc207d6..d41925bab 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -138,6 +138,10 @@ Ordinal proofs must match the ordered source-catalog digest before becoming a Git visibility index; the existing full-object encoding preserves the same caller-supplied Git identity. A missing proof remains distinct from an empty ref set, so recovery can reject it explicitly. +Checkpoint ordinal decoding validates source/member positions against that +checkpoint's own inventory, before newer frontier sources are appended. Indexed +opening can then avoid decoding the complete proof again for placement hints; +unindexed readers retain the admission map they actually consume. Footer-only checkpoints have control offset zero when both optional catalog and visibility sections are absent. The storage reader accepts that canonical range while retaining footer-hash, pointer identity, source and size validation. diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index 0699b8960..87a99750c 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -279,6 +279,8 @@ impl LayeredVisibilitySnapshot { if snapshot.encode()?.as_ref() != bytes { return Err(corrupt("layered visibility snapshot is not canonical")); } + #[cfg(feature = "storage")] + tracing::debug!(target: "crab_metadata::index_decode", index_format = "checkpoint_visibility", decoded_bytes = bytes.len() as u64); Ok(snapshot) } @@ -1684,12 +1686,29 @@ impl LayeredCheckpoint { crate::capsule_protocol::CapsuleVisibilitySnapshot::decode(§ion).map(Some) } - /// Decode the compact ordinal visibility proof, when present. + /// Decode the compact ordinal proof and validate its checkpoint source bindings. pub fn visibility_ordinal_snapshot(&self) -> Result> { let Some(section) = self.section(CapsuleSectionKind::VisibilityOrdinalSnapshot)? else { return Ok(None); }; - LayeredVisibilitySnapshot::decode(§ion).map(Some) + let snapshot = LayeredVisibilitySnapshot::decode(§ion)?; + // Admission ordinals belong to this checkpoint, before frontier sources + // are appended. Validating against the combined inventory would let an + // out-of-range ordinal alias an unrelated newer source. + if snapshot.member_admission().is_some_and(|members| { + members.iter().any(|member| { + self.footer + .sources + .get(usize::from(member.source_index())) + .and_then(|source| source.members().get(usize::from(member.member_index()))) + .is_none() + }) + }) { + return Err(corrupt( + "layered visibility admission is outside its checkpoint source inventory", + )); + } + Ok(Some(snapshot)) } /// Decode visibility bound to this source inventory and the caller's Git identity. @@ -2307,6 +2326,46 @@ mod tests { ); } + #[test] + fn checkpoint_visibility_rejects_admission_outside_its_own_sources() { + let layer = PackLayer::build(&pack()).unwrap(); + let source = layer.source_descriptor().unwrap(); + let digest = source_catalog_digest(std::slice::from_ref(&source)).unwrap(); + let index = GitVisibilityIndex::new( + 1, + "1".repeat(64), + "2".repeat(64), + BTreeMap::from([("refs/heads/main".to_owned(), vec!["a".repeat(40)])]), + ) + .unwrap(); + for (member, valid) in [ + (LayeredObjectMember::new(0, 0), true), + (LayeredObjectMember::new(1, 0), false), + (LayeredObjectMember::new(0, 1), false), + ] { + let snapshot = LayeredVisibilitySnapshot::from_index_with_member_admission( + &index, + &digest, + vec![member], + ) + .unwrap(); + let checkpoint = LayeredCheckpoint::build_with_ordinal_visibility( + 1, + &"3".repeat(64), + vec![source.clone()], + PointerCatalog::new(), + Some(snapshot), + ) + .unwrap(); + assert_eq!( + checkpoint + .visibility_index(1, &"1".repeat(64), &"2".repeat(64)) + .is_ok(), + valid + ); + } + } + #[test] fn ordinal_visibility_member_admission_round_trips() { let refs = BTreeMap::from([( diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index fe6b1e2a8..02aa7cec1 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -964,10 +964,12 @@ impl CapsuleRepositoryView { // admission, while the complete ordinal join is only available to // maintenance/strict readers that loaded the full checkpoint. // A ready physical index resolves every placement before hint paths run. - // Keep validating source/member admission, but avoid a second OID dictionary. + // Checkpoint visibility decoding already validates source/member bounds. + // Only unindexed placement needs the complete admission dictionary here. let mut preferred_object_admission_by_oid = object_index.is_none().then(std::collections::HashMap::new); - if let Some(checkpoint) = checkpoint + if let Some(admission_by_oid) = preferred_object_admission_by_oid.as_mut() + && let Some(checkpoint) = checkpoint && !checkpoint.is_control_only() && let Some(snapshot) = checkpoint.visibility_ordinal_snapshot()? && let Some(admission) = snapshot.member_admission() @@ -986,9 +988,10 @@ impl CapsuleRepositoryView { })?; let pack_id = crab_xet::hash::MerkleHash::from_hex(pack.pack().blake3()) .map_err(|error| corrupt_path("layered visibility", error.to_string()))?; - if let Some(admission) = preferred_object_admission_by_oid.as_mut() { - admission.entry(*oid).or_insert_with(Vec::new).push(pack_id); - } + admission_by_oid + .entry(*oid) + .or_insert_with(Vec::new) + .push(pack_id); } } for (oid, pack_ids) in self.frontier_object_admission() { diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 49923e50f..aa2735df6 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -82,10 +82,10 @@ this orchestration boundary. The low-level remote Git reader retains its narrow Git/storage/metadata dependency boundary. See the [indexed-read qualification contract](../../crab/docs/architecture/indexed-remote-reads.md) for commands, native Git oracles and cold/warm measurement semantics. -Foreground qualification and browse-index builders capture complete metadata -through `open_view_from_root_with_metadata`; pack payloads stay in authenticated -origin ranges before and after checkpoints. Catalog/control decoding and ref -capture remain measured opening work, not a completed bounded-opening gate. +Foreground qualification uses `open_git_view_from_root`; browse-index builders +use the complete metadata entry point. Pack payloads stay in authenticated origin +ranges before and after checkpoints. Checkpoint catalog/visibility decoding and +ref capture remain measured opening work, not a completed bounded-opening gate. `qualify_remote graph-pages` measures sampled ordinal/parent records and duplicate OID batches through `CommitGraphReader`. It verifies the derived graph's captured state and Git identity before reads. @@ -96,3 +96,10 @@ workload does not claim bounded foreground opening or replace ref visibility. ref view. The native oracle uses `git merge-base --is-ancestor`, treating only exit status 1 as a valid negative answer. Foreground graph conversion is a separate gate; this component qualification does not bypass repository visibility. + +`checkpoint` publishes through the real logical-checkpoint API, then verifies +the advertised checkpoint pointer and authenticated footer. It mutates the +chosen repository; use a dedicated qualification prefix. Run `build-indexes` +afterward, then repeat the same native-Git query oracles. This separates +checkpoint maintenance costs from foreground opening and prevents a no-op +maintenance command from standing in for checkpoint qualification. diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index f929fd9c3..36ffe5c1c 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -36,12 +36,13 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint]", ) .into()), }; if selection != "all" && selection != "build-indexes" + && selection != "checkpoint" && selection != "graph-pages" && selection != "graph-ancestry" && !WORKLOADS.contains(&selection) @@ -67,7 +68,32 @@ async fn main() -> Result<()> { .with_storage_observer(counters.clone()); let layout = StoreLayout::new(store, prefix.to_owned()); let identity = RepositoryIdentity::new(format!("s3:{bucket}"), prefix.as_str(), 1)?; - if selection == "build-indexes" { + if selection == "checkpoint" { + counters + .measure("checkpoint", "maintenance", async { + let outcome = crab_remote::checkpoint::publish_capsule_checkpoint( + &layout, + 0, + 2 * 1024 * 1024 * 1024, + &CancellationToken::new(), + ) + .await?; + if !outcome.published { + return Err(io::Error::other("checkpoint publication lost its captured root").into()); + } + let root = crab_metadata::capsule_protocol::load_root(&layout).await?; + let pointer = root.record().root().checkpoint().ok_or_else(|| { + io::Error::other("published root does not advertise a checkpoint") + })?; + let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint_control( + &layout, pointer, + ) + .await?; + println!("{}", json!({"kind":"checkpoint", "hash":pointer.hash(), "bytes":pointer.size(), "control_bytes":pointer.control_size(), "sources":checkpoint.source_count(), "objects":checkpoint.object_count()?, "pack_bytes_read":outcome.pack_bytes_read, "pack_bytes_written":outcome.pack_bytes_written})); + Ok(()) + }) + .await?; + } else if selection == "build-indexes" { let runtime = Arc::new(RemoteGitRuntime::default()); let cancel = CancellationToken::new(); let result = counters diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index 5edeed14b..693b25ba9 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -25,8 +25,8 @@ pub struct Measurements { read_bytes: AtomicU64, written_bytes: AtomicU64, active: AtomicU64, - decoded_blocks: [AtomicU64; 5], - decoded_bytes: [AtomicU64; 5], + decoded_blocks: [AtomicU64; 6], + decoded_bytes: [AtomicU64; 6], } #[derive(Clone, Copy)] @@ -35,8 +35,8 @@ struct Snapshot { outcomes: [u64; 9], read_bytes: u64, written_bytes: u64, - decoded_blocks: [u64; 5], - decoded_bytes: [u64; 5], + decoded_blocks: [u64; 6], + decoded_bytes: [u64; 6], } impl StorageObserver for Measurements { @@ -73,6 +73,7 @@ impl tracing::field::Visit for DecodeEvent { "git_object_index" => Some(2), "capsule_admission" => Some(3), "capsule_visibility" => Some(4), + "checkpoint_visibility" => Some(5), _ => None, }; } @@ -168,6 +169,7 @@ impl Measurements { "git_object_index", "capsule_admission", "capsule_visibility", + "checkpoint_visibility", ] .into_iter() .enumerate() @@ -245,12 +247,13 @@ mod tests { tracing::debug!(target:"crab_metadata::index_decode", index_format="git_object_index", decoded_bytes=67_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_admission", decoded_bytes=29_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_visibility", decoded_bytes=31_u64); + tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility", decoded_bytes=37_u64); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), - ([1, 1, 1, 1, 1], [41, 123, 67, 29, 31]) + ([1, 1, 1, 1, 1, 1], [41, 123, 67, 29, 31, 37]) ); } From 53633ca90f22e398147d71fed09a6b11d40292fe Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 13:10:04 -0700 Subject: [PATCH 029/127] perf(read): page checkpoint visibility Require unshipped checkpoint format 6 and bind visibility page roots to the captured footer and source catalog. Keep complete catalog access explicit and async across hydration and publication consumers. --- .../docs/architecture/indexed-remote-reads.md | 45 ++ crab/docs/design/capsule-layered-packs.md | 9 + .../design/capsule-publication-protocol.md | 15 +- crab/docs/design/capsule-xorbs-shards.md | 9 + crab/src/cmd/compact.rs | 4 +- crab/src/cmd/fsck_store.rs | 4 +- crab/src/cmd/gc/bucket.rs | 2 +- crab/src/cmd/history_recovery_v2.rs | 25 +- .../cmd/history_recovery_v2/layered_tests.rs | 2 +- crab/src/cmd/metadb.rs | 6 +- crab/src/cmd/metadb/capsule_tests.rs | 2 +- crab/src/cmd/mount.rs | 2 +- crab/src/cmd/optimize/xorbs.rs | 2 +- crab/src/git/xet_publication.rs | 2 +- crab/src/import/publish.rs | 1 + crab/src/optimize/xorbs/reconcile.rs | 4 +- crab/src/read/mod.rs | 2 +- crab/src/replication/mod.rs | 2 +- .../crab-auth-server/src/receive/capsule.rs | 6 +- .../crab-auth-server/src/receive/workflow.rs | 2 +- crates/crab-metadata/README.md | 22 +- .../src/capsule_protocol/history.rs | 15 +- .../src/capsule_protocol/layered.rs | 686 ++++-------------- .../layered/visibility_pages.rs | 430 +++++++++++ .../layered/visibility_pages/reader.rs | 249 +++++++ .../layered/visibility_pages/tests.rs | 183 +++++ .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/root.rs | 10 +- .../src/capsule_protocol/store.rs | 4 +- .../src/capsule_protocol/visibility.rs | 146 +--- crates/crab-metadata/src/git_visibility.rs | 32 - crates/crab-metadata/src/pages.rs | 19 +- crates/crab-read/README.md | 9 + crates/crab-read/src/capsule_protocol.rs | 128 +++- .../src/capsule_protocol/visibility.rs | 89 ++- crates/crab-read/src/selection.rs | 2 +- crates/crab-remote/src/checkpoint.rs | 23 +- crates/crab-remote/src/prepare.rs | 2 +- crates/crab-write/src/capsule_protocol.rs | 12 +- 39 files changed, 1362 insertions(+), 849 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/layered/visibility_pages.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 7fca50f43..29d5b7192 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1126,3 +1126,48 @@ wire lazy initial reconciliation and explicit bulk materialization. Retained but invisible OIDs must stay separate from ref membership. Footer/source/ref inventory costs, historical Xet catalog paging, comparison caching, lifecycle and provider proof and fuller blame semantics remain incomplete. + + +## Checkpoint visibility pages in progress + +The new canonical checkpoint format is 6 (`CRBCKP06`). A local release-tag audit +checked all 19 tags; none contains the layered checkpoint module. The released +v1 manifest reader is unchanged. Retired v5 checkpoint pointers are rejected +explicitly; existing v5 qualification prefixes must not be used for v6 proof. +No foreground v5 decoder or full-scan codec fallback is added. + +The implementation replaces the flat ordinal-body codec with authenticated +dictionary, ref-closure and transition pages. Footer roots bind the body region, +retained-object count and source-catalog digest. Normal Git capture loads the +footer and initial ref headers; reconciliation queries dictionary retention and +actual ref membership separately. Explicit bulk inspection verifies the entire +section against those captured roots and its canonical encoding. Source/member +bounds are checked by both producer and reader against checkpoint sources. + +The obsolete flat snapshot type, decoder, builder branch and encoding-ratio test +are removed. Canonical membership and incremental-history behavior remain in +the checkpoint tests. Recovery uses the same builder and pointer format. + +CLI hydration also consumes Git views, so the shared view's `pointer_catalog` +API is now asynchronous. It explicitly materializes the captured checkpoint +when required; Git opening does not fetch that body. Publication, hydration, +protected receive, replication and maintenance callers await that canonical API. +Full catalog materialization still exists as the explicit bulk operation; +normal historical Xet lookups remain a later, incomplete gate. + +Current proof: targeted capsule suites pass for metadata (96), read (46), and +write (22). Metadata includes four lazy-reader tests: a 50k-object lookup within +128 KiB despite a body over 2 MiB, retained-but-invisible denial, warm zero origin +reads, full-materialization parity, and corrupted/missing page failure. Shared +read/write/remote compile checks, the CLI link build, protected-receive all-target +compile check and 84 payload-only capsule tests pass. Strict owner-crate +Clippy passes with `--no-deps --all-targets` for metadata/read/write/remote. Format-specific +fixtures use v6 without changing their epoch/visibility assertions. These are +focused proofs, not fresh-prefix RustFS or end-to-end v6 qualification. + +Before marking this draft ready: finish live scope-wide API consumer proof; +exercise checkpoint plus later ref edits, corruption, cancellation, physical +index absence and explicit catalog materialization; run fresh-prefix RustFS +qualification with frozen native Git fixtures and 20 cold/warm samples. Source +inventory/ref capture, comparison caches, catalog indexing, lifecycle/provider +qualification, unified staging and fuller blame remain incomplete. diff --git a/crab/docs/design/capsule-layered-packs.md b/crab/docs/design/capsule-layered-packs.md index f0ca45fd8..cf1ecb070 100644 --- a/crab/docs/design/capsule-layered-packs.md +++ b/crab/docs/design/capsule-layered-packs.md @@ -1,5 +1,14 @@ # Protocol v2 Stable Layered Packs +Current development checkpoint format is `CRBCKP06`: visibility dictionaries, +ref closures and transitions use authenticated pages rooted in the footer. +Git views load those pages lazily; explicit catalog access remains a complete +async bulk read. See [indexed remote reads](../architecture/indexed-remote-reads.md#checkpoint-visibility-pages-in-progress) +for proof and remaining gates. `CRBCKP05` records below describe the preceding +format and its historical qualification; they do not qualify v6. Readers reject +retired v5 pointers, and fresh-prefix v6 live qualification remains pending. + + ## Document metadata | Field | Value | diff --git a/crab/docs/design/capsule-publication-protocol.md b/crab/docs/design/capsule-publication-protocol.md index 170003aed..f4f14b87b 100644 --- a/crab/docs/design/capsule-publication-protocol.md +++ b/crab/docs/design/capsule-publication-protocol.md @@ -1,5 +1,14 @@ # Capsule Publication Protocol +Current development checkpoint format is `CRBCKP06`: visibility dictionaries, +ref closures and transitions use authenticated pages rooted in the footer. +Git views load those pages lazily; explicit catalog access remains a complete +async bulk read. See [indexed remote reads](../architecture/indexed-remote-reads.md#checkpoint-visibility-pages-in-progress) +for proof and remaining gates. `CRBCKP05` records below describe the preceding +format and its historical qualification; they do not qualify v6. Readers reject +retired v5 pointers, and fresh-prefix v6 live qualification remains pending. + + ## Document metadata | Field | Value | @@ -13,7 +22,7 @@ ### Implementation status -The implemented checkpoint is metadata-only `CRBCKP05`. Its ordered source +The implemented checkpoint is metadata-only `CRBCKP06`. Its ordered source directory binds immutable `CRBRUN01` capsule runs and `CRBPKL01` pack layers; canonical xorbs and shards remain external. The [stable layered-pack design](capsule-layered-packs.md) is the detailed read, maintenance, and @@ -64,8 +73,8 @@ The hard-cutover implementation is wired to the user-facing ordinary Git path: or ref-head CAS according to the authority being changed; - `crab-read::capsule_protocol` loads the root and its bounded capsule frontier concurrently, verifying every size, content, transaction, and base binding; -- checkpoint format `CRBCKP05` binds the immutable source directory, pointer - catalog, compact ordinal visibility, and bounded transition evidence. +- checkpoint format `CRBCKP06` binds the immutable source directory, pointer + catalog, paged visibility closures, and bounded transition evidence. Ordinary incremental reads authenticate its footer range without transferring the complete visibility body or stable pack bodies. Selected source ranges retain Git pack checksum, entry CRC/delta, and reconstructed-object checks; diff --git a/crab/docs/design/capsule-xorbs-shards.md b/crab/docs/design/capsule-xorbs-shards.md index 0dd1c1fa8..0f100867a 100644 --- a/crab/docs/design/capsule-xorbs-shards.md +++ b/crab/docs/design/capsule-xorbs-shards.md @@ -1,5 +1,14 @@ # Protocol v2 Xorb and Shard Integration +Current development checkpoint format is `CRBCKP06`: visibility dictionaries, +ref closures and transitions use authenticated pages rooted in the footer. +Git views load those pages lazily; explicit catalog access remains a complete +async bulk read. See [indexed remote reads](../architecture/indexed-remote-reads.md#checkpoint-visibility-pages-in-progress) +for proof and remaining gates. `CRBCKP05` records below describe the preceding +format and its historical qualification; they do not qualify v6. Readers reject +retired v5 pointers, and fresh-prefix v6 live qualification remains pending. + + ## Document metadata | Field | Value | diff --git a/crab/src/cmd/compact.rs b/crab/src/cmd/compact.rs index c0b084656..79a406ad3 100644 --- a/crab/src/cmd/compact.rs +++ b/crab/src/cmd/compact.rs @@ -237,7 +237,7 @@ async fn run_capsule_compact_inner( ..CompactOutcome::default() }); } - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; let source_hashes = catalog .files() .values() @@ -1394,7 +1394,7 @@ mod tests { ) .await .unwrap(); - let compacted = view.pointer_catalog().unwrap(); + let compacted = view.pointer_catalog().await.unwrap(); assert_eq!(compacted.shards().len(), 1); assert_eq!( compacted diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 982b9cb78..58eef16df 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -133,7 +133,7 @@ impl StoreChecker { }, ) .await?; - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; verify_capsule_history(&layout, view.root().root()).await?; Ok(Self { store, @@ -2211,7 +2211,7 @@ mod tests { view.root().root().generation(), view.root().digest(), vec![layer.source_descriptor().unwrap()], - view.pointer_catalog().unwrap(), + view.pointer_catalog().await.unwrap(), None, ) .unwrap(); diff --git a/crab/src/cmd/gc/bucket.rs b/crab/src/cmd/gc/bucket.rs index cc980cb6a..991eb75ea 100644 --- a/crab/src/cmd/gc/bucket.rs +++ b/crab/src/cmd/gc/bucket.rs @@ -1094,7 +1094,7 @@ async fn capsule_repository_roots( ) .await?; let root_digest = view.state_digest(); - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; Ok(Some(CapsuleRepositoryRoots { root_digest, shard_hashes: catalog.shards().keys().cloned().collect(), diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index f162c7d41..f46a75818 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -614,7 +614,7 @@ async fn restore_history( let previous_generation = view.root().root().generation(); let (refs_added, refs_updated, refs_deleted) = ref_change_counts(view.refs(), verified.segment.refs()); - let current_catalog = view.pointer_catalog()?; + let current_catalog = view.pointer_catalog().await?; // Keep both catalogs: history may use a recipe no longer in the current // view, while newer content must remain available after a rollback. let mut restored_catalog = verified.checkpoint.pointer_catalog()?; @@ -674,22 +674,13 @@ async fn restore_history( .await?; let restore = async { - let checkpoint = match verified.checkpoint.visibility_ordinal_snapshot()? { - Some(visibility) => LayeredCheckpoint::build_with_ordinal_visibility( - fenced.record().root().generation(), - fenced.record().digest(), - verified.checkpoint.sources().to_vec(), - restored_catalog, - Some(visibility), - )?, - None => LayeredCheckpoint::build( - fenced.record().root().generation(), - fenced.record().digest(), - verified.checkpoint.sources().to_vec(), - restored_catalog, - verified.checkpoint.visibility_snapshot()?, - )?, - }; + let checkpoint = LayeredCheckpoint::build( + fenced.record().root().generation(), + fenced.record().digest(), + verified.checkpoint.sources().to_vec(), + restored_catalog, + verified.checkpoint.visibility_ordinal_snapshot()?, + )?; check_cancelled(cancel)?; crab_write::capsule_protocol::restore_checkpoint( layout, diff --git a/crab/src/cmd/history_recovery_v2/layered_tests.rs b/crab/src/cmd/history_recovery_v2/layered_tests.rs index b837fe1b1..468da9e87 100644 --- a/crab/src/cmd/history_recovery_v2/layered_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_tests.rs @@ -129,7 +129,7 @@ async fn layered_history_verification_preserves_historical_refs_and_bytes() { let (layout, root) = fixture().await; let chain = history_chain(&layout, &root).await.unwrap(); let oldest = chain.last().unwrap(); - assert_eq!(oldest.checkpoint().format(), 5); + assert_eq!(oldest.checkpoint().format(), 6); let verified = verify_history( &layout, diff --git a/crab/src/cmd/metadb.rs b/crab/src/cmd/metadb.rs index fc7f1f0f6..1cf983cb2 100644 --- a/crab/src/cmd/metadb.rs +++ b/crab/src/cmd/metadb.rs @@ -1737,7 +1737,7 @@ async fn diagnose_capsule( }, ) .await?; - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; view.git_visibility_index().await?; // Verification must read layered sources and prove reachable content, // not rebuild a complete pack or accept catalog membership as file proof. @@ -2662,7 +2662,7 @@ async fn run_capsule_rebuild_in( }, ) .await?; - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; view.git_visibility_index().await?; // Authenticate the complete closure before either a no-op success or a new // checkpoint; immutable publication alone does not verify reachable files. @@ -2862,7 +2862,7 @@ async fn verify_capsule_file_index( ) .await?; Ok(capsule_file_index_matches( - &view.pointer_catalog()?, + &view.pointer_catalog().await?, entries, )) } diff --git a/crab/src/cmd/metadb/capsule_tests.rs b/crab/src/cmd/metadb/capsule_tests.rs index a9dccf2a0..4bd0aa6ed 100644 --- a/crab/src/cmd/metadb/capsule_tests.rs +++ b/crab/src/cmd/metadb/capsule_tests.rs @@ -109,7 +109,7 @@ async fn layered_diagnosis_and_rebuild_verify_nonempty_repository_without_rewrit .await .unwrap(); let before = root.record().digest().to_owned(); - assert_eq!(root.record().root().checkpoint().unwrap().format(), 5); + assert_eq!(root.record().root().checkpoint().unwrap().format(), 6); let diagnosis = diagnose_capsule(&layout, root.clone(), DbSelector::Both, true, &cancel) .await .unwrap(); diff --git a/crab/src/cmd/mount.rs b/crab/src/cmd/mount.rs index 506989c29..0ed8c170a 100644 --- a/crab/src/cmd/mount.rs +++ b/crab/src/cmd/mount.rs @@ -2087,7 +2087,7 @@ async fn resolve_mount_read_context_from_remote_url( ) .await .ok()?; - let catalog = view.pointer_catalog().ok()?; + let catalog = view.pointer_catalog().await.ok()?; Some( crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( read_layout.clone(), diff --git a/crab/src/cmd/optimize/xorbs.rs b/crab/src/cmd/optimize/xorbs.rs index ec568a1dc..1a2f39062 100644 --- a/crab/src/cmd/optimize/xorbs.rs +++ b/crab/src/cmd/optimize/xorbs.rs @@ -391,7 +391,7 @@ async fn enumerate_capsule_sources( view: &crab_read::capsule_protocol::CapsuleRepositoryView, cancel: &CancellationToken, ) -> Result> { - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; let selected_shards = catalog .files() .values() diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 20d6d2dcf..4e9a0f68f 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -65,7 +65,7 @@ pub(crate) async fn prepare_delta( }, ) .await?; - let base_catalog = view.pointer_catalog()?; + let base_catalog = view.pointer_catalog().await?; let mut pointer_sizes = BTreeMap::::new(); for pointer in pointers { diff --git a/crab/src/import/publish.rs b/crab/src/import/publish.rs index 285571b82..05dda8019 100644 --- a/crab/src/import/publish.rs +++ b/crab/src/import/publish.rs @@ -804,6 +804,7 @@ mod tests { ); let catalog = view .pointer_catalog() + .await .expect("decode imported pointer catalog"); for entry in &e2e.staged_entries { let EntryState::Staged { file_hash } = entry.state else { diff --git a/crab/src/optimize/xorbs/reconcile.rs b/crab/src/optimize/xorbs/reconcile.rs index 95e92c333..e4ff344d4 100644 --- a/crab/src/optimize/xorbs/reconcile.rs +++ b/crab/src/optimize/xorbs/reconcile.rs @@ -1136,7 +1136,7 @@ async fn finalize_capsule( }, ) .await?; - let current = view.pointer_catalog()?; + let current = view.pointer_catalog().await?; verify_capsule_sources(¤t, mapping)?; let (shard_hashes, selected_files) = capsule_selection(¤t)?; let plan = build_plan( @@ -2516,7 +2516,7 @@ mod tests { ) .await .unwrap(); - let replacement = view.pointer_catalog().unwrap(); + let replacement = view.pointer_catalog().await.unwrap(); assert!(!replacement.xorbs().contains_key(&source.hash.hex())); assert!(replacement.xorbs().contains_key(&destination_a.hash.hex())); assert!(replacement.xorbs().contains_key(&destination_b.hash.hex())); diff --git a/crab/src/read/mod.rs b/crab/src/read/mod.rs index 93dfa94bd..ee5a83395 100644 --- a/crab/src/read/mod.rs +++ b/crab/src/read/mod.rs @@ -531,7 +531,7 @@ impl Inner { &self.cancel, ) .await?; - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; let file_index_lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( read_layout, diff --git a/crab/src/replication/mod.rs b/crab/src/replication/mod.rs index 84d28482c..22673681f 100644 --- a/crab/src/replication/mod.rs +++ b/crab/src/replication/mod.rs @@ -8122,7 +8122,7 @@ async fn apply_active_active_repair_action( }, ) .await?; - let mut catalog = view.pointer_catalog()?; + let mut catalog = view.pointer_catalog().await?; if let Some(delta) = crab_write::capsule_protocol::coordinated_pointer_catalog_delta( &target_layout, diff --git a/crates/crab-auth-server/src/receive/capsule.rs b/crates/crab-auth-server/src/receive/capsule.rs index ae00187a0..f8bfc37cf 100644 --- a/crates/crab-auth-server/src/receive/capsule.rs +++ b/crates/crab-auth-server/src/receive/capsule.rs @@ -360,7 +360,7 @@ pub(super) async fn verify_capsule_candidate( if prepared_root != plan.base_root_digest { return Err(conflict("capsule base root differs from prepare record")); } - let mut catalog = source_view.pointer_catalog()?; + let mut catalog = source_view.pointer_catalog().await?; if let Some(delta) = candidate.pointer_catalog_delta()? { catalog.apply(&delta)?; } @@ -396,7 +396,7 @@ pub(super) async fn verify_capsule_candidate( } validate_ref_heads(&prepare.view_ref_updates, filtered_view.refs())?; let candidate_delta = candidate.pointer_catalog_delta()?.unwrap_or_default(); - let mut candidate_catalog = filtered_view.pointer_catalog()?; + let mut candidate_catalog = filtered_view.pointer_catalog().await?; candidate_catalog.apply(&candidate_delta)?; let materialized = materialize_capsule_source_push( ctx.router(), @@ -409,7 +409,7 @@ pub(super) async fn verify_capsule_candidate( MAX_PROTECTED_CAPSULE_BYTES, ) .await?; - let source_catalog = source_view.pointer_catalog()?; + let source_catalog = source_view.pointer_catalog().await?; let delta = source_pointer_delta( &source_catalog, &candidate_catalog, diff --git a/crates/crab-auth-server/src/receive/workflow.rs b/crates/crab-auth-server/src/receive/workflow.rs index a4e35ae56..818e2ac31 100644 --- a/crates/crab-auth-server/src/receive/workflow.rs +++ b/crates/crab-auth-server/src/receive/workflow.rs @@ -1699,7 +1699,7 @@ mod tests { .map_err(|error| invalid(error.to_string()))?, lfs_pointer ); - let source_catalog = source_view.pointer_catalog()?; + let source_catalog = source_view.pointer_catalog().await?; assert!( source_catalog .files() diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index d41925bab..6e5a1f603 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -130,18 +130,16 @@ Current file lookup selects the protocol from the root alone: only an absent v2 root permits v1 lookup. A missing v2 dependency remains an error, and failed shared-session initialization can retry after that dependency is repaired. -Layered checkpoint visibility decoding is shared by readers and recovery. -Checkpoint pointers require an explicit format 5 (`CRBCKP05`). Root, history -and ref-head decoding reject retired or missing formats; there is no embedded -checkpoint decoder. The separately released v1 manifest protocol is unchanged. -Ordinal proofs must match the ordered source-catalog digest before becoming a -Git visibility index; the existing full-object encoding preserves the same -caller-supplied Git identity. A missing proof remains distinct from an empty -ref set, so recovery can reject it explicitly. -Checkpoint ordinal decoding validates source/member positions against that -checkpoint's own inventory, before newer frontier sources are appended. Indexed -opening can then avoid decoding the complete proof again for placement hints; -unindexed readers retain the admission map they actually consume. +Layered checkpoint visibility uses authenticated dictionary, ref-closure and +transition pages, shared by readers and recovery. Checkpoint pointers require +explicit format 6 (`CRBCKP06`); retired or missing formats are rejected. No +release tag contains this layered format; the released v1 protocol is unchanged. +Footer roots bind the visibility section, retained-object count and ordered +source-catalog digest. `LayeredVisibilityReader` loads only pages needed for +requested ref membership; dictionary retention never authorizes visibility. +Loaded source/member positions are checked against checkpoint-owned sources, +before newer frontier sources are appended. Explicit bulk materialization +verifies the complete section hash and canonical encoding against captured roots. Footer-only checkpoints have control offset zero when both optional catalog and visibility sections are absent. The storage reader accepts that canonical range while retaining footer-hash, pointer identity, source and size validation. diff --git a/crates/crab-metadata/src/capsule_protocol/history.rs b/crates/crab-metadata/src/capsule_protocol/history.rs index f30f84ad6..96030a2ce 100644 --- a/crates/crab-metadata/src/capsule_protocol/history.rs +++ b/crates/crab-metadata/src/capsule_protocol/history.rs @@ -441,7 +441,7 @@ mod tests { #[test] fn history_rejects_unsupported_checkpoint_formats() { - for format in [0, 1, 2, 3, 4, 6, u32::MAX] { + for format in [0, 1, 2, 3, 4, 5, 7, u32::MAX] { let segment = history_with_unvalidated_pointers(format, 1); let built = HistorySegment::build( segment.checkpoint().clone(), @@ -467,8 +467,8 @@ mod tests { fn history_rejects_implicit_checkpoint_format() { let segment = HistorySegment::build(checkpoint(), None, state()).unwrap(); let body = std::str::from_utf8(segment.bytes()).unwrap(); - assert_eq!(body.matches("\"format\":5,").count(), 1); - let bytes = Bytes::from(body.replace("\"format\":5,", "")); + assert_eq!(body.matches("\"format\":6,").count(), 1); + let bytes = Bytes::from(body.replace("\"format\":6,", "")); let error = HistorySegment::decode(bytes) .expect_err("an implicit retired checkpoint format must fail"); @@ -480,7 +480,8 @@ mod tests { #[test] fn history_rejects_inconsistent_capsule_counts() { for count in [0, 2, u32::MAX] { - let segment = history_with_unvalidated_pointers(5, count); + let segment = + history_with_unvalidated_pointers(super::super::layered::CHECKPOINT_VERSION, count); let mut state = state(); state.capsule_runs = segment.capsule_runs().to_vec(); let built = HistorySegment::build( @@ -518,7 +519,11 @@ mod tests { let layout = crab_storage::StoreLayout::new(store, "history-format".to_owned()); for (format, count, reason) in [ (u32::MAX, 1, "checkpoint format is unsupported"), - (5, 2, "root capsule run descriptor is invalid"), + ( + super::super::layered::CHECKPOINT_VERSION, + 2, + "root capsule run descriptor is invalid", + ), ] { let segment = history_with_unvalidated_pointers(format, count); let pointer = segment.pointer().unwrap(); diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index 87a99750c..0724e0d5c 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -11,8 +11,12 @@ use crate::git_visibility::{ }; use crate::validation::{validate_content_hash, validate_sha1}; -const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP05"; -const CHECKPOINT_VERSION: u32 = 5; +mod visibility_pages; +#[cfg(feature = "storage")] +pub use visibility_pages::LayeredVisibilityReader; + +const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP06"; +pub(super) const CHECKPOINT_VERSION: u32 = 6; const LAYER_MAGIC: &[u8; 8] = b"CRBPKL01"; const LAYER_VERSION: u32 = 1; const TRAILER_BYTES: usize = 8 + 32 + 8; @@ -20,8 +24,6 @@ const MAX_CHECKPOINT_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_LAYER_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_PHYSICAL_SOURCES: usize = 64; const MAX_SOURCE_MEMBERS: usize = 512; -const LAYERED_VISIBILITY_MAGIC: &[u8; 8] = b"CRBVORD1"; -const LAYERED_VISIBILITY_VERSION: u32 = 2; const MAX_LAYERED_VISIBILITY_BYTES: usize = 128 * 1024 * 1024; const MAX_LAYERED_VISIBILITY_REFS: usize = 100_000; const MAX_LAYERED_VISIBILITY_TRANSITIONS: usize = 1_000_000; @@ -74,6 +76,19 @@ impl LayeredObjectMember { } } + fn validate_sources(self, sources: &[PackSourceDescriptor]) -> Result<()> { + if sources + .get(usize::from(self.source_index)) + .and_then(|source| source.members().get(usize::from(self.member_index))) + .is_none() + { + return Err(corrupt( + "layered visibility admission is outside its checkpoint source inventory", + )); + } + Ok(()) + } + /// Return the zero-based source ordinal. #[must_use] pub const fn source_index(self) -> u16 { @@ -178,110 +193,14 @@ impl LayeredVisibilitySnapshot { .map(|index| index.recent_checkpoint_history()) } - /// Encode the proof in its canonical bounded binary representation. + /// Encode the proof as canonical authenticated pages. pub fn encode(&self) -> Result { - self.validate()?; - let mut writer = BinaryWriter::default(); - writer.bytes(LAYERED_VISIBILITY_MAGIC); - writer.u32(LAYERED_VISIBILITY_VERSION); - let digest = blake3::Hash::from_hex(&self.catalog_digest) - .map_err(|_| contract_error("layered visibility catalog digest is invalid"))?; - writer.bytes(digest.as_bytes()); - writer.u64( - u64::try_from(self.objects.len()) - .map_err(|_| contract_error("layered visibility object count overflows"))?, - ); - for object in &self.objects { - writer.bytes(object); - } - match self.member_admission.as_ref() { - Some(admission) => { - writer.bytes(&[1]); - for member in admission { - writer.u16(member.source_index); - writer.u16(member.member_index); - } - } - None => writer.bytes(&[0]), - } - let object_count = self.objects.len(); - encode_closure_map(&mut writer, &self.refs, object_count)?; - encode_transition_map(&mut writer, &self.transitions, object_count)?; - encode_transition_map(&mut writer, &self.incremental_history, object_count)?; - if writer.bytes.len() > MAX_LAYERED_VISIBILITY_BYTES { - return Err(contract_error( - "layered visibility snapshot exceeds its size bound", - )); - } - Ok(Bytes::from(writer.bytes)) + visibility_pages::encode(self) } - /// Decode and validate one canonical compact proof. + /// Decode and validate the complete canonical page representation. pub(crate) fn decode(bytes: &[u8]) -> Result { - if bytes.len() > MAX_LAYERED_VISIBILITY_BYTES { - return Err(corrupt( - "layered visibility snapshot exceeds its size bound", - )); - } - let mut reader = BinaryReader::new(bytes); - if reader.bytes(LAYERED_VISIBILITY_MAGIC.len())? != LAYERED_VISIBILITY_MAGIC { - return Err(corrupt("layered visibility snapshot magic is invalid")); - } - if reader.u32()? != LAYERED_VISIBILITY_VERSION { - return Err(corrupt( - "layered visibility snapshot version is unsupported", - )); - } - let digest = reader.bytes(32)?; - let catalog_digest = encode_hex(digest); - let object_count = reader.u64()?; - let object_count = usize::try_from(object_count) - .map_err(|_| corrupt("layered visibility object count cannot be represented"))?; - if object_count > crate::git_visibility::MAX_GIT_VISIBILITY_OBJECTS as usize { - return Err(corrupt("layered visibility object dictionary is too large")); - } - let mut objects = Vec::with_capacity(object_count); - for _ in 0..object_count { - let bytes = reader.bytes(20)?; - let object = bytes - .try_into() - .map_err(|_| corrupt("layered visibility object ID is truncated"))?; - objects.push(object); - } - let member_admission = match reader.bytes(1)?[0] { - 0 => None, - 1 => { - let mut admission = Vec::with_capacity(object_count); - for _ in 0..object_count { - admission.push(LayeredObjectMember::new(reader.u16()?, reader.u16()?)); - } - Some(admission) - } - _ => { - return Err(corrupt( - "layered visibility member admission flag is invalid", - )); - } - }; - let refs = decode_closure_map(&mut reader, object_count)?; - let transitions = decode_transition_map(&mut reader, object_count)?; - let incremental_history = decode_transition_map(&mut reader, object_count)?; - reader.finish()?; - let snapshot = Self { - catalog_digest, - objects, - member_admission, - refs, - transitions, - incremental_history, - }; - snapshot.validate()?; - if snapshot.encode()?.as_ref() != bytes { - return Err(corrupt("layered visibility snapshot is not canonical")); - } - #[cfg(feature = "storage")] - tracing::debug!(target: "crab_metadata::index_decode", index_format = "checkpoint_visibility", decoded_bytes = bytes.len() as u64); - Ok(snapshot) + visibility_pages::decode(bytes) } /// Restore the materialized visibility index after checking its catalog binding. @@ -391,296 +310,8 @@ fn remap_member_admission( .collect() } -#[derive(Default)] -struct BinaryWriter { - bytes: Vec, -} - -impl BinaryWriter { - fn bytes(&mut self, bytes: &[u8]) { - self.bytes.extend_from_slice(bytes); - } - - fn u32(&mut self, value: u32) { - self.bytes.extend_from_slice(&value.to_be_bytes()); - } - - fn u16(&mut self, value: u16) { - self.bytes.extend_from_slice(&value.to_be_bytes()); - } - - fn u64(&mut self, value: u64) { - self.bytes.extend_from_slice(&value.to_be_bytes()); - } -} - -struct BinaryReader<'a> { - bytes: &'a [u8], - offset: usize, -} - -impl<'a> BinaryReader<'a> { - fn new(bytes: &'a [u8]) -> Self { - Self { bytes, offset: 0 } - } - - fn bytes(&mut self, length: usize) -> Result<&'a [u8]> { - let end = self - .offset - .checked_add(length) - .ok_or_else(|| corrupt("layered visibility reader overflowed"))?; - let bytes = self - .bytes - .get(self.offset..end) - .ok_or_else(|| corrupt("layered visibility snapshot is truncated"))?; - self.offset = end; - Ok(bytes) - } - - fn u32(&mut self) -> Result { - Ok(u32::from_be_bytes(self.bytes(4)?.try_into().map_err( - |_| corrupt("layered visibility integer is truncated"), - )?)) - } - - fn u16(&mut self) -> Result { - Ok(u16::from_be_bytes(self.bytes(2)?.try_into().map_err( - |_| corrupt("layered visibility integer is truncated"), - )?)) - } - - fn u64(&mut self) -> Result { - Ok(u64::from_be_bytes(self.bytes(8)?.try_into().map_err( - |_| corrupt("layered visibility integer is truncated"), - )?)) - } - - fn finish(&self) -> Result<()> { - if self.offset == self.bytes.len() { - Ok(()) - } else { - Err(corrupt("layered visibility snapshot has trailing bytes")) - } - } -} - -fn encode_closure_map( - writer: &mut BinaryWriter, - closures: &BTreeMap>, - object_count: usize, -) -> Result<()> { - writer.u32( - u32::try_from(closures.len()) - .map_err(|_| contract_error("layered visibility ref count overflows"))?, - ); - for (name, positions) in closures { - encode_name(writer, name)?; - encode_positions(writer, positions, object_count)?; - } - Ok(()) -} - -fn decode_closure_map( - reader: &mut BinaryReader<'_>, - object_count: usize, -) -> Result>> { - let count = usize::try_from(reader.u32()?) - .map_err(|_| corrupt("layered visibility ref count cannot be represented"))?; - if count > MAX_LAYERED_VISIBILITY_REFS { - return Err(corrupt("layered visibility ref count exceeds its bound")); - } - let mut closures = BTreeMap::new(); - for _ in 0..count { - let name = decode_name(reader)?; - let positions = decode_positions(reader, object_count)?; - if closures.insert(name, positions).is_some() { - return Err(corrupt("layered visibility proof repeats a ref")); - } - } - Ok(closures) -} - -fn encode_transition_map( - writer: &mut BinaryWriter, - transitions: &BTreeMap>, - object_count: usize, -) -> Result<()> { - writer.u32( - u32::try_from(transitions.len()) - .map_err(|_| contract_error("layered visibility transition ref count overflows"))?, - ); - for (name, entries) in transitions { - encode_name(writer, name)?; - writer.u32( - u32::try_from(entries.len()) - .map_err(|_| contract_error("layered visibility transition count overflows"))?, - ); - for entry in entries { - writer.u32(entry.from_ordinal); - writer.u32(entry.to_ordinal); - encode_positions(writer, &entry.objects, object_count)?; - } - } - Ok(()) -} - -fn decode_transition_map( - reader: &mut BinaryReader<'_>, - object_count: usize, -) -> Result>> { - let ref_count = usize::try_from(reader.u32()?) - .map_err(|_| corrupt("layered visibility transition ref count cannot be represented"))?; - if ref_count > MAX_LAYERED_VISIBILITY_REFS { - return Err(corrupt( - "layered visibility transition ref count exceeds its bound", - )); - } - let mut output = BTreeMap::new(); - for _ in 0..ref_count { - let name = decode_name(reader)?; - let count = usize::try_from(reader.u32()?) - .map_err(|_| corrupt("layered visibility transition count cannot be represented"))?; - if count > MAX_LAYERED_VISIBILITY_TRANSITIONS { - return Err(corrupt( - "layered visibility transition count exceeds its bound", - )); - } - let mut entries = Vec::with_capacity(count); - for _ in 0..count { - entries.push(GitVisibilityOrdinalTransition { - from_ordinal: reader.u32()?, - to_ordinal: reader.u32()?, - objects: decode_positions(reader, object_count)?, - }); - } - if output.insert(name, entries).is_some() { - return Err(corrupt("layered visibility proof repeats transition refs")); - } - } - Ok(output) -} - -fn encode_name(writer: &mut BinaryWriter, name: &str) -> Result<()> { - let bytes = name.as_bytes(); - writer.u32( - u32::try_from(bytes.len()) - .map_err(|_| contract_error("layered visibility ref name is too long"))?, - ); - writer.bytes(bytes); - Ok(()) -} - -fn decode_name(reader: &mut BinaryReader<'_>) -> Result { - let length = usize::try_from(reader.u32()?) - .map_err(|_| corrupt("layered visibility ref name length cannot be represented"))?; - if length == 0 || length > 4 * 1024 { - return Err(corrupt("layered visibility ref name length is invalid")); - } - let name = std::str::from_utf8(reader.bytes(length)?) - .map_err(|_| corrupt("layered visibility ref name is not UTF-8"))?; - validate_visibility_ref(name)?; - Ok(name.to_owned()) -} - -fn encode_positions( - writer: &mut BinaryWriter, - positions: &[u32], - object_count: usize, -) -> Result<()> { - validate_positions_for_encoding(positions)?; - if positions.iter().any(|position| { - usize::try_from(*position) - .ok() - .is_none_or(|position| position >= object_count) - }) { - return Err(contract_error( - "layered visibility position is outside its dictionary", - )); - } - let bitmap_len = object_count.div_ceil(8); - let sparse_bytes = positions - .len() - .checked_mul(4) - .ok_or_else(|| contract_error("layered visibility closure size overflows"))?; - if bitmap_len == 0 || sparse_bytes <= bitmap_len { - writer.bytes(&[0]); - writer.u32( - u32::try_from(positions.len()) - .map_err(|_| contract_error("layered visibility closure count overflows"))?, - ); - for position in positions { - writer.u32(*position); - } - } else { - let mut bitmap = vec![0_u8; bitmap_len]; - for position in positions { - let position = usize::try_from(*position) - .map_err(|_| contract_error("layered visibility position overflows"))?; - bitmap[position / 8] |= 1 << (position % 8); - } - writer.bytes(&[1]); - writer.u32( - u32::try_from(bitmap.len()) - .map_err(|_| contract_error("layered visibility bitmap length overflows"))?, - ); - writer.bytes(&bitmap); - } - Ok(()) -} - -fn decode_positions(reader: &mut BinaryReader<'_>, object_count: usize) -> Result> { - let kind = reader.bytes(1)?[0]; - let count = usize::try_from(reader.u32()?) - .map_err(|_| corrupt("layered visibility closure count cannot be represented"))?; - let positions = match kind { - 0 => { - if count > object_count { - return Err(corrupt("layered visibility sparse closure is too large")); - } - let mut positions = Vec::with_capacity(count); - for _ in 0..count { - positions.push(reader.u32()?); - } - positions - } - 1 => { - if count != object_count.div_ceil(8) { - return Err(corrupt( - "layered visibility bitmap length does not match its dictionary", - )); - } - let bitmap = reader.bytes(count)?; - if let Some(last) = bitmap.last() - && !object_count.is_multiple_of(8) - && last >> (object_count % 8) != 0 - { - return Err(corrupt( - "layered visibility bitmap sets an out-of-range position", - )); - } - let mut positions = Vec::new(); - for (byte_index, byte) in bitmap.iter().enumerate() { - for bit_index in 0..8 { - if byte & (1 << bit_index) != 0 - && let Some(position) = byte_index - .checked_mul(8) - .and_then(|value| value.checked_add(bit_index)) - .and_then(|value| u32::try_from(value).ok()) - { - positions.push(position); - } - } - } - positions - } - _ => return Err(corrupt("layered visibility closure encoding is invalid")), - }; - validate_positions(&positions, object_count)?; - Ok(positions) -} - fn validate_positions(positions: &[u32], object_count: usize) -> Result<()> { - validate_positions_for_encoding(positions)?; + validate_position_order(positions)?; if positions.iter().any(|position| { usize::try_from(*position) .ok() @@ -693,7 +324,7 @@ fn validate_positions(positions: &[u32], object_count: usize) -> Result<()> { Ok(()) } -fn validate_positions_for_encoding(positions: &[u32]) -> Result<()> { +fn validate_position_order(positions: &[u32]) -> Result<()> { if positions.windows(2).any(|pair| pair[0] >= pair[1]) { return Err(corrupt( "layered visibility positions must be sorted and deduplicated", @@ -753,16 +384,6 @@ fn validate_visibility_ref(name: &str) -> Result<()> { Ok(()) } -fn encode_hex(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut output = String::with_capacity(bytes.len() * 2); - for byte in bytes { - output.push(char::from(HEX[usize::from(byte >> 4)])); - output.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - output -} - /// The immutable object containing one bounded pack inventory. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -1311,6 +932,7 @@ struct CheckpointFooter { cold_clone_object_count: Option, #[serde(default)] visibility_transitions: BTreeMap>, + visibility_pages: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -1335,43 +957,19 @@ pub struct LayeredCheckpoint { } impl LayeredCheckpoint { - /// Build a metadata-only checkpoint from immutable pack sources. - pub fn build( - covered_generation: u64, - covered_root_digest: &str, - sources: Vec, - pointer_catalog: PointerCatalog, - visibility: Option, - ) -> Result { - let visibility_transitions = visibility - .as_ref() - .map(crate::capsule_protocol::CapsuleVisibilitySnapshot::recent_transitions) - .transpose()? - .unwrap_or_default(); - let visibility = match visibility { - Some(value) => Some((CapsuleSectionKind::VisibilitySnapshot, value.encode()?)), - None => None, - }; - Self::build_inner( - covered_generation, - covered_root_digest, - sources, - pointer_catalog, - visibility, - None, - None, - visibility_transitions, - ) - } - /// Build a metadata-only checkpoint with the compact ordinal proof. - pub fn build_with_ordinal_visibility( + pub fn build( covered_generation: u64, covered_root_digest: &str, sources: Vec, pointer_catalog: PointerCatalog, visibility: Option, ) -> Result { + if let Some(snapshot) = &visibility { + for member in snapshot.member_admission().unwrap_or_default() { + member.validate_sources(&sources)?; + } + } let visibility_transitions = visibility .as_ref() .map(LayeredVisibilitySnapshot::recent_transitions) @@ -1439,6 +1037,15 @@ impl LayeredCheckpoint { if let Some((kind, visibility)) = visibility { append_section(&mut body, &mut sections, kind, visibility)?; } + let visibility_pages = sections + .iter() + .find(|section| section.kind == CapsuleSectionKind::VisibilityOrdinalSnapshot) + .map(|section| { + visibility_pages::Descriptor::decode( + &body[section.offset as usize..(section.offset + section.length) as usize], + ) + }) + .transpose()?; let control_offset = 0_u64; let footer = CheckpointFooter { version: CHECKPOINT_VERSION, @@ -1450,6 +1057,7 @@ impl LayeredCheckpoint { cold_clone_object_set_digest, cold_clone_object_count, visibility_transitions, + visibility_pages, }; validate_source_inventory(&footer.sources)?; for source in &footer.sources { @@ -1552,6 +1160,34 @@ impl LayeredCheckpoint { }) } + /// Capture lazy visibility roots authenticated by this checkpoint footer. + #[cfg(feature = "storage")] + pub fn visibility_reader( + &self, + layout: &crab_storage::StoreLayout, + maximum: u64, + ) -> Result> { + let Some(descriptor) = &self.footer.visibility_pages else { + return Ok(None); + }; + let section = self + .footer + .sections + .iter() + .find(|section| section.kind == CapsuleSectionKind::VisibilityOrdinalSnapshot) + .ok_or_else(|| corrupt("checkpoint visibility pages have no section"))?; + let range = PackRange::from_parts(section.offset, section.length, section.blake3.clone())?; + LayeredVisibilityReader::new( + layout, + &self.hash, + range, + descriptor.clone(), + self.footer.sources.clone(), + maximum, + ) + .map(Some) + } + /// Return the complete checkpoint bytes. #[must_use] pub fn bytes(&self) -> &Bytes { @@ -1578,7 +1214,7 @@ impl LayeredCheckpoint { /// Verify every identity field of the root's authenticated checkpoint pointer. pub fn matches_pointer(&self, pointer: &super::CheckpointPointer) -> Result { - Ok(pointer.format() == 5 + Ok(pointer.format() == CHECKPOINT_VERSION && self.hash() == pointer.hash() && self.object_size == pointer.size() && self.control_offset() == pointer.control_offset() @@ -1676,38 +1312,22 @@ impl LayeredCheckpoint { PointerCatalog::decode(§ion) } - /// Decode the complete visibility snapshot compacted by this checkpoint. - pub fn visibility_snapshot( - &self, - ) -> Result> { - let Some(section) = self.section(CapsuleSectionKind::VisibilitySnapshot)? else { - return Ok(None); - }; - crate::capsule_protocol::CapsuleVisibilitySnapshot::decode(§ion).map(Some) - } - /// Decode the compact ordinal proof and validate its checkpoint source bindings. pub fn visibility_ordinal_snapshot(&self) -> Result> { let Some(section) = self.section(CapsuleSectionKind::VisibilityOrdinalSnapshot)? else { return Ok(None); }; - let snapshot = LayeredVisibilitySnapshot::decode(§ion)?; - // Admission ordinals belong to this checkpoint, before frontier sources - // are appended. Validating against the combined inventory would let an - // out-of-range ordinal alias an unrelated newer source. - if snapshot.member_admission().is_some_and(|members| { - members.iter().any(|member| { - self.footer - .sources - .get(usize::from(member.source_index())) - .and_then(|source| source.members().get(usize::from(member.member_index()))) - .is_none() - }) - }) { + if Some(&visibility_pages::Descriptor::decode(§ion)?) + != self.footer.visibility_pages.as_ref() + { return Err(corrupt( - "layered visibility admission is outside its checkpoint source inventory", + "checkpoint body roots differ from its authenticated footer", )); } + let snapshot = LayeredVisibilitySnapshot::decode(§ion)?; + for member in snapshot.member_admission().unwrap_or_default() { + member.validate_sources(self.sources())?; + } Ok(Some(snapshot)) } @@ -1728,9 +1348,7 @@ impl LayeredCheckpoint { ) .map(Some); } - self.visibility_snapshot()? - .map(|snapshot| snapshot.to_index(generation, pack_index_hash, git_validation_digest)) - .transpose() + Ok(None) } fn section(&self, kind: CapsuleSectionKind) -> Result> { @@ -1908,8 +1526,37 @@ fn validate_checkpoint_footer( "layered checkpoint cold-clone proof has an incomplete object-set commitment", )); } + let sections = footer + .sections + .iter() + .filter(|section| section.kind == CapsuleSectionKind::VisibilityOrdinalSnapshot) + .collect::>(); + match (&footer.visibility_pages, sections.as_slice()) { + (Some(descriptor), [section]) => { + descriptor.validate(section.length)?; + if descriptor.catalog_digest != source_catalog_digest(&footer.sources)? + || footer.cold_clone_object_count != Some(u64::from(descriptor.object_count)) + { + return Err(corrupt( + "checkpoint visibility roots differ from their captured inventory", + )); + } + } + (None, []) => {} + _ => { + return Err(corrupt( + "checkpoint visibility section and footer roots differ", + )); + } + } let mut expected_offset = 0_u64; for section in &footer.sections { + if !matches!( + section.kind, + CapsuleSectionKind::CatalogDelta | CapsuleSectionKind::VisibilityOrdinalSnapshot + ) { + return Err(corrupt("checkpoint section kind is retired or unsupported")); + } validate_content_hash( §ion.blake3, "layered checkpoint section hash", @@ -2059,42 +1706,39 @@ mod tests { } #[test] - fn layered_visibility_encodings_preserve_ref_membership() { + fn checkpoint_pages_preserve_ref_membership() { let source = PackLayer::build(&pack()) .unwrap() .source_descriptor() .unwrap(); - let refs = BTreeMap::from([("refs/tags/blob".to_owned(), vec!["a".repeat(40)])]); - let index = GitVisibilityIndex::new(0, "1".repeat(64), "2".repeat(64), refs).unwrap(); - let catalog_digest = source_catalog_digest(std::slice::from_ref(&source)).unwrap(); - let ordinal = LayeredVisibilitySnapshot::from_index(&index, &catalog_digest).unwrap(); - let full = crate::capsule_protocol::CapsuleVisibilitySnapshot::from_index(&index).unwrap(); - for checkpoint in [ - LayeredCheckpoint::build_with_ordinal_visibility( - 0, - &"3".repeat(64), - vec![source.clone()], - PointerCatalog::new(), - Some(ordinal), - ) - .unwrap(), - LayeredCheckpoint::build( - 0, - &"3".repeat(64), - vec![source], - PointerCatalog::new(), - Some(full), - ) - .unwrap(), - ] { - let decoded = checkpoint - .visibility_index(1, &"4".repeat(64), &"5".repeat(64)) - .unwrap() - .unwrap(); - assert_eq!(decoded.ref_count(), 1); - assert!(decoded.contains_hex_in_ref("refs/tags/blob", &"a".repeat(40))); - assert!(!decoded.contains_hex_in_ref("refs/tags/blob", &"b".repeat(40))); - } + let index = GitVisibilityIndex::new( + 0, + "1".repeat(64), + "2".repeat(64), + BTreeMap::from([("refs/tags/blob".to_owned(), vec!["a".repeat(40)])]), + ) + .unwrap(); + let digest = source_catalog_digest(std::slice::from_ref(&source)).unwrap(); + let visibility = LayeredVisibilitySnapshot::from_index(&index, &digest).unwrap(); + let checkpoint = LayeredCheckpoint::build( + 0, + &"3".repeat(64), + vec![source], + PointerCatalog::new(), + Some(visibility), + ) + .unwrap(); + let decoded = checkpoint + .visibility_index(1, &"4".repeat(64), &"5".repeat(64)) + .unwrap() + .unwrap(); + assert_eq!( + [ + decoded.contains_hex_in_ref("refs/tags/blob", &"a".repeat(40)), + decoded.contains_hex_in_ref("refs/tags/blob", &"b".repeat(40)) + ], + [true, false] + ); } #[test] @@ -2111,18 +1755,15 @@ mod tests { ) .unwrap(); let visibility = LayeredVisibilitySnapshot::from_index(&index, &"f".repeat(64)).unwrap(); - let checkpoint = LayeredCheckpoint::build_with_ordinal_visibility( - 0, - &"3".repeat(64), - vec![source], - PointerCatalog::new(), - Some(visibility), - ) - .unwrap(); assert!( - checkpoint - .visibility_index(1, &"4".repeat(64), &"5".repeat(64)) - .is_err() + LayeredCheckpoint::build( + 0, + &"3".repeat(64), + vec![source], + PointerCatalog::new(), + Some(visibility) + ) + .is_err() ); } @@ -2349,20 +1990,14 @@ mod tests { vec![member], ) .unwrap(); - let checkpoint = LayeredCheckpoint::build_with_ordinal_visibility( + let checkpoint = LayeredCheckpoint::build( 1, &"3".repeat(64), vec![source.clone()], PointerCatalog::new(), Some(snapshot), - ) - .unwrap(); - assert_eq!( - checkpoint - .visibility_index(1, &"1".repeat(64), &"2".repeat(64)) - .is_ok(), - valid ); + assert_eq!(checkpoint.is_ok(), valid); } } @@ -2403,7 +2038,7 @@ mod tests { ) .unwrap(); let expected_digest = snapshot.object_set_digest(); - let checkpoint = LayeredCheckpoint::build_with_ordinal_visibility( + let checkpoint = LayeredCheckpoint::build( 1, &"4".repeat(64), vec![source], @@ -2459,7 +2094,7 @@ mod tests { let source = layer.source_descriptor().unwrap(); let catalog_digest = source_catalog_digest(std::slice::from_ref(&source)).unwrap(); let snapshot = LayeredVisibilitySnapshot::from_index(&index, &catalog_digest).unwrap(); - let checkpoint = LayeredCheckpoint::build_with_ordinal_visibility( + let checkpoint = LayeredCheckpoint::build( 1, &"3".repeat(64), vec![source], @@ -2534,25 +2169,4 @@ mod tests { ); LayeredVisibilitySnapshot::decode(&snapshot.encode().unwrap()).unwrap(); } - - #[test] - fn ordinal_visibility_is_smaller_than_hex_snapshot_for_shared_history() { - let mut refs = BTreeMap::new(); - refs.insert( - "refs/heads/main".to_owned(), - (0..128) - .map(|value| format!("{value:040x}")) - .collect::>(), - ); - let index = GitVisibilityIndex::new(1, "1".repeat(64), "2".repeat(64), refs).unwrap(); - let full = crate::capsule_protocol::CapsuleVisibilitySnapshot::from_index(&index) - .unwrap() - .encode() - .unwrap(); - let compact = LayeredVisibilitySnapshot::from_index(&index, &"3".repeat(64)) - .unwrap() - .encode() - .unwrap(); - assert!(compact.len() < full.len()); - } } diff --git a/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages.rs b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages.rs new file mode 100644 index 000000000..cb70a7950 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages.rs @@ -0,0 +1,430 @@ +//! One canonical page representation for checkpoint visibility and admission. +use super::*; +use crate::pages::{Kind, PageRef, Writer}; + +#[cfg(feature = "storage")] +mod reader; +#[cfg(feature = "storage")] +pub use reader::LayeredVisibilityReader; +#[cfg(all(test, feature = "storage"))] +mod tests; + +const MAGIC: &[u8; 8] = b"CRBVPG01"; +const MAX_DESCRIPTOR_BYTES: usize = 4096; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Descriptor { + version: u32, + pub(super) catalog_digest: String, + body_bytes: u64, + body_hash: [u8; 32], + pub(super) object_count: u32, + member_admission: bool, + objects: Option, + refs: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RefRow { + name: String, + objects: Option, + recent_present: bool, + recent: Option, + history_present: bool, + history: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct TransitionRow { + from: [u8; 20], + to: [u8; 20], + objects: Option, +} + +fn json(value: &T) -> Result> { + serde_json::to_vec(value).map_err(|source| MetadataError::CapsuleVisibilityRecord { source }) +} + +fn optional_tree( + writer: &mut Writer, + kind: Kind, + rows: crate::pages::Rows, +) -> Result> { + if rows.is_empty() { + return Ok(None); + } + writer.tree(kind, rows).map(Some) +} + +fn object_set( + writer: &mut Writer, + objects: &[[u8; 20]], + positions: &[u32], +) -> Result> { + let rows = positions + .iter() + .map(|position| { + objects + .get(*position as usize) + .map(|oid| (oid.to_vec(), Vec::new())) + .ok_or_else(|| corrupt("checkpoint visibility ordinal is outside its dictionary")) + }) + .collect::>()?; + optional_tree(writer, Kind::CheckpointSets, rows) +} + +fn transitions( + writer: &mut Writer, + objects: &[[u8; 20]], + entries: &[GitVisibilityOrdinalTransition], +) -> Result> { + let mut rows = Vec::new(); + for (position, entry) in entries.iter().enumerate() { + let from = *objects + .get(entry.from_ordinal as usize) + .ok_or_else(|| corrupt("transition origin is absent"))?; + let to = *objects + .get(entry.to_ordinal as usize) + .ok_or_else(|| corrupt("transition target is absent"))?; + let objects = object_set(writer, objects, &entry.objects)?; + rows.push(( + (position as u32).to_be_bytes().to_vec(), + json(&TransitionRow { from, to, objects })?, + )); + } + optional_tree(writer, Kind::CheckpointTransitions, rows) +} + +pub(super) fn encode(snapshot: &LayeredVisibilitySnapshot) -> Result { + snapshot.validate()?; + let mut writer = Writer::default(); + let dictionary = snapshot + .objects + .iter() + .enumerate() + .map(|(index, oid)| { + let value = snapshot + .member_admission + .as_ref() + .map(|members| { + let member = members[index]; + [ + member.source_index.to_be_bytes(), + member.member_index.to_be_bytes(), + ] + .concat() + }) + .unwrap_or_default(); + (oid.to_vec(), value) + }) + .collect(); + let objects = optional_tree(&mut writer, Kind::CheckpointObjects, dictionary)?; + let mut refs = BTreeMap::new(); + for (name, positions) in &snapshot.refs { + let objects = object_set(&mut writer, &snapshot.objects, positions)?; + let recent = transitions( + &mut writer, + &snapshot.objects, + snapshot.transitions.get(name).map_or(&[], Vec::as_slice), + )?; + let history = transitions( + &mut writer, + &snapshot.objects, + snapshot + .incremental_history + .get(name) + .map_or(&[], Vec::as_slice), + )?; + let row = RefRow { + name: name.clone(), + objects, + recent_present: snapshot.transitions.contains_key(name), + recent, + history_present: snapshot.incremental_history.contains_key(name), + history, + }; + if refs + .insert( + blake3::hash(name.as_bytes()).as_bytes().to_vec(), + json(&row)?, + ) + .is_some() + { + return Err(corrupt("checkpoint visibility ref hashes collide")); + } + } + let refs = optional_tree( + &mut writer, + Kind::CheckpointRefs, + refs.into_iter().collect(), + )?; + let descriptor = Descriptor { + version: 1, + catalog_digest: snapshot.catalog_digest.clone(), + body_bytes: writer.bytes.len() as u64, + body_hash: *blake3::hash(&writer.bytes).as_bytes(), + object_count: snapshot.objects.len() as u32, + member_admission: snapshot.member_admission.is_some(), + objects, + refs, + }; + let header = json(&descriptor)?; + if header.len() > MAX_DESCRIPTOR_BYTES { + return Err(corrupt("checkpoint visibility descriptor is oversized")); + } + writer.bytes.extend_from_slice(&header); + crate::pages::put_u32(&mut writer.bytes, header.len())?; + writer.bytes.extend_from_slice(MAGIC); + if writer.bytes.len() > MAX_LAYERED_VISIBILITY_BYTES { + return Err(corrupt( + "checkpoint visibility proof exceeds its byte bound", + )); + } + Ok(writer.bytes.into()) +} + +impl Descriptor { + pub(super) fn decode(bytes: &[u8]) -> Result { + if bytes.len() < 12 + || bytes.len() > MAX_LAYERED_VISIBILITY_BYTES + || &bytes[bytes.len() - 8..] != MAGIC + { + return Err(corrupt("checkpoint visibility page trailer is invalid")); + } + let count = u32::from_le_bytes( + bytes[bytes.len() - 12..bytes.len() - 8] + .try_into() + .map_err(|_| corrupt("visibility descriptor length is truncated"))?, + ) as usize; + let start = bytes + .len() + .checked_sub(12 + count) + .filter(|_| count <= MAX_DESCRIPTOR_BYTES) + .ok_or_else(|| corrupt("visibility descriptor is outside its section"))?; + let descriptor: Self = serde_json::from_slice(&bytes[start..bytes.len() - 12]) + .map_err(|source| MetadataError::CapsuleVisibilityRecord { source })?; + descriptor.validate(bytes.len() as u64)?; + if descriptor.body_bytes != start as u64 { + return Err(corrupt( + "visibility body boundary differs from its descriptor", + )); + } + Ok(descriptor) + } + + pub(super) fn validate(&self, length: u64) -> Result<()> { + validate_content_hash( + &self.catalog_digest, + "checkpoint visibility catalog digest", + "layered checkpoint", + )?; + if self.version != 1 + || self.body_bytes > length.saturating_sub(12) + || length > MAX_LAYERED_VISIBILITY_BYTES as u64 + || u64::from(self.object_count) > crate::git_visibility::MAX_GIT_VISIBILITY_OBJECTS + || self.objects.as_ref().map_or(0, |root| root.count) != self.object_count + { + return Err(corrupt("checkpoint visibility descriptor shape is invalid")); + } + validate_root( + self.objects.as_ref(), + Kind::CheckpointObjects, + self.body_bytes, + u64::from(self.object_count), + )?; + validate_root( + self.refs.as_ref(), + Kind::CheckpointRefs, + self.body_bytes, + MAX_LAYERED_VISIBILITY_REFS as u64, + ) + } +} + +fn validate_root(root: Option<&PageRef>, kind: Kind, bytes: u64, maximum: u64) -> Result<()> { + if let Some(root) = root { + root.validate(kind, bytes)?; + if u64::from(root.count) > maximum { + return Err(corrupt("checkpoint visibility page count is oversized")); + } + if matches!(kind, Kind::CheckpointTransitions) + && (root.first != 0u32.to_be_bytes() || root.last != (root.count - 1).to_be_bytes()) + { + return Err(corrupt("checkpoint transition positions are incomplete")); + } + } + Ok(()) +} + +impl RefRow { + fn decode(key: &[u8], value: &[u8], descriptor: &Descriptor) -> Result { + let row: Self = serde_json::from_slice(value) + .map_err(|source| MetadataError::CapsuleVisibilityRecord { source })?; + validate_visibility_ref(&row.name)?; + if key != blake3::hash(row.name.as_bytes()).as_bytes() + || row.name.len() > 4096 + || (!row.recent_present && row.recent.is_some()) + || (!row.history_present && row.history.is_some()) + { + return Err(corrupt( + "checkpoint ref row differs from its key or coverage", + )); + } + validate_root( + row.objects.as_ref(), + Kind::CheckpointSets, + descriptor.body_bytes, + u64::from(descriptor.object_count), + )?; + validate_root( + row.recent.as_ref(), + Kind::CheckpointTransitions, + descriptor.body_bytes, + 64, + )?; + validate_root( + row.history.as_ref(), + Kind::CheckpointTransitions, + descriptor.body_bytes, + MAX_LAYERED_VISIBILITY_TRANSITIONS as u64, + )?; + Ok(row) + } +} + +fn member(value: &[u8], admitted: bool) -> Result> { + if !admitted && value.is_empty() { + return Ok(None); + } + if !admitted || value.len() != 4 { + return Err(corrupt("checkpoint member row has invalid width")); + } + let source_index = u16::from_be_bytes([value[0], value[1]]); + let member_index = u16::from_be_bytes([value[2], value[3]]); + if usize::from(source_index) >= MAX_PHYSICAL_SOURCES + || usize::from(member_index) >= MAX_SOURCE_MEMBERS + { + return Err(corrupt("checkpoint member row escapes its format bounds")); + } + Ok(Some(LayeredObjectMember::new(source_index, member_index))) +} + +pub(super) fn decode(bytes: &[u8]) -> Result { + let descriptor = Descriptor::decode(bytes)?; + let body = &bytes[..descriptor.body_bytes as usize]; + if blake3::hash(body).as_bytes() != &descriptor.body_hash { + return Err(corrupt("checkpoint visibility body hash differs")); + } + let mut seen = BTreeMap::new(); + let mut scan = |root: Option<&PageRef>, kind| -> Result { + root.map(|root| crate::pages::scan(body, root, kind, "checkpoint visibility", &mut seen)) + .transpose() + .map(Option::unwrap_or_default) + }; + let dictionary = scan(descriptor.objects.as_ref(), Kind::CheckpointObjects)?; + let mut objects = Vec::new(); + let mut admission = descriptor.member_admission.then(Vec::new); + for (oid, value) in dictionary { + objects.push( + oid.try_into() + .map_err(|_| corrupt("checkpoint dictionary OID width is invalid"))?, + ); + if let Some(member) = member(&value, descriptor.member_admission)? + && let Some(members) = admission.as_mut() + { + members.push(member); + } + } + let ordinal = |oid: &[u8; 20]| -> Result { + objects + .binary_search(oid) + .map(|index| index as u32) + .map_err(|_| corrupt("checkpoint closure escapes its retained dictionary")) + }; + let mut refs = BTreeMap::new(); + let mut recent = BTreeMap::new(); + let mut history = BTreeMap::new(); + let ref_rows = scan(descriptor.refs.as_ref(), Kind::CheckpointRefs)?; + for (key, value) in ref_rows { + let row = RefRow::decode(&key, &value, &descriptor)?; + let mut positions = |root: Option<&PageRef>| -> Result> { + scan(root, Kind::CheckpointSets)? + .into_iter() + .map(|(oid, value)| { + if !value.is_empty() { + return Err(corrupt("checkpoint closure has unexpected payload")); + } + ordinal( + &oid.try_into() + .map_err(|_| corrupt("checkpoint closure OID width is invalid"))?, + ) + }) + .collect() + }; + let closure = positions(row.objects.as_ref())?; + let mut read_history = + |root: Option<&PageRef>| -> Result> { + let rows = scan(root, Kind::CheckpointTransitions)?; + rows.into_iter() + .enumerate() + .map(|(index, (key, value))| { + if key != (index as u32).to_be_bytes() { + return Err(corrupt("checkpoint transition position is invalid")); + } + let transition: TransitionRow = serde_json::from_slice(&value) + .map_err(|source| MetadataError::CapsuleVisibilityRecord { source })?; + validate_root( + transition.objects.as_ref(), + Kind::CheckpointSets, + descriptor.body_bytes, + u64::from(descriptor.object_count), + )?; + let objects = scan(transition.objects.as_ref(), Kind::CheckpointSets)? + .into_iter() + .map(|(oid, value)| { + if !value.is_empty() { + return Err(corrupt("transition set has unexpected payload")); + } + ordinal( + &oid.try_into() + .map_err(|_| corrupt("transition OID width is invalid"))?, + ) + }) + .collect::>()?; + Ok(GitVisibilityOrdinalTransition { + from_ordinal: ordinal(&transition.from)?, + to_ordinal: ordinal(&transition.to)?, + objects, + }) + }) + .collect() + }; + if row.recent_present { + recent.insert(row.name.clone(), read_history(row.recent.as_ref())?); + } + if row.history_present { + history.insert(row.name.clone(), read_history(row.history.as_ref())?); + } + if refs.insert(row.name, closure).is_some() { + return Err(corrupt("checkpoint visibility repeats a ref")); + } + } + let snapshot = LayeredVisibilitySnapshot { + catalog_digest: descriptor.catalog_digest, + objects, + member_admission: admission, + refs, + transitions: recent, + incremental_history: history, + }; + snapshot.validate()?; + if snapshot.encode()?.as_ref() != bytes { + return Err(corrupt("checkpoint visibility pages are not canonical")); + } + #[cfg(feature = "storage")] + tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility", decoded_bytes=bytes.len() as u64); + Ok(snapshot) +} diff --git a/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs new file mode 100644 index 000000000..0f545b847 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs @@ -0,0 +1,249 @@ +//! Bounded reads from roots authenticated by the captured checkpoint footer. +use super::*; +use crate::pages::{PageCache, PageWork}; +use crab_storage::{Store, StoreLayout}; +use std::sync::{Arc, OnceLock}; + +/// Captured checkpoint dictionary and ref-closure pages; retention never grants visibility. +#[derive(Debug)] +pub struct LayeredVisibilityReader { + path: object_store::path::Path, + range: PackRange, + descriptor: Descriptor, + sources: Vec, + maximum: u64, + cache: PageCache, + complete: OnceLock>, +} + +impl LayeredVisibilityReader { + pub(in crate::capsule_protocol::layered) fn new( + layout: &StoreLayout, + hash: &str, + range: PackRange, + descriptor: Descriptor, + sources: Vec, + maximum: u64, + ) -> Result { + descriptor.validate(range.length())?; + range.end()?; + if descriptor.catalog_digest != source_catalog_digest(&sources)? { + return Err(corrupt( + "checkpoint visibility pages name a different source catalog", + )); + } + Ok(Self { + path: layout.capsule_checkpoint_path(hash), + range, + descriptor, + sources, + maximum, + cache: PageCache::default(), + complete: OnceLock::new(), + }) + } + + /// Return the retained dictionary count, independently of ref visibility. + #[must_use] + pub fn object_count(&self) -> u64 { + u64::from(self.descriptor.object_count) + } + + fn work(&self) -> PageWork { + PageWork { + maximum: self.maximum, + bytes: 0, + seen: BTreeSet::new(), + } + } + + async fn rows( + &self, + store: &Store, + root: Option<&PageRef>, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + let Some(root) = root else { + return Ok(vec![None; keys.len()]); + }; + let end = self + .range + .offset() + .checked_add(self.descriptor.body_bytes) + .ok_or_else(|| corrupt("checkpoint visibility region overflows"))?; + self.cache + .lookup( + store, + &self.path, + 0, + self.range.offset()..end, + root, + kind, + keys, + work, + ) + .await + } + + async fn ref_rows( + &self, + store: &Store, + names: &[&str], + work: &mut PageWork, + ) -> Result>> { + for name in names { + validate_visibility_ref(name)?; + } + let keys = names + .iter() + .map(|name| blake3::hash(name.as_bytes()).as_bytes().to_vec()) + .collect::>(); + self.rows( + store, + self.descriptor.refs.as_ref(), + Kind::CheckpointRefs, + &keys, + work, + ) + .await? + .into_iter() + .zip(keys) + .zip(names) + .map(|((value, key), name)| { + let row = value + .map(|value| RefRow::decode(&key, &value, &self.descriptor)) + .transpose()?; + if row.as_ref().is_some_and(|row| row.name != *name) { + return Err(corrupt("checkpoint ref name hash collides")); + } + Ok(row) + }) + .collect() + } + + /// Return authenticated ref presence without loading closures or transitions. + pub async fn refs_present(&self, store: &Store, names: &[&str]) -> Result> { + Ok(self + .ref_rows(store, names, &mut self.work()) + .await? + .into_iter() + .map(|row| row.is_some()) + .collect()) + } + + async fn dictionary( + &self, + store: &Store, + oids: &[[u8; 20]], + work: &mut PageWork, + ) -> Result> { + let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + self.rows( + store, + self.descriptor.objects.as_ref(), + Kind::CheckpointObjects, + &keys, + work, + ) + .await? + .into_iter() + .map(|value| { + let Some(value) = value else { + return Ok(false); + }; + if let Some(member) = member(&value, self.descriptor.member_admission)? { + member.validate_sources(&self.sources)?; + } + Ok(true) + }) + .collect() + } + + /// Inspect retained object presence; these results do not authorize a ref read. + pub async fn dictionary_contains(&self, store: &Store, oids: &[[u8; 20]]) -> Result> { + self.dictionary(store, oids, &mut self.work()).await + } + + /// Inspect one actual ref closure, preserving requested order and absent-ref distinction. + pub async fn contains_in_ref( + &self, + store: &Store, + name: &str, + oids: &[[u8; 20]], + ) -> Result>> { + let mut work = self.work(); + let Some(row) = self + .ref_rows(store, &[name], &mut work) + .await? + .pop() + .flatten() + else { + return Ok(None); + }; + let keys = oids.iter().map(|oid| oid.to_vec()).collect::>(); + let present = self + .rows( + store, + row.objects.as_ref(), + Kind::CheckpointSets, + &keys, + &mut work, + ) + .await? + .into_iter() + .map(|value| { + if value.as_ref().is_some_and(|value| !value.is_empty()) { + return Err(corrupt("checkpoint closure row has unexpected payload")); + } + Ok(value.is_some()) + }) + .collect::>>()?; + // Loaded closure hits must also belong to the retained dictionary. The + // dictionary alone cannot grant membership in this or another ref. + let hits = oids + .iter() + .zip(&present) + .filter_map(|(oid, present)| present.then_some(*oid)) + .collect::>(); + if self + .dictionary(store, &hits, &mut work) + .await? + .iter() + .any(|present| !present) + { + return Err(corrupt( + "checkpoint closure escapes its retained dictionary", + )); + } + Ok(Some(present)) + } + + /// Authenticate the whole captured section for explicit bulk inspection. + pub async fn materialize(&self, store: &Store) -> Result> { + if let Some(snapshot) = self.complete.get() { + return Ok(snapshot.clone()); + } + if self.range.length() > self.maximum { + return Err(corrupt( + "complete checkpoint visibility exceeds its byte budget", + )); + } + let bytes = store + .range_get(&self.path, self.range.offset()..self.range.end()?) + .await?; + if blake3::hash(&bytes).to_hex().as_str() != self.range.blake3() + || Descriptor::decode(&bytes)? != self.descriptor + { + return Err(corrupt( + "checkpoint visibility section differs from captured roots", + )); + } + let snapshot = Arc::new(LayeredVisibilitySnapshot::decode(&bytes)?); + for member in snapshot.member_admission().unwrap_or_default() { + member.validate_sources(&self.sources)?; + } + Ok(self.complete.get_or_init(|| snapshot).clone()) + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/tests.rs b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/tests.rs new file mode 100644 index 000000000..7d7a070af --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/tests.rs @@ -0,0 +1,183 @@ +use super::*; +use crab_storage::{Store, StoreLayout}; +use object_store::memory::InMemory; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +fn object(value: u32) -> [u8; 20] { + let mut oid = [0; 20]; + oid[16..].copy_from_slice(&value.to_be_bytes()); + oid +} + +async fn fixture( + count: u32, + maximum: u64, +) -> ( + StoreLayout, + LayeredCheckpoint, + LayeredVisibilityReader, + Arc, +) { + let mut index = GitVisibilityIndex::new( + 1, + "1".repeat(64), + "2".repeat(64), + BTreeMap::from([ + ( + "refs/heads/main".to_owned(), + (1..=count).map(|value| format!("{value:040x}")).collect(), + ), + ( + "refs/heads/removed".to_owned(), + vec![format!("{:040x}", count + 1)], + ), + ]), + ) + .unwrap(); + index.remove_ref("refs/heads/removed"); + let pack = CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "1".repeat(40), + u64::from(count) + 1, + ) + .unwrap(); + let source = PackLayer::build(&pack) + .unwrap() + .source_descriptor() + .unwrap(); + let digest = source_catalog_digest(std::slice::from_ref(&source)).unwrap(); + let visibility = LayeredVisibilitySnapshot::from_index_with_member_admission( + &index, + &digest, + vec![LayeredObjectMember::new(0, 0); count as usize + 1], + ) + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + 1, + &"3".repeat(64), + vec![source], + PointerCatalog::new(), + Some(visibility), + ) + .unwrap(); + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let store = + Store::new(Arc::new(InMemory::new())).with_read_request_observer(Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + })); + let layout = StoreLayout::new(store, "checkpoint-pages".to_owned()); + layout + .store() + .put_overwrite( + &layout.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + let control = LayeredCheckpoint::decode_control( + checkpoint + .bytes() + .slice(checkpoint.control_offset() as usize..), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.hash(), + checkpoint.footer_hash(), + ) + .unwrap(); + let reader = control + .visibility_reader(&layout, maximum) + .unwrap() + .unwrap(); + (layout, checkpoint, reader, reads) +} + +#[tokio::test] +async fn checkpoint_membership_is_bounded_and_retained_objects_do_not_grant_visibility() { + let (layout, checkpoint, reader, _) = fixture(50_000, 128 * 1024).await; + assert!(checkpoint.bytes().len() > 2 * 1024 * 1024); + let requested = [object(42), object(50_001), object(99_999), object(42)]; + assert_eq!( + reader + .contains_in_ref(layout.store(), "refs/heads/main", &requested) + .await + .unwrap(), + Some(vec![true, false, false, true]) + ); + assert!( + reader + .dictionary_contains(layout.store(), &[object(50_001)]) + .await + .unwrap()[0] + ); + assert!( + reader + .contains_in_ref(layout.store(), "refs/heads/removed", &requested) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn warm_checkpoint_membership_rereads_no_origin_pages() { + let (layout, _, reader, reads) = fixture(50_000, 128 * 1024).await; + let requested = [object(42), object(50_000)]; + reader + .contains_in_ref(layout.store(), "refs/heads/main", &requested) + .await + .unwrap(); + let before = reads.load(Ordering::SeqCst); + reader + .contains_in_ref(layout.store(), "refs/heads/main", &requested) + .await + .unwrap(); + assert_eq!(reads.load(Ordering::SeqCst), before); +} + +#[tokio::test] +async fn bulk_checkpoint_visibility_matches_the_captured_complete_proof() { + let (layout, checkpoint, reader, _) = fixture(1000, 128 * 1024 * 1024).await; + assert_eq!( + *reader.materialize(layout.store()).await.unwrap(), + checkpoint.visibility_ordinal_snapshot().unwrap().unwrap() + ); +} + +#[tokio::test] +async fn checkpoint_page_tampering_and_missing_pages_fail_instead_of_membership_misses() { + for missing in [false, true] { + let (layout, checkpoint, reader, _) = fixture(1000, 128 * 1024).await; + let descriptor = checkpoint.footer.visibility_pages.as_ref().unwrap(); + let section = checkpoint + .footer + .sections + .iter() + .find(|section| section.kind == CapsuleSectionKind::VisibilityOrdinalSnapshot) + .unwrap(); + let path = layout.capsule_checkpoint_path(checkpoint.hash()); + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + let mut bytes = checkpoint.bytes().to_vec(); + bytes[(section.offset + descriptor.objects.as_ref().unwrap().offset) as usize] ^= 1; + layout + .store() + .put_overwrite(&path, bytes.into()) + .await + .unwrap(); + } + assert!( + reader + .dictionary_contains(layout.store(), &[object(42)]) + .await + .is_err() + ); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 680b807e9..1e6750fa0 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -29,6 +29,8 @@ pub use history::{ HistorySegment, HistorySegmentPointer, HistorySegmentState, MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, }; +#[cfg(feature = "storage")] +pub use layered::LayeredVisibilityReader; pub use layered::{ LayeredCheckpoint, LayeredObjectMember, LayeredVisibilitySnapshot, PackLayer, PackMemberDescriptor, PackRange, PackSourceDescriptor, PackSourceKind, source_catalog_digest, @@ -69,7 +71,7 @@ pub use transaction::{CapsuleRefEdit, CapsuleTransaction}; pub use transaction_record::{ CapsuleTransactionRecord, CapsuleTransactionStatus, MAX_CAPSULE_TRANSACTION_RECORD_BYTES, }; -pub use visibility::{CapsuleVisibilityDelta, CapsuleVisibilitySnapshot}; +pub use visibility::CapsuleVisibilityDelta; #[cfg(feature = "storage")] pub use visibility::{CapsuleVisibilityReader, VisibilityChange, VisibilityEditHeader}; diff --git a/crates/crab-metadata/src/capsule_protocol/root.rs b/crates/crab-metadata/src/capsule_protocol/root.rs index 81230c5f6..71484ac45 100644 --- a/crates/crab-metadata/src/capsule_protocol/root.rs +++ b/crates/crab-metadata/src/capsule_protocol/root.rs @@ -187,7 +187,7 @@ impl CheckpointPointer { object_count: u64, ) -> Result { let pointer = Self { - format: 5, + format: super::layered::CHECKPOINT_VERSION, hash: hash.into(), size, control_offset, @@ -926,7 +926,7 @@ pub(super) fn validate_capsule_pointer(pointer: &CapsulePointer) -> Result<()> { } pub(super) fn validate_checkpoint_pointer(pointer: &CheckpointPointer) -> Result<()> { - if pointer.format != 5 { + if pointer.format != super::layered::CHECKPOINT_VERSION { return Err(contract_error("checkpoint format is unsupported")); } validate_content_hash( @@ -1330,9 +1330,9 @@ mod tests { fn root_rejects_retired_or_implicit_checkpoint_formats() { let record = checkpointed_record(); let payload = serde_json::to_string(record.root()).unwrap(); - assert_eq!(payload.matches("\"format\":5,").count(), 1); - for replacement in ["", "\"format\":3,", "\"format\":4,"] { - let payload = payload.replace("\"format\":5,", replacement); + assert_eq!(payload.matches("\"format\":6,").count(), 1); + for replacement in ["", "\"format\":3,", "\"format\":4,", "\"format\":5,"] { + let payload = payload.replace("\"format\":6,", replacement); // Recompute the envelope digest so only format admission can fail. let mut bytes = ROOT_MAGIC.to_vec(); bytes.extend_from_slice(&ROOT_VERSION.to_be_bytes()); diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index 5f6725c38..5bd2de359 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -299,7 +299,7 @@ pub async fn load_layered_checkpoint( router: &StoreLayout, pointer: &super::CheckpointPointer, ) -> Result { - if pointer.format() != 5 { + if pointer.format() != super::layered::CHECKPOINT_VERSION { return Err(corrupt( &router.capsule_checkpoint_path(pointer.hash()), "checkpoint pointer does not name the layered checkpoint format", @@ -325,7 +325,7 @@ pub async fn load_layered_checkpoint_control( router: &StoreLayout, pointer: &super::CheckpointPointer, ) -> Result { - if pointer.format() != 5 { + if pointer.format() != super::layered::CHECKPOINT_VERSION { return Err(corrupt( &router.capsule_checkpoint_path(pointer.hash()), "checkpoint pointer does not name the layered checkpoint format", diff --git a/crates/crab-metadata/src/capsule_protocol/visibility.rs b/crates/crab-metadata/src/capsule_protocol/visibility.rs index b588213e4..f9df73110 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility.rs @@ -4,9 +4,7 @@ use bytes::Bytes; use serde::{Deserialize, Serialize}; use crate::error::{MetadataError, Result}; -use crate::git_visibility::{ - GitVisibilityCheckpointTransition, GitVisibilityEdit, GitVisibilityIndex, -}; +use crate::git_visibility::GitVisibilityEdit; use super::valid_ref_name; @@ -16,7 +14,6 @@ mod paged; pub(super) use paged::Descriptor as VisibilityPages; #[cfg(feature = "storage")] pub use paged::{CapsuleVisibilityReader, VisibilityChange, VisibilityEditHeader}; -const VISIBILITY_SNAPSHOT_VERSION: u32 = 3; /// Ref-keyed reachability changes authenticated by one publication capsule. #[derive(Debug, Clone, PartialEq, Eq)] @@ -68,96 +65,6 @@ impl CapsuleVisibilityDelta { } } -/// Complete ref reachability state compacted into a checkpoint. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct CapsuleVisibilitySnapshot { - version: u32, - refs: BTreeMap>, - incremental_history: BTreeMap>, -} - -impl CapsuleVisibilitySnapshot { - /// Capture complete ref closures independently of a particular pack layout. - pub fn from_index(index: &GitVisibilityIndex) -> Result { - index.validate()?; - let snapshot = Self { - version: VISIBILITY_SNAPSHOT_VERSION, - refs: index.ref_closures(), - incremental_history: index.checkpoint_history(), - }; - snapshot.validate()?; - Ok(snapshot) - } - - /// Encode this complete checkpoint state. - pub fn encode(&self) -> Result { - self.validate()?; - serde_json::to_vec(self).map(Bytes::from).map_err(|source| { - MetadataError::Internal(format!( - "capsule visibility snapshot serialization failed: {source}" - )) - }) - } - - pub(crate) fn decode(bytes: &[u8]) -> Result { - let snapshot: Self = - serde_json::from_slice(bytes).map_err(|source| MetadataError::CorruptObject { - path: "capsule-protocol visibility snapshot".to_owned(), - reason: format!("visibility snapshot is invalid JSON: {source}"), - })?; - snapshot.validate().map_err(as_corruption)?; - if snapshot.encode()?.as_ref() != bytes { - return Err(corrupt("visibility snapshot is not canonically encoded")); - } - Ok(snapshot) - } - - /// Return the complete ref-keyed object closures. - #[must_use] - pub fn refs(&self) -> &BTreeMap> { - &self.refs - } - - /// Return the bounded recent transition suffix for control-only fetches. - /// - /// The complete history remains in the snapshot body. This suffix is a - /// performance hint authenticated by the enclosing checkpoint; readers - /// fall back to the complete visibility proof when a requested have is - /// older than the retained links. - pub fn recent_transitions( - &self, - ) -> Result>> { - self.to_index(0, &"0".repeat(64), &"0".repeat(64)) - .map(|index| index.recent_checkpoint_history()) - } - - /// Restore the checkpoint proof under its current pack identity. - pub fn to_index( - &self, - generation: u64, - pack_index_hash: &str, - git_validation_digest: &str, - ) -> Result { - let mut index = GitVisibilityIndex::new( - generation, - pack_index_hash, - git_validation_digest, - self.refs.clone(), - )?; - index.restore_checkpoint_history(&self.incremental_history)?; - Ok(index) - } - - fn validate(&self) -> Result<()> { - if self.version != VISIBILITY_SNAPSHOT_VERSION { - return Err(contract_error("visibility snapshot version is unsupported")); - } - self.to_index(0, &"0".repeat(64), &"0".repeat(64))?; - Ok(()) - } -} - fn as_corruption(error: MetadataError) -> MetadataError { corrupt(error.to_string()) } @@ -214,55 +121,4 @@ mod tests { assert!(error.to_string().contains("invalid ref name")); } - - #[test] - fn visibility_snapshot_preserves_complete_ref_closures() { - let tip = oid('a'); - let refs = BTreeMap::from([("refs/heads/main".to_owned(), vec![tip.clone(), oid('b')])]); - let index = GitVisibilityIndex::new(7, "1".repeat(64), "2".repeat(64), refs.clone()) - .expect("valid visibility index"); - let snapshot = CapsuleVisibilitySnapshot::from_index(&index).expect("visibility snapshot"); - - let encoded = snapshot.encode().expect("encode visibility snapshot"); - let decoded = CapsuleVisibilitySnapshot::decode(&encoded).expect("decode snapshot"); - - assert_eq!(decoded.refs(), &refs); - } - - #[test] - fn visibility_snapshot_preserves_incremental_fetch_history() { - let old_tip = oid('a'); - let new_tip = oid('b'); - let added = oid('c'); - let mut index = GitVisibilityIndex::new( - 7, - "1".repeat(64), - "2".repeat(64), - BTreeMap::from([("refs/heads/main".to_owned(), vec![old_tip.clone()])]), - ) - .expect("valid visibility index"); - index - .apply_ref_edit( - "refs/heads/main".to_owned(), - &GitVisibilityEdit::from_delta_objects( - Some(old_tip.clone()), - new_tip.clone(), - vec![added.clone(), new_tip.clone()], - Vec::new(), - ), - ) - .expect("apply visibility edit"); - let snapshot = CapsuleVisibilitySnapshot::from_index(&index).expect("visibility snapshot"); - let decoded = CapsuleVisibilitySnapshot::decode(&snapshot.encode().unwrap()).unwrap(); - let restored = decoded - .to_index(8, &"3".repeat(64), &"4".repeat(64)) - .expect("restore visibility snapshot"); - let old_tip = [0xaa; 20]; - let new_tip = [0xbb; 20]; - - assert_eq!( - restored.incremental_objects("refs/heads/main", &new_tip, &[old_tip]), - Some(vec![[0xbb; 20], [0xcc; 20]]) - ); - } } diff --git a/crates/crab-metadata/src/git_visibility.rs b/crates/crab-metadata/src/git_visibility.rs index 872282715..1d9ff13e4 100644 --- a/crates/crab-metadata/src/git_visibility.rs +++ b/crates/crab-metadata/src/git_visibility.rs @@ -1558,38 +1558,6 @@ impl GitVisibilityIndex { Ok(index) } - pub(crate) fn restore_checkpoint_history( - &mut self, - history: &BTreeMap>, - ) -> Result<()> { - let mut restored = BTreeMap::new(); - for (name, transitions) in history { - let mut restored_transitions = Vec::with_capacity(transitions.len()); - for transition in transitions { - validate_sorted_oids(&transition.objects, "checkpoint history")?; - let positions = transition - .objects - .iter() - .map(|oid| { - let oid = decode_oid(oid)?; - self.positions - .get(&oid) - .copied() - .ok_or_else(|| corrupt("checkpoint history object is absent")) - }) - .collect::>>()?; - restored_transitions.push(GitVisibilityTransition { - from_oid: decode_oid(&transition.from_oid)?, - to_oid: decode_oid(&transition.to_oid)?, - objects: GitVisibilityClosure::from_positions(positions, self.objects.len())?, - }); - } - restored.insert(name.clone(), restored_transitions); - } - self.incremental_history = restored; - self.validate() - } - /// Remove one ref and every transition whose authority depends on it. pub fn remove_ref(&mut self, name: &str) { self.refs.remove(name); diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 5047dbd62..4e4a7d2c5 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -59,6 +59,10 @@ pub(crate) enum Kind { RunAdmissionCounts = 13, VisibilityRefs = 14, VisibilityChanges = 15, + CheckpointObjects = 16, + CheckpointTransitions = 17, + CheckpointRefs = 18, + CheckpointSets = 19, } impl Kind { @@ -68,7 +72,8 @@ impl Kind { | Self::Nodes | Self::CommitRecords | Self::CommitAncestryRoots - | Self::RunAdmissionCounts => true, + | Self::RunAdmissionCounts + | Self::CheckpointTransitions => true, #[cfg(feature = "remote-index")] Self::PackSlots => true, _ => false, @@ -77,8 +82,15 @@ impl Kind { pub(crate) fn key_bytes(self) -> usize { match self { - Self::Names | Self::CommitAncestryNodes | Self::VisibilityRefs => 32, - Self::CommitOids | Self::RunAdmission | Self::VisibilityChanges => 20, + Self::Names + | Self::CommitAncestryNodes + | Self::VisibilityRefs + | Self::CheckpointRefs => 32, + Self::CommitOids + | Self::RunAdmission + | Self::VisibilityChanges + | Self::CheckpointObjects + | Self::CheckpointSets => 20, #[cfg(feature = "remote-index")] Self::PackInventory => 32, #[cfg(feature = "remote-index")] @@ -345,6 +357,7 @@ pub(crate) fn decode( Kind::ObjectLocators | Kind::PackOffsets | Kind::PackInventory | Kind::PackSlots => "git_object_index", Kind::RunAdmission | Kind::RunAdmissionCounts => "capsule_admission", Kind::VisibilityRefs | Kind::VisibilityChanges => "capsule_visibility", + Kind::CheckpointObjects | Kind::CheckpointRefs | Kind::CheckpointSets | Kind::CheckpointTransitions => "checkpoint_visibility_pages", _ => "path_state", }, decoded_bytes=bytes.len() as u64, "index page decoded"); diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 29aa1425f..db66ae725 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -440,3 +440,12 @@ The shared runtime defaults to 128 MiB of decoded-output admission. Compressed fetch buffers and cache I/O coexist with that output, so this is not a total RSS cap. Explicit caller buffer budgets remain supported. Large-file qualification tracks physical memory separately from this admission limit. + +## Captured capsule catalogs and visibility + +Git views capture footer-rooted checkpoint visibility readers and load requested +membership pages separately from retained-object presence. Complete visibility +inspection is an explicit async operation. `CapsuleRepositoryView::pointer_catalog` +is also async: hydration, publication and maintenance materialize the complete +catalog from the captured checkpoint when its body was omitted during opening. +Catalog indexing and bounded historical Xet lookup remain separate work. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 02aa7cec1..8dfb7066a 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -76,10 +76,11 @@ pub struct CapsuleRepositoryView { root: crab_metadata::capsule_protocol::RootSnapshot, browse_indexes: Option, checkpoint: Option, + checkpoint_visibility: Option>, capsules: Vec, capsule_controls: Vec, tip_bound_transitions: CapsuleTipBoundTransitions, - visibility_store: Option, + metadata_layout: Option>, tip_bound_controls: Vec, refs: BTreeMap, peeled_refs: BTreeMap, @@ -365,11 +366,11 @@ impl CapsuleRepositoryView { /// must not use a partial chain as an authorization proof. pub async fn tip_bound_transitions(&self) -> Result { let mut transitions = self.tip_bound_transitions.clone(); - if let Some(store) = &self.visibility_store { + if let Some(layout) = &self.metadata_layout { let controls = try_join_all( self.tip_bound_controls .iter() - .map(|control| control.materialize_visibility(store)), + .map(|control| control.materialize_visibility(layout.store())), ) .await?; for (name, values) in build_tip_bound_transitions(&[], &controls)? { @@ -738,11 +739,24 @@ impl CapsuleRepositoryView { } /// Materialize the complete generation-pinned external pointer catalog. - pub fn pointer_catalog(&self) -> Result { - let mut catalog = match self.checkpoint.as_ref() { - Some(checkpoint) => checkpoint.pointer_catalog()?, - None => PointerCatalog::new(), - }; + pub async fn pointer_catalog(&self) -> Result { + let mut catalog = + match self.checkpoint.as_ref() { + Some(checkpoint) if checkpoint.is_control_only() => { + let layout = self.metadata_layout.as_ref().ok_or_else(|| { + ReadError::internal("catalog materialization has no captured origin") + })?; + let pointer = + self.root.record().root().checkpoint().ok_or_else(|| { + ReadError::internal("captured checkpoint has no pointer") + })?; + crab_metadata::capsule_protocol::load_layered_checkpoint(layout, pointer) + .await? + .pointer_catalog()? + } + Some(checkpoint) => checkpoint.pointer_catalog()?, + None => PointerCatalog::new(), + }; for capsule in &self.capsules { if let Some(delta) = capsule.pointer_catalog_delta()? { catalog.apply(&delta)?; @@ -1239,14 +1253,25 @@ impl CapsuleRepositoryView { pub async fn git_visibility_index( &self, ) -> Result { - let mut index = visibility_index_for_checkpoint(self.checkpoint.as_ref())?; + let mut index = match (&self.checkpoint_visibility, &self.metadata_layout) { + (Some(reader), Some(layout)) => { + let snapshot = reader.materialize(layout.store()).await?; + snapshot.to_index( + 0, + &"0".repeat(64), + &"0".repeat(64), + snapshot.catalog_digest(), + )? + } + _ => visibility_index_for_checkpoint(self.checkpoint.as_ref())?, + }; for capsule in &self.capsules { apply_capsule_visibility_index(capsule, &mut index)?; } for capsule in &self.capsule_controls { - let complete = match &self.visibility_store { - Some(store) => capsule.materialize_visibility(store).await?, + let complete = match &self.metadata_layout { + Some(layout) => capsule.materialize_visibility(layout.store()).await?, None => capsule.clone(), }; apply_capsule_control_visibility_index(&complete, &mut index)?; @@ -1544,7 +1569,7 @@ pub async fn verify_reachable_dependencies( if cancellation.is_cancelled() { return Err(ReadError::Cancelled); } - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; let mut sources = BTreeMap::new(); for source in view .layered_checkpoint() @@ -4255,10 +4280,11 @@ pub fn compacted_view_from_checkpoint( root: snapshot, browse_indexes: None, checkpoint: Some(checkpoint), + checkpoint_visibility: None, capsules: Vec::new(), capsule_controls: Vec::new(), tip_bound_transitions, - visibility_store: None, + metadata_layout: None, tip_bound_controls: Vec::new(), visible_ref_transactions: BTreeMap::new(), ref_capsule_counts: BTreeMap::new(), @@ -4567,10 +4593,11 @@ async fn assemble_view( root: snapshot, browse_indexes: None, checkpoint, + checkpoint_visibility: None, capsules: root_capsules, capsule_controls: Vec::new(), tip_bound_transitions, - visibility_store: None, + metadata_layout: None, tip_bound_controls: Vec::new(), refs, peeled_refs, @@ -4605,7 +4632,7 @@ async fn assemble_layered_control_view( admit_frontier(&pointers, limits)?; let checkpoint = match snapshot.record().root().checkpoint() { Some(pointer) => { - let checkpoint = if footer_only { + let checkpoint = if !matches!(read, ControlRead::Complete) { load_layered_checkpoint_control(router, pointer, limits).await? } else { load_layered_checkpoint(router, pointer, limits).await? @@ -4815,10 +4842,11 @@ async fn assemble_layered_control_view( root: snapshot, browse_indexes: None, checkpoint, + checkpoint_visibility: None, capsules: Vec::new(), capsule_controls: Vec::new(), tip_bound_transitions, - visibility_store: None, + metadata_layout: None, tip_bound_controls: Vec::new(), refs: expected_refs, peeled_refs: expected_peeled, @@ -4854,15 +4882,36 @@ async fn assemble_layered_control_view( } } } - let initial = visibility_index_for_checkpoint(checkpoint.as_ref())?; + let checkpoint_visibility = if matches!(read, ControlRead::Git) { + checkpoint + .as_ref() + .map(|checkpoint| checkpoint.visibility_reader(router, limits.max_capsule_bytes)) + .transpose()? + .flatten() + .map(Arc::new) + } else { + None + }; let mut visibility = if matches!(read, ControlRead::Git) { - visibility::Reconciliation::Paged(Box::new(visibility::Visibility::new( - router.store().clone(), - initial, - snapshot.record().root().refs().keys().cloned(), - ))) + let names = snapshot.record().root().refs().keys().cloned(); + let initial = match &checkpoint_visibility { + Some(reader) => { + visibility::Visibility::from_checkpoint( + router.store().clone(), + reader.clone(), + names, + ) + .await? + } + None => visibility::Visibility::new( + router.store().clone(), + visibility_index_for_checkpoint(None)?, + names, + ), + }; + visibility::Reconciliation::Paged(Box::new(initial)) } else { - visibility::Reconciliation::Complete(initial) + visibility::Reconciliation::Complete(visibility_index_for_checkpoint(checkpoint.as_ref())?) }; for capsule in &root_capsules { visibility.apply(capsule).await?; @@ -4889,10 +4938,11 @@ async fn assemble_layered_control_view( root: snapshot, browse_indexes: None, checkpoint, + checkpoint_visibility, capsules: Vec::new(), capsule_controls: root_capsules, tip_bound_transitions, - visibility_store: matches!(read, ControlRead::Git).then(|| router.store().clone()), + metadata_layout: matches!(read, ControlRead::Git).then(|| router.clone()), tip_bound_controls: if matches!(read, ControlRead::Git) { all_capsule_controls } else { @@ -5739,8 +5789,8 @@ mod tests { use bytes::Bytes; use crab_metadata::capsule_protocol::{ CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, - CapsuleVisibilityDelta, CapsuleVisibilitySnapshot, CheckpointPointer, HistorySegment, - HistorySegmentState, PackLayer, PackRange, RepositoryRoot, + CapsuleVisibilityDelta, CheckpointPointer, HistorySegment, HistorySegmentState, PackLayer, + PackRange, RepositoryRoot, }; use crab_metadata::git_visibility::GitVisibilityEdit; use crab_storage::{StorageObservation, StorageObserver, StorageOperation, StorageOutcome}; @@ -6422,8 +6472,8 @@ mod tests { complete.git_snapshot().unwrap() ); assert_eq!( - metadata.pointer_catalog().unwrap(), - complete.pointer_catalog().unwrap() + metadata.pointer_catalog().await.unwrap(), + complete.pointer_catalog().await.unwrap() ); assert!(metadata.capsules().is_empty()); let observations = observer.observations.lock().unwrap(); @@ -6456,12 +6506,12 @@ mod tests { ( git.git_snapshot().unwrap(), git.git_visibility_index().await.unwrap(), - git.pointer_catalog().unwrap() + git.pointer_catalog().await.unwrap() ), ( complete.git_snapshot().unwrap(), complete.git_visibility_index().await.unwrap(), - complete.pointer_catalog().unwrap() + complete.pointer_catalog().await.unwrap() ) ); } @@ -6524,12 +6574,7 @@ mod tests { #[tokio::test] async fn layered_control_view_range_reads_checkpoint_suffix_without_pack_body() { - for visibility in [ - None, - Some( - CapsuleVisibilitySnapshot::from_index(&visibility_index(BTreeMap::new())).unwrap(), - ), - ] { + for include_visibility in [false, true] { let inner = Arc::new(InMemory::new()); seed_one_capsule(inner.clone(), None).await; let seed_store = Store::new(inner.clone()); @@ -6554,6 +6599,17 @@ mod tests { ) .await .unwrap(); + let source = layer.source_descriptor().unwrap(); + let visibility = include_visibility.then(|| { + crab_metadata::capsule_protocol::LayeredVisibilitySnapshot::from_index( + &visibility_index(BTreeMap::new()), + &crab_metadata::capsule_protocol::source_catalog_digest(std::slice::from_ref( + &source, + )) + .unwrap(), + ) + .unwrap() + }); let checkpoint = LayeredCheckpoint::build( current.record().root().generation(), current.record().digest(), diff --git a/crates/crab-read/src/capsule_protocol/visibility.rs b/crates/crab-read/src/capsule_protocol/visibility.rs index cd8089dfc..4682e2fa2 100644 --- a/crates/crab-read/src/capsule_protocol/visibility.rs +++ b/crates/crab-read/src/capsule_protocol/visibility.rs @@ -1,7 +1,7 @@ //! Captured visibility closure layers; physical presence never grants membership. use super::*; use crab_metadata::capsule_protocol::{ - CapsuleVisibilityReader, VisibilityChange, VisibilityEditHeader, + CapsuleVisibilityReader, LayeredVisibilityReader, VisibilityChange, VisibilityEditHeader, }; use crab_metadata::git_visibility::{GitVisibilityIndex, MAX_GIT_VISIBILITY_OBJECTS}; @@ -38,9 +38,48 @@ struct Layer { edit: Option<(Arc, String, VisibilityEditHeader)>, } +enum Initial { + Complete(GitVisibilityIndex), + Paged(Arc), +} + +impl Initial { + fn count(&self) -> u64 { + match self { + Self::Complete(index) => index.objects().len() as u64, + Self::Paged(reader) => reader.object_count(), + } + } + async fn contains(&self, store: &Store, name: &str, oids: &[[u8; 20]]) -> Result> { + match self { + Self::Complete(index) => Ok(oids + .iter() + .map(|oid| index.contains_in_ref(name, oid)) + .collect()), + Self::Paged(reader) => { + reader + .contains_in_ref(store, name, oids) + .await? + .ok_or_else(|| { + corrupt_path("checkpoint visibility", "captured initial ref is absent") + }) + } + } + } + async fn dictionary(&self, store: &Store, oids: &[[u8; 20]]) -> Result> { + match self { + Self::Complete(index) => Ok(oids + .iter() + .map(|oid| index.objects().binary_search(oid).is_ok()) + .collect()), + Self::Paged(reader) => Ok(reader.dictionary_contains(store, oids).await?), + } + } +} + pub(super) struct Visibility { store: Store, - initial: GitVisibilityIndex, + initial: Initial, layers: Vec, refs: BTreeMap, dictionary: BTreeSet<[u8; 20]>, @@ -62,9 +101,32 @@ impl Visibility { initial: GitVisibilityIndex, names: impl Iterator, ) -> Self { + let names = names + .filter(|name| initial.contains_ref(name)) + .collect::>(); + Self::seed(store, Initial::Complete(initial), names) + } + + pub(super) async fn from_checkpoint( + store: Store, + reader: Arc, + names: impl Iterator, + ) -> Result { + let names = names.collect::>(); + let borrowed = names.iter().map(String::as_str).collect::>(); + let present = reader.refs_present(&store, &borrowed).await?; + let names = names + .into_iter() + .zip(present) + .filter_map(|(name, present)| present.then_some(name)) + .collect(); + Ok(Self::seed(store, Initial::Paged(reader), names)) + } + + fn seed(store: Store, initial: Initial, names: Vec) -> Self { let mut layers = Vec::new(); let mut refs = BTreeMap::new(); - for name in names.filter(|name| initial.contains_ref(name)) { + for name in names { refs.insert(name.clone(), layers.len()); layers.push(Layer { parent: None, @@ -72,7 +134,7 @@ impl Visibility { edit: None, }); } - let dictionary_count = initial.objects().len() as u64; + let dictionary_count = initial.count(); Self { store, initial, @@ -93,8 +155,13 @@ impl Visibility { corrupt_path("capsule Git visibility", "closure parent is invalid") })?; if let Some(name) = &layer.initial_ref { - for position in pending { - result[position] = self.initial.contains_in_ref(name, &oids[position]); + let keys = pending + .iter() + .map(|position| oids[*position]) + .collect::>(); + let present = self.initial.contains(&self.store, name, &keys).await?; + for (position, present) in pending.into_iter().zip(present) { + result[position] = present; } break; } @@ -129,12 +196,10 @@ impl Visibility { } async fn dictionary_contains(&self, oids: &[[u8; 20]]) -> Result> { - let mut result = oids - .iter() - .map(|oid| { - self.initial.objects().binary_search(oid).is_ok() || self.dictionary.contains(oid) - }) - .collect::>(); + let mut result = self.initial.dictionary(&self.store, oids).await?; + for (present, oid) in result.iter_mut().zip(oids) { + *present |= self.dictionary.contains(oid); + } for (reader, name, _) in &self.dictionary_roots { let positions = result .iter() diff --git a/crates/crab-read/src/selection.rs b/crates/crab-read/src/selection.rs index 02bd10569..dc5fcea13 100644 --- a/crates/crab-read/src/selection.rs +++ b/crates/crab-read/src/selection.rs @@ -479,7 +479,7 @@ pub async fn check_capsule_read_replica_readiness( } let mut stats = ReadinessProbeStats::default(); - let catalog = replica.pointer_catalog()?; + let catalog = replica.pointer_catalog().await?; if let Some(reason) = capsule_pointer_object_gap(replica_router, &catalog, &mut stats, options).await? { diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index fcdac69ee..f93487e31 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -201,7 +201,7 @@ pub async fn maintain_capsule_repository_from_view( publish_checkpoint_with_catalog( layout, view, - view.pointer_catalog()?, + view.pointer_catalog().await?, maximum_bytes, cancel, ) @@ -295,7 +295,7 @@ pub async fn repack_capsule_repository_from_view( view, consolidation.sources, member_oids, - view.pointer_catalog()?, + view.pointer_catalog().await?, maximum_bytes, cancel, ) @@ -311,7 +311,7 @@ pub async fn repack_capsule_repository_from_view( publication = publish_checkpoint_with_catalog( layout, view, - view.pointer_catalog()?, + view.pointer_catalog().await?, maximum_bytes, cancel, ) @@ -353,7 +353,7 @@ pub async fn publish_capsule_checkpoint_from_view( if view.capsule_count()? < u64::from(threshold) { return Ok(CheckpointOutcome::default()); } - let catalog = view.pointer_catalog()?; + let catalog = view.pointer_catalog().await?; publish_capsule_checkpoint_with_catalog_from_view(layout, view, catalog, maximum_bytes, cancel) .await } @@ -543,14 +543,13 @@ async fn publish_layered_sources( &catalog_digest, member_admission, )?; - let checkpoint = - crab_metadata::capsule_protocol::LayeredCheckpoint::build_with_ordinal_visibility( - view.root().root().generation(), - view.root().digest(), - sources, - catalog, - Some(visibility), - )?; + let checkpoint = crab_metadata::capsule_protocol::LayeredCheckpoint::build( + view.root().root().generation(), + view.root().digest(), + sources, + catalog, + Some(visibility), + )?; if checkpoint.bytes().len() as u64 > maximum_bytes && maximum_bytes > 0 { return Err(CheckpointError::OutputLimit { maximum: maximum_bytes, diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 906f6269c..ddd0e4dec 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -837,7 +837,7 @@ impl Prepared { "Capsule view differs from the prepared repository version", )); } - let base_catalog = view.pointer_catalog()?; + let base_catalog = view.pointer_catalog().await?; let prepared_files = self .content .as_ref() diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index 91652a133..abfac50a9 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -3190,10 +3190,6 @@ mod tests { )]), ) .unwrap(); - let visibility = crab_metadata::capsule_protocol::CapsuleVisibilitySnapshot::from_index( - &visibility_index, - ) - .unwrap(); let pack = CapsuleGitPack::new( Bytes::from_static(b"RESTORED PACK"), Bytes::from_static(b"restored index"), @@ -3207,6 +3203,12 @@ mod tests { .unwrap() .source_descriptor() .unwrap(); + let visibility = crab_metadata::capsule_protocol::LayeredVisibilitySnapshot::from_index( + &visibility_index, + &crab_metadata::capsule_protocol::source_catalog_digest(std::slice::from_ref(&source)) + .unwrap(), + ) + .unwrap(); let mut restored_catalog = crab_metadata::capsule_protocol::PointerCatalog::new(); restored_catalog .insert_xorb( @@ -3251,7 +3253,7 @@ mod tests { ) .await .unwrap(); - assert_eq!(restored.record().root().checkpoint().unwrap().format(), 5); + assert_eq!(restored.record().root().checkpoint().unwrap().format(), 6); assert_eq!( restored.record().root().generation(), fenced.record().root().generation() + 1 From 35aa789b69b9ec17520d10017c1c62b1dc445bbf Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 13:40:54 -0700 Subject: [PATCH 030/127] test(read): qualify paged checkpoint visibility --- .../docs/architecture/indexed-remote-reads.md | 38 ++++- .../src/capsule_protocol/dependency_tests.rs | 146 ++++++++++++++++++ crates/crab-remote/examples/support/mod.rs | 13 +- 3 files changed, 187 insertions(+), 10 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 29d5b7192..16e6ef75f 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1156,7 +1156,7 @@ Full catalog materialization still exists as the explicit bulk operation; normal historical Xet lookups remain a later, incomplete gate. Current proof: targeted capsule suites pass for metadata (96), read (46), and -write (22). Metadata includes four lazy-reader tests: a 50k-object lookup within +write (22); the added captured-catalog regression brings read to 47. Metadata includes four lazy-reader tests: a 50k-object lookup within 128 KiB despite a body over 2 MiB, retained-but-invisible denial, warm zero origin reads, full-materialization parity, and corrupted/missing page failure. Shared read/write/remote compile checks, the CLI link build, protected-receive all-target @@ -1165,9 +1165,37 @@ Clippy passes with `--no-deps --all-targets` for metadata/read/write/remote. For fixtures use v6 without changing their epoch/visibility assertions. These are focused proofs, not fresh-prefix RustFS or end-to-end v6 qualification. -Before marking this draft ready: finish live scope-wide API consumer proof; -exercise checkpoint plus later ref edits, corruption, cancellation, physical -index absence and explicit catalog materialization; run fresh-prefix RustFS -qualification with frozen native Git fixtures and 20 cold/warm samples. Source +Fresh-prefix RustFS proof passed under +`checkpoint-pages-fresh-live-r1` outside the checkout. Native push published +unchanged Git fixtures, then the actual checkpoint publisher and index builder +ran before comparison. All three cells (100, 1,000 and 10,000 commits) pass 20 cold/warm samples for +resolve, file history and blame, plus all ten query oracles. The 189 processes +include six maintenance runs and prove 210 native query comparisons and 3,150 +blame-line origins. Every measured operation drains backend work; warm history +and blame queries read zero origin bytes. + +| Commits | v5 opening bytes | v6 opening bytes | v6 checkpoint page bytes | Complete proof decodes | +| --- | ---: | ---: | ---: | ---: | +| 100 | 29,255 | 20,588 | 504 | 0 | +| 1,000 | 119,259 | 45,456 | 499 | 0 | +| 10,000 | 771,540 | 46,376 | 499 | 0 | + +Opening performs 12/13/14 backend operations in these fixed-ref/source fixtures. +These results establish no bound across independently growing refs or source +inventories. Loopback RustFS latency is not representative hosted-S3 latency. + +The v6 checkpoint plus a subsequent native add/commit/push passes all ten native +Git query comparisons. Removing only the physical-index readiness field also +passes, then the exact readiness record is restored. Tampering with its retained +dictionary page fails capture with a page-integrity error and zero active +backend operations; restoring the object restores native Git parity. A focused +catalog regression passes: a captured Git view materializes its original +catalog after the mutable root changes. Qualification now counts checkpoint +visibility page decoding separately from complete proof decoding. + +Before marking this draft ready: finish tagged-ref, force-push, cancellation, +Git-helper fetch/hydration and live lifecycle qualification; independently grow +ref count, source inventory and uncompacted history. The standard Makefile +installer is building current helpers into an isolated workspace prefix. Source inventory/ref capture, comparison caches, catalog indexing, lifecycle/provider qualification, unified staging and fuller blame remain incomplete. diff --git a/crates/crab-read/src/capsule_protocol/dependency_tests.rs b/crates/crab-read/src/capsule_protocol/dependency_tests.rs index bf0f3564e..9bb0eaeb9 100644 --- a/crates/crab-read/src/capsule_protocol/dependency_tests.rs +++ b/crates/crab-read/src/capsule_protocol/dependency_tests.rs @@ -325,3 +325,149 @@ async fn catalog_file_proof_cancellation_interrupts_pending_shard_read() { Err(ReadError::Cancelled), )); } + +#[tokio::test] +async fn git_view_catalog_materialization_uses_the_captured_checkpoint_after_root_changes() { + use crab_metadata::capsule_protocol::{ + CapsuleGitPack, CapsuleRefEdit, CapsuleTransaction, HistorySegment, HistorySegmentState, + LayeredVisibilitySnapshot, PackLayer, RepositoryRoot, source_catalog_digest, + }; + use crab_metadata::git_visibility::GitVisibilityIndex; + + let (layout, catalog, _workspace, refs) = fixture("valid").await; + let initial = + RootRecord::encode(RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap()) + .unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "2".repeat(40), + 1, + ) + .unwrap(); + let transaction = CapsuleTransaction::new( + initial.digest(), + vec![CapsuleRefEdit::new( + "refs/tags/file", + None, + refs.get("refs/tags/file").cloned(), + None, + )], + ) + .unwrap(); + let run = + CapsuleRun::leaf(Capsule::build(&transaction, vec![pack.clone()], Vec::new()).unwrap()) + .unwrap(); + let run_pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let initial = RootRecord::encode( + initial + .root() + .advance( + initial.digest(), + refs.clone(), + BTreeMap::new(), + vec![run_pointer], + &transaction.id().unwrap(), + ) + .unwrap(), + ) + .unwrap(); + let layer = PackLayer::build(&pack).unwrap(); + let source = layer.source_descriptor().unwrap(); + let visibility = LayeredVisibilitySnapshot::from_index( + &GitVisibilityIndex::new( + 1, + "0".repeat(64), + "0".repeat(64), + refs.into_iter() + .map(|(name, oid)| (name, vec![oid])) + .collect(), + ) + .unwrap(), + &source_catalog_digest(std::slice::from_ref(&source)).unwrap(), + ) + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + initial.root().generation(), + initial.digest(), + vec![source], + catalog.clone(), + Some(visibility), + ) + .unwrap(); + let pointer = CheckpointPointer::new_layered( + checkpoint.hash(), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.control_size(), + checkpoint.footer_hash(), + checkpoint.covered_generation(), + checkpoint.covered_root_digest(), + checkpoint.pack_count().unwrap(), + checkpoint.object_count().unwrap(), + ) + .unwrap(); + let history = HistorySegment::build( + pointer.clone(), + None, + HistorySegmentState::new( + initial.root().refs().clone(), + initial.root().peeled_refs().clone(), + initial.root().head().to_owned(), + initial.root().compacted_ref_transactions().clone(), + initial.root().capsule_frontier().to_vec(), + ), + ) + .unwrap(); + let root = RootRecord::encode( + initial + .root() + .install_checkpoint(initial.digest(), pointer, Some(history.pointer().unwrap())) + .unwrap(), + ) + .unwrap(); + layout + .store() + .put( + &layout.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + layout + .store() + .put(&layout.capsule_root_path(), root.bytes().clone()) + .await + .unwrap(); + let view = open_git_view_from_root( + &layout, + load_root(&layout).await.unwrap(), + CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 1024 * 1024, + }, + ) + .await + .unwrap(); + assert!(view.layered_checkpoint().unwrap().is_control_only()); + // The bulk read must follow the captured pointer rather than recapturing a + // mutable root; otherwise hydration could silently select a different recipe. + layout + .store() + .put_overwrite(&layout.capsule_root_path(), initial.bytes().clone()) + .await + .unwrap(); + assert_eq!(view.pointer_catalog().await.unwrap(), catalog); +} diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index 693b25ba9..fe10d1693 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -25,8 +25,8 @@ pub struct Measurements { read_bytes: AtomicU64, written_bytes: AtomicU64, active: AtomicU64, - decoded_blocks: [AtomicU64; 6], - decoded_bytes: [AtomicU64; 6], + decoded_blocks: [AtomicU64; 7], + decoded_bytes: [AtomicU64; 7], } #[derive(Clone, Copy)] @@ -35,8 +35,8 @@ struct Snapshot { outcomes: [u64; 9], read_bytes: u64, written_bytes: u64, - decoded_blocks: [u64; 6], - decoded_bytes: [u64; 6], + decoded_blocks: [u64; 7], + decoded_bytes: [u64; 7], } impl StorageObserver for Measurements { @@ -74,6 +74,7 @@ impl tracing::field::Visit for DecodeEvent { "capsule_admission" => Some(3), "capsule_visibility" => Some(4), "checkpoint_visibility" => Some(5), + "checkpoint_visibility_pages" => Some(6), _ => None, }; } @@ -170,6 +171,7 @@ impl Measurements { "capsule_admission", "capsule_visibility", "checkpoint_visibility", + "checkpoint_visibility_pages", ] .into_iter() .enumerate() @@ -248,12 +250,13 @@ mod tests { tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_admission", decoded_bytes=29_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_visibility", decoded_bytes=31_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility", decoded_bytes=37_u64); + tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility_pages", decoded_bytes=43_u64); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), - ([1, 1, 1, 1, 1, 1], [41, 123, 67, 29, 31, 37]) + ([1, 1, 1, 1, 1, 1, 1], [41, 123, 67, 29, 31, 37, 43]) ); } From c579f6b87f622d2860eaaad71645899de2eef0e1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 13:45:05 -0700 Subject: [PATCH 031/127] docs: clarify checkpoint qualification evidence --- crab/docs/architecture/indexed-remote-reads.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 16e6ef75f..26329328a 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1155,20 +1155,23 @@ protected receive, replication and maintenance callers await that canonical API. Full catalog materialization still exists as the explicit bulk operation; normal historical Xet lookups remain a later, incomplete gate. -Current proof: targeted capsule suites pass for metadata (96), read (46), and -write (22); the added captured-catalog regression brings read to 47. Metadata includes four lazy-reader tests: a 50k-object lookup within +Current proof: targeted capsule suites pass for metadata (96), read (47), and +write (22), including the added captured-catalog regression. Metadata includes +four lazy-reader tests: a 50k-object lookup within 128 KiB despite a body over 2 MiB, retained-but-invisible denial, warm zero origin reads, full-materialization parity, and corrupted/missing page failure. Shared read/write/remote compile checks, the CLI link build, protected-receive all-target compile check and 84 payload-only capsule tests pass. Strict owner-crate -Clippy passes with `--no-deps --all-targets` for metadata/read/write/remote. Format-specific -fixtures use v6 without changing their epoch/visibility assertions. These are -focused proofs, not fresh-prefix RustFS or end-to-end v6 qualification. +Clippy passes with `--no-deps --all-targets` for metadata/read/write/remote. +Format-specific fixtures use v6 without changing their epoch/visibility +assertions. This local suite is separate from the fresh-prefix end-to-end proof +below. Fresh-prefix RustFS proof passed under `checkpoint-pages-fresh-live-r1` outside the checkout. Native push published unchanged Git fixtures, then the actual checkpoint publisher and index builder -ran before comparison. All three cells (100, 1,000 and 10,000 commits) pass 20 cold/warm samples for +ran before comparison. All three cells (100, 1,000 and 10,000 commits) pass +20 cold/warm samples for resolve, file history and blame, plus all ten query oracles. The 189 processes include six maintenance runs and prove 210 native query comparisons and 3,150 blame-line origins. Every measured operation drains backend work; warm history From 3e8f9b6e5333e2a24212d113fbbefdac93964882 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 14:41:52 -0700 Subject: [PATCH 032/127] test(read): qualify refs and pack inventory growth --- .../docs/architecture/indexed-remote-reads.md | 53 ++++++++++++++++- crates/crab-remote/examples/qualify_remote.rs | 57 ++++++++++++++++--- 2 files changed, 99 insertions(+), 11 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 26329328a..24983c4a2 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1187,8 +1187,12 @@ Opening performs 12/13/14 backend operations in these fixed-ref/source fixtures. These results establish no bound across independently growing refs or source inventories. Loopback RustFS latency is not representative hosted-S3 latency. -The v6 checkpoint plus a subsequent native add/commit/push passes all ten native -Git query comparisons. Removing only the physical-index readiness field also +A subsequent commit and native push passes all ten native Git query +comparisons. The first small-text harness invoked `crab add` without a tracking +pattern, so that command correctly skipped the edit; its commit changed only +other staged metadata. This did not qualify a committed text edit. The helper +probe subsequently staged the text with `git add`, checked committed bytes and +pushed the actual edit through the installed Git helper. Removing only the physical-index readiness field also passes, then the exact readiness record is restored. Tampering with its retained dictionary page fails capture with a page-integrity error and zero active backend operations; restoring the object restores native Git parity. A focused @@ -1199,6 +1203,49 @@ visibility page decoding separately from complete proof decoding. Before marking this draft ready: finish tagged-ref, force-push, cancellation, Git-helper fetch/hydration and live lifecycle qualification; independently grow ref count, source inventory and uncompacted history. The standard Makefile -installer is building current helpers into an isolated workspace prefix. Source +installer completed into an isolated workspace prefix. Its Git helper passes +clone and strict fsck, actual text-edit push/fetch, annotated tag push/fetch/ +delete, force-push/fetch and shallow clone/fsck. A fresh empty client rejects +the removed commit with upload-pack policy error CRAB-E0060; an already-populated +client is not a valid oracle for that denial. The fixture main is restored. Source inventory/ref capture, comparison caches, catalog indexing, lifecycle/provider qualification, unified staging and fuller blame remain incomplete. + + +### Ref and physical inventory qualification + +The existing qualifier passed 20 cold/warm samples each for resolve, object point +and object batch reads, plus its full native-oracle set, at 1/10/100/1,000 refs. +All refs point to the same fixed history; selected query output stays unchanged. +The 252 processes prove 280 native query comparisons. Complete repository +opening reads 20,588/34,934/175,073/1,577,466 bytes with 12/22/114/1,028 backend +operations. Its fixed resolve query reads 37,365 bytes at every ref count. This +exposes full ref-head capture cost; it does not establish bounded single-ref +opening. The existing selected-ref API separately passes 20 cold/warm samples per +cell. Its query reads 1,442/1,985/1,989/1,993 bytes in 2/3/3/3 backend operations. +Root loading is measured separately and reads 1,312/2,734/16,955/159,158 bytes; +this is not constant total opening cost. Native push and import already consume +this selected-ref API; the qualifier now exercises it directly. + +A second 252-process/280-query matrix holds the final 164 commits and 613 Git +objects fixed while varying publication schedules. Actual pack counts are +1/8/32/64; capsule-run rollup makes actual source counts 1/8/1/1. Logical +checkpointing reads/writes zero pack-body bytes. Opening reads +26,951/45,215/59,169/83,443 bytes, and resolve reads +54,283/55,067/57,755/61,339 bytes. Peak process RSS for resolve stays below +35 MB. These cells expose member-inventory and publication-history growth; +they do not prove independent scaling to 64 source objects. + + +The direct commit-visibility probe publishes a distinct commit on a temporary +ref and resolves its immutable OID successfully. After deletion and index +rebuild, the same retained OID fails with `Revision { reason: NotReachable }` and +zero active backend operations. This checks the reader's visibility boundary, +separately from Git-helper upload-pack's default want policy. + +The existing large-file harness is running against the isolated Makefile-installed +v6 binary in a fresh RustFS bucket. It uses two actual 10 GiB files with +deterministic entropy, 5,000 code files and four versions, alternating native +add and Git clean. It checks pointer staging, incremental upload, fresh-cache +historical hydration and integrity after removing source authority. Results are +pending; earlier v5 or small-file receipts do not qualify this run. diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 36ffe5c1c..16313c075 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -36,7 +36,7 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes|checkpoint]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint|ref-point|resolve-oid]", ) .into()), }; @@ -45,6 +45,8 @@ async fn main() -> Result<()> { && selection != "checkpoint" && selection != "graph-pages" && selection != "graph-ancestry" + && selection != "ref-point" + && selection != "resolve-oid" && !WORKLOADS.contains(&selection) { return Err(io::Error::other("unknown qualification operation").into()); @@ -68,7 +70,35 @@ async fn main() -> Result<()> { .with_storage_observer(counters.clone()); let layout = StoreLayout::new(store, prefix.to_owned()); let identity = RepositoryIdentity::new(format!("s3:{bucket}"), prefix.as_str(), 1)?; - if selection == "checkpoint" { + if selection == "ref-point" { + let root = counters + .measure("open", "cold", async { + Ok(crab_metadata::capsule_protocol::load_root(&layout).await?) + }) + .await?; + let names = std::collections::BTreeSet::from([root.record().root().head().to_owned()]); + let mut previous = None; + for cache in ["cold", "warm"] { + let refs = counters + .measure("ref-point", cache, async { + Ok( + crab_read::capsule_protocol::read_visible_refs_from_root_for_refs( + &layout, &root, &names, + ) + .await?, + ) + }) + .await?; + if previous.as_ref().is_some_and(|value| value != &refs) { + return Err(io::Error::other("selected refs changed during qualification").into()); + } + previous = Some(refs); + } + println!( + "{}", + json!({"kind":"result", "operation":"ref-point", "value":previous}) + ); + } else if selection == "checkpoint" { counters .measure("checkpoint", "maintenance", async { let outcome = crab_remote::checkpoint::publish_capsule_checkpoint( @@ -136,7 +166,12 @@ async fn main() -> Result<()> { } else if matches!(selection, "graph-pages" | "graph-ancestry") { qualify_graph_pages(&layout, &path, &counters, selection).await?; } else { - for &name in WORKLOADS + let workloads = if selection == "resolve-oid" { + &["resolve-oid"][..] + } else { + WORKLOADS + }; + for &name in workloads .iter() .filter(|&&name| selection == "all" || selection == name) { @@ -354,7 +389,7 @@ async fn query( cancel: &CancellationToken, ) -> Result { let kind = match name { - "resolve" => OperationKind::Resolve, + "resolve" | "resolve-oid" => OperationKind::Resolve, "commit" => OperationKind::Commit, "tree" => OperationKind::Tree, "history" => OperationKind::History, @@ -363,16 +398,22 @@ async fn query( "blame" => OperationKind::Blame, _ => OperationKind::Content, }; + let requested_oid = if name == "resolve-oid" { + Some(gix_hash::ObjectId::from_hex(path.as_bytes())?) + } else { + None + }; let operation = repository.operation(kind, cancel).await?; // Keep every failure inside the semantic result until the locator closes. let result = async { let snapshot = repository.snapshot(revision, &operation).await?; match name { - "resolve" => { - // Ref resolution alone bypasses ancestor membership. Resolve the - // same immutable OID through the foreground visibility boundary. + "resolve" | "resolve-oid" => { + // Ref-tip resolution alone bypasses ancestor membership. Explicit + // commit reads must also exercise the foreground visibility gate. + let oid = requested_oid.unwrap_or(snapshot.commit_oid()); let resolved = repository - .resolve(&Revision::Commit(snapshot.commit_oid()), &operation) + .resolve(&Revision::Commit(oid), &operation) .await?; Ok(json!({"commit":resolved.commit.to_string()})) } From 61ee40c1ec9e4dcbe2c6c5f63c3495ed5c0eb019 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 15:07:24 -0700 Subject: [PATCH 033/127] perf(blame): cache immutable blob comparisons --- .../docs/architecture/indexed-remote-reads.md | 27 +++ crab/scripts/e2e/verify_remote_browse.py | 8 + crates/crab-remote-git/README.md | 10 + crates/crab-remote-git/src/operation.rs | 23 +++ crates/crab-remote-git/src/runtime.rs | 181 ++++++++++++++++-- crates/crab-remote-git/src/snapshot.rs | 43 ++++- .../tests/remote_repository.rs | 78 ++++++++ crates/crab-remote/examples/qualify_remote.rs | 45 ++++- crates/crab-remote/examples/support/mod.rs | 29 ++- 9 files changed, 412 insertions(+), 32 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 24983c4a2..221cd7515 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1249,3 +1249,30 @@ deterministic entropy, 5,000 code files and four versions, alternating native add and Git clean. It checks pointer staging, incremental upload, fresh-cache historical hydration and integrity after removing source authority. Results are pending; earlier v5 or small-file receipts do not qualify this run. + + +### Blame comparison cache + +The next branch adds reuse of first-parent Myers line mappings across immutable +blob pairs. Attribution and comparison entries share one bounded LRU and its +existing configuration. Ordered blob IDs, repository identity and an explicit +semantics version identify comparisons; full attribution also includes commit, +raw path and generation. The caller still reads/verifies both blobs, charges the +comparison-cell budget and checks cancellation before using a retained mapping. + +The qualifier's `blame-history` case requests ten successive first-parent +snapshots and checks each range against native Git. Hit/miss instrumentation is +separate from graph/path-page decode counts. Two focused cache isolation/capacity +tests, five blame integration tests and two qualifier counter tests pass, +including rejection under a stricter comparison budget. The 11 snapshot-index +contract tests pass, including caller/shutdown/deadline cancellation of graph +and path pages. Strict Clippy passes for both changed owner crates with +`--all-targets --no-deps -D warnings` and the publication feature. + +The dedicated RustFS comparison run uses cache-enabled and disabled queries at +100, 1,000 and 10,000 commits, with 20 process samples per cell and native Git +line-origin checks on ten successive snapshots. At 100 commits, all 20 paired +samples pass; cold comparison misses fall from 192 to 21 with 171 hits. The +larger cells remain running, so the complete acceptance gate and latency +conclusions remain pending. External receipts live in +`blame-comparison-live-r2`; hosted S3 latency qualification is still outstanding. diff --git a/crab/scripts/e2e/verify_remote_browse.py b/crab/scripts/e2e/verify_remote_browse.py index cbb0bd4a7..674f04cc4 100644 --- a/crab/scripts/e2e/verify_remote_browse.py +++ b/crab/scripts/e2e/verify_remote_browse.py @@ -44,6 +44,8 @@ def git(*args) -> bytes: expected_names = {"resolve", "commit", "tree", "blob", "object-point", "object-batch", "history", "path-history", "attribution", "blame"} selection = records[-1]["selection"] + if selection == "blame-history-uncached": + selection = "blame-history" require(len(names) == len(set(names)), "duplicate query evidence") require(set(names) == (expected_names if selection == "all" else {selection}), "missing query evidence") measurements = [record for record in records if record["kind"] == "measurement"] @@ -100,6 +102,12 @@ def check_object(value: dict) -> None: require(item["commit"] == expected, "last-change commit differs") elif name == "blame": lines_checked += check_line_origins(git, head, path, value["ranges"]) + elif name == "blame-history": + expected = git("rev-list", "--first-parent", "--max-count=10", head).decode().splitlines() + require([entry["commit"] for entry in value["entries"]] == expected, + "successive blame commits differ") + for entry in value["entries"]: + lines_checked += check_line_origins(git, entry["commit"], path, entry["ranges"]) elif name in {"graph-pages", "graph-ancestry"}: rows = value["records"] require(len(rows) == 4 and rows[1] == rows[3], "graph batch order/duplicates differ") diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index 9237d4cca..1a5175166 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -254,3 +254,13 @@ permits ordinary raw reads. The manifest-based `open` retains its release-tagged optional-graph policy; cancellation and admission failures are never suppressed. Operation finalization normalizes wrapped cancellation only when its own scope was interrupted, preserving real failures and typed locator-close errors. + +## Blame comparison reuse + +First-parent blame retains verified line mappings by repository identity, ordered +parent/current blob IDs and a fixed algorithm/semantics version. Complete blame +results retain their commit, raw-path and generation identity. Both entry kinds +share the existing blame cache's entry and byte limits; no additional cache +configuration is introduced. Blob verification, comparison-budget charging and +cancellation checks happen before consuming a retained line map. A cached map +never authorizes a commit or substitutes for fetching an unverified blob. diff --git a/crates/crab-remote-git/src/operation.rs b/crates/crab-remote-git/src/operation.rs index b18e1dd24..bd8bb1e2d 100644 --- a/crates/crab-remote-git/src/operation.rs +++ b/crates/crab-remote-git/src/operation.rs @@ -1245,6 +1245,29 @@ impl OperationContext { .await; } + pub(crate) async fn cached_blame_comparison( + &self, + parent: gix_hash::ObjectId, + current: gix_hash::ObjectId, + ) -> Option>> { + self.state + .runtime + .cached_blame_comparison(&self.state.identity, parent, current) + .await + } + + pub(crate) async fn insert_blame_comparison( + &self, + parent: gix_hash::ObjectId, + current: gix_hash::ObjectId, + matches: Arc>, + ) { + self.state + .runtime + .insert_blame_comparison(self.state.identity.clone(), parent, current, matches) + .await; + } + pub(crate) fn object_limits(&self) -> crate::ObjectLimits { self.state.options.object_limits() } diff --git a/crates/crab-remote-git/src/runtime.rs b/crates/crab-remote-git/src/runtime.rs index bc9daa68b..643fbdccd 100644 --- a/crates/crab-remote-git/src/runtime.rs +++ b/crates/crab-remote-git/src/runtime.rs @@ -48,9 +48,9 @@ pub struct RuntimeOptions { pub max_parsed_cache_entries: usize, /// Maximum estimated parsed-object bytes retained in memory. pub max_parsed_cache_bytes: usize, - /// Maximum immutable blame results retained in memory. + /// Maximum immutable blame results and blob comparisons retained in memory. pub max_blame_cache_entries: usize, - /// Maximum estimated immutable blame-result bytes retained in memory. + /// Maximum estimated bytes shared by immutable blame results and blob comparisons. pub max_blame_cache_bytes: usize, /// Maximum short-lived manifest entries retained in memory. pub max_manifest_cache_entries: usize, @@ -85,9 +85,9 @@ pub struct RuntimeSnapshot { pub parsed_entries: usize, /// Estimated resident parsed-object bytes. pub parsed_bytes: usize, - /// Retained immutable blame results. + /// Retained immutable blame results and blob comparisons. pub blame_entries: usize, - /// Estimated resident immutable blame-result bytes. + /// Estimated resident blame-result and blob-comparison bytes. pub blame_bytes: usize, /// Retained manifest pointers. pub manifest_entries: usize, @@ -188,11 +188,29 @@ struct InventoryCacheKey { } #[derive(Clone, PartialEq, Eq, Hash)] -struct BlameCacheKey { - identity: RepositoryIdentity, - generation: u64, - commit: ObjectId, - path: GitPath, +enum BlameCacheKey { + Attribution { + identity: RepositoryIdentity, + generation: u64, + commit: ObjectId, + path: GitPath, + semantics: u32, + }, + Comparison { + identity: RepositoryIdentity, + parent: ObjectId, + current: ObjectId, + semantics: u32, + }, +} + +// Increment when first-parent attribution or Myers line-slider semantics change. +const BLAME_SEMANTICS: u32 = 1; + +#[derive(Clone)] +enum BlameCacheValue { + Attribution(CachedBlame), + Comparison(Arc>), } #[derive(Clone, PartialEq, Eq, Hash)] @@ -304,7 +322,7 @@ pub struct RemoteGitRuntime { object_cache: Mutex>>, pack_index_cache: Mutex>>, parsed_cache: Mutex>, - blame_cache: Mutex>, + blame_cache: Mutex>, manifest_cache: Mutex>>, inventory_cache: Mutex>>>, @@ -618,13 +636,17 @@ impl RemoteGitRuntime { commit: ObjectId, path: &GitPath, ) -> Option { - let key = BlameCacheKey { + let key = BlameCacheKey::Attribution { identity: identity.clone(), generation, commit, path: path.clone(), + semantics: BLAME_SEMANTICS, + }; + let value = match self.blame_cache.lock().await.get(&key) { + Some(BlameCacheValue::Attribution(value)) => Some(value), + _ => None, }; - let value = self.blame_cache.lock().await.get(&key); self.metrics.record(MetricObservation { kind: MetricKind::Cache, value: u64::from(value.is_some()), @@ -650,13 +672,14 @@ impl RemoteGitRuntime { ) { let bytes = blame_bytes(&value); let evicted = self.blame_cache.lock().await.insert( - BlameCacheKey { + BlameCacheKey::Attribution { identity, generation, commit, path, + semantics: BLAME_SEMANTICS, }, - CachedBlame { value, usage }, + BlameCacheValue::Attribution(CachedBlame { value, usage }), bytes, ); if evicted > 0 { @@ -670,6 +693,50 @@ impl RemoteGitRuntime { } } + pub(crate) async fn cached_blame_comparison( + &self, + identity: &RepositoryIdentity, + parent: ObjectId, + current: ObjectId, + ) -> Option>> { + let key = BlameCacheKey::Comparison { + identity: identity.clone(), + parent, + current, + semantics: BLAME_SEMANTICS, + }; + let value = match self.blame_cache.lock().await.get(&key) { + Some(BlameCacheValue::Comparison(value)) => Some(value), + _ => None, + }; + tracing::debug!(target: "crab_remote_git::blame_comparison", cache_hit = value.is_some()); + value + } + + pub(crate) async fn insert_blame_comparison( + &self, + identity: RepositoryIdentity, + parent: ObjectId, + current: ObjectId, + matches: Arc>, + ) { + let bytes = std::mem::size_of_val(matches.as_ref()).saturating_add( + matches + .capacity() + .saturating_mul(std::mem::size_of::<(usize, usize)>()), + ); + self.blame_cache.lock().await.insert( + BlameCacheKey::Comparison { + identity, + parent, + current, + semantics: BLAME_SEMANTICS, + }, + BlameCacheValue::Comparison(matches), + bytes, + ); + } + pub(crate) async fn insert_commit( &self, key: ObjectCacheKey, @@ -1220,6 +1287,92 @@ mod tests { assert!(RemoteGitRuntime::new(options, Arc::new(NoopMetrics)).is_ok()); } + #[tokio::test] + async fn blame_comparisons_preserve_direction_and_repository_scope() { + let runtime = RemoteGitRuntime::default(); + let identity = RepositoryIdentity::new("memory", "repo", 1).unwrap(); + let other = RepositoryIdentity::new("memory", "other", 1).unwrap(); + let parent = ObjectId::empty_blob(gix_hash::Kind::Sha1); + let current = ObjectId::empty_tree(gix_hash::Kind::Sha1); + let matches = Arc::new(vec![(0, 1)]); + runtime + .insert_blame_comparison(identity.clone(), parent, current, matches.clone()) + .await; + let cached = runtime + .cached_blame_comparison(&identity, parent, current) + .await + .unwrap(); + assert!(Arc::ptr_eq(&matches, &cached)); + assert!( + runtime + .cached_blame_comparison(&identity, current, parent) + .await + .is_none() + ); + assert!( + runtime + .cached_blame_comparison(&other, parent, current) + .await + .is_none() + ); + } + + #[tokio::test] + async fn blame_results_and_comparisons_share_one_capacity() { + let runtime = Arc::new( + RemoteGitRuntime::new( + RuntimeOptions { + max_blame_cache_entries: 1, + ..RuntimeOptions::default() + }, + Arc::new(NoopMetrics), + ) + .unwrap(), + ); + let identity = RepositoryIdentity::new("memory", "repo", 1).unwrap(); + let parent = ObjectId::empty_blob(gix_hash::Kind::Sha1); + let current = ObjectId::empty_tree(gix_hash::Kind::Sha1); + let path = GitPath::new(Bytes::from_static(b"file")).unwrap(); + runtime + .insert_blame_comparison(identity.clone(), parent, current, Arc::new(vec![(0, 0)])) + .await; + let usage = + crate::budget::OperationBudget::new(crate::OperationLimits::default(), runtime.clone()) + .usage() + .await; + runtime + .insert_blame( + identity.clone(), + 1, + current, + path.clone(), + Arc::new(Blame { + commit: current, + path: path.clone(), + ranges: Vec::new(), + }), + usage, + ) + .await; + assert!( + runtime + .cached_blame_comparison(&identity, parent, current) + .await + .is_none() + ); + assert_eq!(runtime.snapshot().await.blame_entries, 1); + runtime + .insert_blame_comparison(identity.clone(), parent, current, Arc::new(vec![(0, 0)])) + .await; + assert!( + runtime + .cached_blame(&identity, 1, current, &path) + .await + .is_none() + ); + assert_eq!(runtime.snapshot().await.blame_entries, 1); + } + #[tokio::test] async fn object_cache_isolates_provider_repository_and_placement_generation() { let runtime = RemoteGitRuntime::default(); diff --git a/crates/crab-remote-git/src/snapshot.rs b/crates/crab-remote-git/src/snapshot.rs index 30a3357a7..eddb3e67f 100644 --- a/crates/crab-remote-git/src/snapshot.rs +++ b/crates/crab-remote-git/src/snapshot.rs @@ -870,15 +870,38 @@ impl RemoteGitSnapshot { operation .charge(BudgetDimension::BlameComparisons, comparison_cells) .await?; - let matches = git_line_matches( - operation.runtime(), - parent_bytes.clone(), - parent_line_count, - current_bytes.clone(), - current_lines.len(), - operation.cancellation().clone(), - ) - .await?; + // Both blobs and the comparison budget are validated before a + // cache hit. Reusing line mappings cannot authorize an object or + // bypass a stricter operation limit. + let matches = match operation + .cached_blame_comparison(parent_blob.metadata.oid, current_oid) + .await + { + Some(matches) => matches, + None => { + let matches = Arc::new( + git_line_matches( + operation.runtime(), + parent_bytes.clone(), + parent_line_count, + current_bytes.clone(), + current_lines.len(), + operation.cancellation().clone(), + ) + .await?, + ); + operation.ensure_active()?; + operation + .insert_blame_comparison( + parent_blob.metadata.oid, + current_oid, + matches.clone(), + ) + .await; + matches + } + }; + operation.ensure_active()?; let mut parent_tracked = Vec::new(); parent_tracked .try_reserve_exact(parent_lines.len()) @@ -889,7 +912,7 @@ impl RemoteGitSnapshot { source, })?; parent_tracked.resize(parent_lines.len(), None); - for (parent_line, current_line) in matches { + for &(parent_line, current_line) in matches.iter() { if let Some(final_line) = tracked[current_line] { origins[final_line] = parent_commit.oid; parent_tracked[parent_line] = Some(final_line); diff --git a/crates/crab-remote-git/tests/remote_repository.rs b/crates/crab-remote-git/tests/remote_repository.rs index 143bb62ae..5859920d0 100644 --- a/crates/crab-remote-git/tests/remote_repository.rs +++ b/crates/crab-remote-git/tests/remote_repository.rs @@ -4708,6 +4708,84 @@ async fn immutable_blame_cache_is_bounded_and_replays_semantic_budgets() { fixture.runtime.shutdown().await; } +#[tokio::test(flavor = "multi_thread")] +async fn retained_blob_comparison_does_not_bypass_a_stricter_blame_budget() { + let runtime_options = RuntimeOptions { + max_blame_cache_bytes: 256, + ..RuntimeOptions::default() + }; + let fixture = publish_with_runtime( + DeltaKind::Ref, + false, + RepositoryOptions::default(), + runtime_options, + ) + .await; + let cancel = CancellationToken::new(); + let path = GitPath::new(Bytes::from_static(b"text.txt")).unwrap(); + let mut first = None; + for _ in 0..2 { + let operation = fixture + .repository + .operation(OperationKind::Blame, &cancel) + .await + .unwrap(); + let result = async { + let snapshot = fixture + .repository + .snapshot(&Revision::Commit(fixture.semantic_head_commit), &operation) + .await?; + snapshot.blame(&path, &operation).await + } + .await; + let blame = operation.finish(result).await.unwrap(); + if let Some(first) = &first { + assert_eq!(&blame, first); + } + first = Some(blame); + } + // The result exceeds this cache's byte limit; only its reusable line map fits. + assert_eq!(fixture.runtime.snapshot().await.blame_entries, 1); + let strict = RepositoryOptions::new( + ObjectLimits::default(), + OperationLimits { + max_blame_comparison_cells: 1, + ..OperationLimits::default() + }, + ) + .unwrap(); + let repository = RemoteGitRepository::open( + fixture.store.clone(), + fixture.layout.clone(), + fixture.repository.identity().clone(), + fixture.runtime.clone(), + strict, + &cancel, + ) + .await + .unwrap(); + let operation = repository + .operation(OperationKind::Blame, &cancel) + .await + .unwrap(); + let result = async { + let snapshot = repository + .snapshot(&Revision::Commit(fixture.semantic_head_commit), &operation) + .await?; + snapshot.blame(&path, &operation).await + } + .await; + let error = operation.finish(result).await.unwrap_err(); + assert!(matches!( + error, + Error::LimitExceeded { + limit: "blame comparison cells", + .. + } + )); + fixture.runtime.shutdown().await; +} + #[tokio::test(flavor = "multi_thread")] async fn archive_traversal_preserves_modes_links_submodules_and_raw_order() { let fixture = publish(DeltaKind::Ref, false, RepositoryOptions::default()).await; diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 16313c075..c28b959e5 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -36,7 +36,7 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes|checkpoint|ref-point|resolve-oid]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint|ref-point|resolve-oid|blame-history|blame-history-uncached]", ) .into()), }; @@ -47,6 +47,8 @@ async fn main() -> Result<()> { && selection != "graph-ancestry" && selection != "ref-point" && selection != "resolve-oid" + && selection != "blame-history" + && selection != "blame-history-uncached" && !WORKLOADS.contains(&selection) { return Err(io::Error::other("unknown qualification operation").into()); @@ -166,16 +168,32 @@ async fn main() -> Result<()> { } else if matches!(selection, "graph-pages" | "graph-ancestry") { qualify_graph_pages(&layout, &path, &counters, selection).await?; } else { - let workloads = if selection == "resolve-oid" { - &["resolve-oid"][..] + let selected_workload = if selection == "blame-history-uncached" { + "blame-history" } else { - WORKLOADS + selection + }; + let workloads = match selected_workload { + "resolve-oid" => &["resolve-oid"][..], + "blame-history" => &["blame-history"][..], + _ => WORKLOADS, }; for &name in workloads .iter() - .filter(|&&name| selection == "all" || selection == name) + .filter(|&&name| selected_workload == "all" || selected_workload == name) { - let runtime = Arc::new(RemoteGitRuntime::default()); + let runtime = if selection == "blame-history-uncached" { + RemoteGitRuntime::new( + crab_remote_git::RuntimeOptions { + max_blame_cache_entries: 0, + ..crab_remote_git::RuntimeOptions::default() + }, + Arc::new(crab_remote_git::NoopMetrics), + )? + } else { + RemoteGitRuntime::default() + }; + let runtime = Arc::new(runtime); let result = qualify( name, &layout, @@ -395,7 +413,7 @@ async fn query( "history" => OperationKind::History, "path-history" => OperationKind::PathHistory, "attribution" => OperationKind::DirectoryAttribution, - "blame" => OperationKind::Blame, + "blame" | "blame-history" => OperationKind::Blame, _ => OperationKind::Content, }; let requested_oid = if name == "resolve-oid" { @@ -444,6 +462,19 @@ async fn query( Ok(json!({"commits":page.items.iter().map(|entry| entry.commit.oid.to_string()).collect::>(), "has_more":page.next.is_some()})) } + "blame-history" => { + let history = snapshot.history(HistoryTraversal::FirstParent, &PageRequest::new(10, None)?, &operation).await?; + let mut entries = Vec::new(); + for commit in history.items { + let version = repository.snapshot(&Revision::Commit(commit.oid), &operation).await?; + let blame = version.blame(path, &operation).await?; + entries.push(json!({"commit":commit.oid.to_string(), "ranges":blame.ranges.iter().map(|range| json!({ + "start":range.start, "lines":range.lines, "commit":range.commit.oid.to_string(), + "source_path":range.source_path.as_bytes(), + })).collect::>()})); + } + Ok(json!({"entries":entries})) + } "blame" => { let blame = snapshot.blame(path, &operation).await?; Ok(json!({"ranges":blame.ranges.iter().map(|range| json!({ diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index fe10d1693..2abfd31d2 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -25,6 +25,7 @@ pub struct Measurements { read_bytes: AtomicU64, written_bytes: AtomicU64, active: AtomicU64, + comparison_work: [AtomicU64; 2], decoded_blocks: [AtomicU64; 7], decoded_bytes: [AtomicU64; 7], } @@ -35,6 +36,7 @@ struct Snapshot { outcomes: [u64; 9], read_bytes: u64, written_bytes: u64, + comparison_work: [u64; 2], decoded_blocks: [u64; 7], decoded_bytes: [u64; 7], } @@ -60,6 +62,7 @@ struct DecodeLayer(Arc); #[derive(Default)] struct DecodeEvent { format: Option, + cache_hit: Option, bytes: u64, } @@ -79,6 +82,11 @@ impl tracing::field::Visit for DecodeEvent { }; } } + fn record_bool(&mut self, field: &tracing::field::Field, value: bool) { + if field.name() == "cache_hit" { + self.cache_hit = Some(value); + } + } fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { if field.name() == "decoded_bytes" { self.bytes = value; @@ -88,11 +96,19 @@ impl tracing::field::Visit for DecodeEvent { impl tracing_subscriber::Layer for DecodeLayer { fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { - if event.metadata().target() != "crab_metadata::index_decode" { + let target = event.metadata().target(); + if target != "crab_metadata::index_decode" && target != "crab_remote_git::blame_comparison" + { return; } let mut decoded = DecodeEvent::default(); event.record(&mut decoded); + if target == "crab_remote_git::blame_comparison" { + if let Some(hit) = decoded.cache_hit { + self.0.comparison_work[usize::from(hit)].fetch_add(1, Ordering::Relaxed); + } + return; + } if let Some(format) = decoded.format { self.0.decoded_blocks[format].fetch_add(1, Ordering::Relaxed); self.0.decoded_bytes[format].fetch_add(decoded.bytes, Ordering::Relaxed); @@ -119,6 +135,10 @@ impl Measurements { .map(|counter| counter.load(Ordering::Relaxed)), read_bytes: self.read_bytes.load(Ordering::Relaxed), written_bytes: self.written_bytes.load(Ordering::Relaxed), + comparison_work: self + .comparison_work + .each_ref() + .map(|counter| counter.load(Ordering::Relaxed)), decoded_blocks: self .decoded_blocks .each_ref() @@ -192,6 +212,10 @@ impl Measurements { "backend_written_bytes": after.written_bytes.saturating_sub(before.written_bytes), "active_backend_operations": self.active.load(Ordering::Relaxed), "decoded_index_blocks": decoded, + "blame_comparisons": { + "misses": after.comparison_work[0].saturating_sub(before.comparison_work[0]), + "hits": after.comparison_work[1].saturating_sub(before.comparison_work[1]), + }, }) ); result @@ -251,9 +275,12 @@ mod tests { tracing::debug!(target:"crab_metadata::index_decode", index_format="capsule_visibility", decoded_bytes=31_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility", decoded_bytes=37_u64); tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility_pages", decoded_bytes=43_u64); + tracing::debug!(target:"crab_remote_git::blame_comparison", cache_hit=false); + tracing::debug!(target:"crab_remote_git::blame_comparison", cache_hit=true); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); + assert_eq!(snapshot.comparison_work, [1, 1]); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), ([1, 1, 1, 1, 1, 1, 1], [41, 123, 67, 29, 31, 37, 43]) From 9880d5fc979b3d570f2e062b8885d164c27f9d2a Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 15:11:35 -0700 Subject: [PATCH 034/127] docs(blame): record native comparison proof --- .../docs/architecture/indexed-remote-reads.md | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 221cd7515..5df7cc18a 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1269,10 +1269,23 @@ contract tests pass, including caller/shutdown/deadline cancellation of graph and path pages. Strict Clippy passes for both changed owner crates with `--all-targets --no-deps -D warnings` and the publication feature. -The dedicated RustFS comparison run uses cache-enabled and disabled queries at +The dedicated RustFS comparison run passes cache-enabled and disabled queries at 100, 1,000 and 10,000 commits, with 20 process samples per cell and native Git -line-origin checks on ten successive snapshots. At 100 commits, all 20 paired -samples pass; cold comparison misses fall from 192 to 21 with 171 hits. The -larger cells remain running, so the complete acceptance gate and latency -conclusions remain pending. External receipts live in -`blame-comparison-live-r2`; hosted S3 latency qualification is still outstanding. +line-origin checks on ten successive snapshots. All 120 processes pass 60,040 +native line-origin comparisons. Cold comparison misses fall from 192 to 21 at +100 commits and from 191 to 20 at both larger sizes, with 171 hits per query. +Warm queries read zero origin bytes and all measured backend work drains. + +Cold query p50/p95 milliseconds, disabled versus enabled: + +| Commits | Disabled | Enabled | +| --- | --- | --- | +| 100 | 693.9 / 1130.4 | 578.0 / 748.4 | +| 1,000 | 2953.3 / 4075.9 | 2925.3 / 4214.2 | +| 10,000 | 2418.9 / 2922.5 | 2235.7 / 2665.1 | + +These debug processes share RustFS with the large-file qualification. The +deterministic work reduction is established; the mixed latency results do not +establish a production latency target. External receipts, binary fingerprint +and source hashes live in `blame-comparison-live-r2`. Hosted S3 latency +qualification remains outstanding. From 47f9f716052fde51f1843bc574cb1c35dcd9fd51 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 15:29:57 -0700 Subject: [PATCH 035/127] feat(metadata): page Xet pointer catalogs --- .../docs/architecture/indexed-remote-reads.md | 51 ++ crab/docs/design/capsule-xorbs-shards.md | 8 + crab/src/core/error.rs | 1 + crates/crab-metadata/README.md | 13 + .../src/capsule_protocol/pointer.rs | 40 +- .../src/capsule_protocol/pointer/paged.rs | 470 ++++++++++++++++++ crates/crab-metadata/src/error.rs | 6 + crates/crab-metadata/src/pages.rs | 17 +- 8 files changed, 575 insertions(+), 31 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/paged.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 5df7cc18a..104d6638f 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1289,3 +1289,54 @@ deterministic work reduction is established; the mixed latency results do not establish a production latency target. External receipts, binary fingerprint and source hashes live in `blame-comparison-live-r2`. Hosted S3 latency qualification remains outstanding. + +### Historical Xet catalog: page-format foundation in progress + +`prepare_delta` in the native push path and `PreparedReceive::upload_capsule` +in shared receive both reconstruct `CapsuleRepositoryView::pointer_catalog` +before selecting a few dependencies. File bindings are applied in publication +order; equal file sizes permit a later shard binding, while shard/xorb identity +conflicts fail. Chunk placement selection follows the lowest xorb hex identity +and then its first matching occurrence. That precedence must survive indexing. + +The next branch replaces the monolithic JSON catalog codec with `CRBPCP01` +authenticated page trees for files, shards, xorbs and reverse chunk placements. +Shard closures and positional xorb chunk lists have nested page roots, so a wide +dependency list cannot inflate one row or the bounded top-level descriptor. +Normal lookups will use authenticated misses and loaded dependency closure; +complete inspection retains body-hash, canonical-layout and full closure proof. +This branch only supplies the format foundation through existing real catalog +encode/decode callers. Foreground full scans are not removed yet. + +The codec preserves the original hex ordering rather than Xet's raw hash byte +ordering. The locked xet-core-structures 1.6.0 implementation encodes its hex +identities by little-endian u64 words; using raw bytes as ordered keys changes +placement precedence. A wide-catalog regression caught this during development. +All 19 local release tags lack the pointer/run/layered source modules. Retired +catalog JSON is rejected; no fallback or legacy format reader is introduced. +Payload-only capsule tests pass 91 cases; storage-enabled capsule tests pass +103 cases, including the new wide/adversarial catalog cases. The CLI compile +check passes through read, write, remote and VFS consumers. A separate official +Makefile installation is building for fresh-prefix RustFS qualification; the +older installed binary remains pinned to the active 10 GiB run. No full stage-6 +or live page-catalog feature claim yet. + +Run aggregation must retain capsule membership, not just merge every physical +catalog into one map. `assemble_layered_control_view` trims each ref's run history +after its compacted transaction, deduplicates root transactions and applies only +required journal capsules in validated publication order. A selected run can +therefore contain a checkpoint-covered prefix that must not override a newer +checkpoint file binding. A whole-run aggregate without that selection proof +would widen the catalog and change precedence. + +The next slice must capture authenticated catalog roots in run/checkpoint +footers, retain the exact admitted capsule positions, and batch requested keys +across those roots. Run-level key aggregation must retain enough occurrence +membership to select only admitted positions and preserve the existing global +capsule order. Shared immutable descriptors may be deduplicated only after +conflict validation; file bindings retain their consistent size and ordered +shard changes. Qualification must include a checkpoint cut through a compacted +run, overlapping multi-ref transactions and incomparable ref histories, besides +the fixed-recipe/increasing-history request and decode bounds. Root frontiers +allow eight runs, ref frontiers 64 and each run 512 capsules; reading each +capsule separately would retain excessive lookup amplification. diff --git a/crab/docs/design/capsule-xorbs-shards.md b/crab/docs/design/capsule-xorbs-shards.md index 0f100867a..5b9f69dd5 100644 --- a/crab/docs/design/capsule-xorbs-shards.md +++ b/crab/docs/design/capsule-xorbs-shards.md @@ -54,6 +54,14 @@ remain in canonical external xorbs and shards. The primary surfaces are `crates/crab-metadata/src/capsule_protocol/`, `crab/src/git/capsule_push.rs`, and the shared file-index lookup. +Catalog sections now use the development `CRBPCP01` paged encoding rather than +one JSON map. File/shard/xorb directories, reverse chunk placements and nested +dependency lists are authenticated independently. Complete readers still +materialize these sections; foreground batched lookup, checkpoint/run descriptor +capture and bounded delta compaction remain the next implementation slices. +No released tag in the 19-tag audit contains the retired catalog codec, so this +development format rejects it without a compatibility reader. + ## 2. Goals The implementation MUST: diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index fb8e468cc..d1f6ec727 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -2048,6 +2048,7 @@ impl From for CrabError { | crab_metadata::error::MetadataError::BrowseIndexRecord { .. } | crab_metadata::error::MetadataError::GitObjectIndexRecord { .. } | crab_metadata::error::MetadataError::CapsuleVisibilityRecord { .. } + | crab_metadata::error::MetadataError::PointerCatalogRecord { .. } | crab_metadata::error::MetadataError::PlanAlreadyAttempted { .. } | crab_metadata::error::MetadataError::RefJournalCommitUncertain { .. } | crab_metadata::error::MetadataError::ManifestCommitUncertain { .. }) => { diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 6e5a1f603..e413e110c 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -46,6 +46,19 @@ Catalog readers resolve only ref heads in that root's authority epoch, before loading multi-ref activation records. Restore can leave retired heads in storage; their checkpoint positions and dependencies must not enter the restored catalog. Current-epoch activation and checkpoint-chain errors still fail closed. +External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor +commits file, shard, xorb and reverse-chunk directories; nested shard closures and +ordered xorb chunk lists are paged too. Keys preserve canonical lowercase Xet hex +ordering, including the previous lowest-xorb/first-occurrence placement rule. +Complete inspection verifies the body hash, complete dependency closure and +canonical page layout. Delta inspection defers base-owned closure to `apply`, +which retains later file bindings at equal size and rejects immutable shard/xorb +conflicts. Encoded catalogs are limited to 2 GiB and descriptors to 4 KiB. +The retired monolithic JSON encoding was absent from all 19 audited release tags +and is rejected. This format foundation still uses complete materialization in +foreground callers; lazy lookup and bounded historical amplification remain +unfinished and must not be inferred from the page representation alone. + `v2/browse-indexes` is a bounded, mutable derived record, not ref authority. Its version 3 record binds a durable immutable commit graph, optional complete physical object lookup and optional complete path attribution to the exact diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 0050002ac..0cdf6156b 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -9,6 +9,8 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; +mod paged; + /// One file identity and the canonical shard that reconstructs it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -208,53 +210,33 @@ impl PointerCatalog { self.validate(true, false) } - /// Canonically encode and validate this catalog. + /// Encode this complete catalog as bounded authenticated page trees. + /// + /// Rejects incomplete dependencies, noncanonical hash keys and oversized catalogs. pub fn encode(&self) -> Result { self.encode_with_validation(true) } - /// Canonically encode a delta whose dependencies may come from its base catalog. + /// Encode paged delta metadata whose dependencies may come from its base catalog. pub fn encode_delta(&self) -> Result { self.encode_with_validation(false) } fn encode_with_validation(&self, require_complete_closure: bool) -> Result { self.validate(require_complete_closure, false)?; - serde_json::to_vec(self) - .map(Bytes::from) - .map_err(|source| MetadataError::Internal(format!("pointer catalog encode: {source}"))) + paged::encode(self) } - /// Decode canonical bytes and validate their complete dependency closure. + /// Authenticate canonical pages and validate their complete dependency closure. pub fn decode(bytes: &[u8]) -> Result { - let catalog: Self = - serde_json::from_slice(bytes).map_err(|source| MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!("invalid JSON: {source}"), - })?; - let canonical = serde_json::to_vec(&catalog).map_err(|source| { - MetadataError::Internal(format!("pointer catalog re-encode: {source}")) - })?; - if canonical != bytes { - return Err(corrupt("catalog is not canonically encoded")); - } + let catalog = paged::decode(bytes)?; catalog.validate(true, true)?; Ok(catalog) } - /// Decode a canonical delta and defer base-dependent closure checks to apply. + /// Authenticate canonical delta pages, deferring base-owned closure checks to apply. pub fn decode_delta(bytes: &[u8]) -> Result { - let catalog: Self = - serde_json::from_slice(bytes).map_err(|source| MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!("invalid JSON: {source}"), - })?; - let canonical = serde_json::to_vec(&catalog).map_err(|source| { - MetadataError::Internal(format!("pointer catalog re-encode: {source}")) - })?; - if canonical != bytes { - return Err(corrupt("catalog is not canonically encoded")); - } + let catalog = paged::decode(bytes)?; catalog.validate(false, true)?; Ok(catalog) } diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs new file mode 100644 index 000000000..571d8b833 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -0,0 +1,470 @@ +//! Canonical pages for checkpoint catalogs and publication deltas. +use super::*; +use crate::pages::{Kind, PageRef, Rows, Writer}; + +const MAGIC: &[u8; 8] = b"CRBPCP01"; +const TRAILER_BYTES: usize = 12; +const MAX_DESCRIPTOR_BYTES: usize = 4096; +const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Descriptor { + version: u32, + body_bytes: u64, + body_hash: [u8; 32], + files: Option, + shards: Option, + xorbs: Option, + chunks: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ShardRow { + encoded_size: u64, + xorbs: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct XorbRow { + encoded_size: u64, + body_digest: [u8; 32], + chunks: PageRef, +} + +fn json(value: &T) -> Result> { + serde_json::to_vec(value).map_err(|source| MetadataError::PointerCatalogRecord { source }) +} + +fn from_json(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|source| MetadataError::PointerCatalogRecord { source }) +} + +fn hash(value: &str) -> Result<[u8; 32]> { + let bytes = value.as_bytes(); + // Binary keys cannot distinguish hex aliases. Reject them instead of silently + // merging two descriptors that were distinct in the caller's map. + if bytes.len() != 64 + || bytes + .iter() + .any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(byte)) + { + return Err(contract_error( + "catalog hashes must be lowercase canonical hex", + )); + } + // Encode text order directly: Xet's raw hash bytes reverse each u64 word. + // Using those bytes as keys would change the original BTreeMap precedence. + let digit = |byte: u8| match byte { + b'0'..=b'9' => byte - b'0', + _ => byte - b'a' + 10, + }; + Ok(std::array::from_fn(|index| { + (digit(bytes[index * 2]) << 4) | digit(bytes[index * 2 + 1]) + })) +} + +fn hex(bytes: &[u8]) -> Result { + if bytes.len() != 32 { + return Err(corrupt("catalog hash has an invalid byte length")); + } + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut value = String::with_capacity(64); + for byte in bytes { + value.push(DIGITS[(byte >> 4) as usize] as char); + value.push(DIGITS[(byte & 15) as usize] as char); + } + Ok(value) +} + +fn tree(writer: &mut Writer, kind: Kind, rows: Rows) -> Result> { + if rows.is_empty() { + return Ok(None); + } + let root = writer.tree(kind, rows)?; + if writer.bytes.len() as u64 > MAX_CATALOG_BYTES { + return Err(contract_error("catalog exceeds its encoded byte bound")); + } + Ok(Some(root)) +} + +pub(super) fn encode(catalog: &PointerCatalog) -> Result { + // Admit the unavoidable row payload before duplicating any caller-owned + // lists. Page headers and directories are checked again after encoding. + let minimum = catalog.xorbs.values().try_fold(0u64, |bytes, xorb| { + bytes.checked_add((xorb.chunks.len() as u64).checked_mul(44)?) + }); + let minimum = minimum.and_then(|minimum| { + catalog.shards.values().try_fold(minimum, |bytes, shard| { + bytes.checked_add((shard.xorb_hashes.len() as u64).checked_mul(36)?) + }) + }); + let minimum = + minimum.and_then(|bytes| bytes.checked_add((catalog.files.len() as u64).checked_mul(76)?)); + if minimum.is_none_or(|bytes| bytes > MAX_CATALOG_BYTES) { + return Err(contract_error( + "catalog rows exceed their encoded byte bound", + )); + } + let mut writer = Writer::default(); + let mut chunks = BTreeMap::new(); + let mut xorbs = Vec::new(); + for (xorb_hash, entry) in &catalog.xorbs { + let xorb_hash = hash(xorb_hash)?; + let mut rows = Vec::new(); + for (index, chunk) in entry.chunks.iter().enumerate() { + let index = u32::try_from(index) + .map_err(|_| contract_error("xorb chunk position exceeds its bound"))?; + let chunk_hash = hash(&chunk.hash)?; + let mut value = chunk_hash.to_vec(); + value.extend_from_slice(&chunk.uncompressed_size.to_be_bytes()); + rows.push((index.to_be_bytes().to_vec(), value)); + // Match the original BTreeMap scan: lowest xorb identity wins, then + // its first occurrence. Repeated chunks retain their full recipe. + chunks.entry(chunk_hash.to_vec()).or_insert_with(|| { + let mut value = xorb_hash.to_vec(); + value.extend_from_slice(&index.to_be_bytes()); + value.extend_from_slice(&chunk.uncompressed_size.to_be_bytes()); + value + }); + } + let chunks = tree(&mut writer, Kind::CatalogXorbChunks, rows)? + .ok_or_else(|| contract_error("catalog xorb contains no chunks"))?; + let row = XorbRow { + encoded_size: entry.encoded_size, + body_digest: hash(&entry.body_digest)?, + chunks, + }; + xorbs.push((xorb_hash.to_vec(), json(&row)?)); + } + let xorbs = tree(&mut writer, Kind::CatalogXorbs, xorbs)?; + let chunks = tree( + &mut writer, + Kind::CatalogChunks, + chunks.into_iter().collect(), + )?; + let mut shards = Vec::new(); + for (shard_hash, entry) in &catalog.shards { + let rows = entry + .xorb_hashes + .iter() + .map(|value| Ok((hash(value)?.to_vec(), Vec::new()))) + .collect::>()?; + let row = ShardRow { + encoded_size: entry.encoded_size, + xorbs: tree(&mut writer, Kind::CatalogShardXorbs, rows)?, + }; + shards.push((hash(shard_hash)?.to_vec(), json(&row)?)); + } + let shards = tree(&mut writer, Kind::CatalogShards, shards)?; + let files = catalog + .files + .iter() + .map(|(file_hash, entry)| { + let mut value = entry.size.to_be_bytes().to_vec(); + value.extend_from_slice(&hash(&entry.shard_hash)?); + Ok((hash(file_hash)?.to_vec(), value)) + }) + .collect::>()?; + let files = tree(&mut writer, Kind::CatalogFiles, files)?; + let descriptor = Descriptor { + version: 1, + body_bytes: writer.bytes.len() as u64, + body_hash: *blake3::hash(&writer.bytes).as_bytes(), + files, + shards, + xorbs, + chunks, + }; + let header = json(&descriptor)?; + if header.len() > MAX_DESCRIPTOR_BYTES { + return Err(contract_error("catalog descriptor exceeds its byte bound")); + } + writer.bytes.extend_from_slice(&header); + crate::pages::put_u32(&mut writer.bytes, header.len())?; + writer.bytes.extend_from_slice(MAGIC); + if writer.bytes.len() as u64 > MAX_CATALOG_BYTES { + return Err(contract_error("catalog exceeds its encoded byte bound")); + } + Ok(writer.bytes.into()) +} + +pub(super) fn decode(bytes: &[u8]) -> Result { + let descriptor = Descriptor::decode(bytes)?; + let body = &bytes[..descriptor.body_bytes as usize]; + if blake3::hash(body).as_bytes() != &descriptor.body_hash { + return Err(corrupt("catalog page body hash differs")); + } + let mut visited = BTreeMap::new(); + let mut scan = |root: Option<&PageRef>, kind| -> Result { + root.map(|root| crate::pages::scan(body, root, kind, "pointer catalog pages", &mut visited)) + .transpose() + .map(Option::unwrap_or_default) + }; + let mut catalog = PointerCatalog::new(); + for (key, value) in scan(descriptor.files.as_ref(), Kind::CatalogFiles)? { + if value.len() != 40 { + return Err(corrupt("catalog file row has an invalid byte length")); + } + let size = u64::from_be_bytes( + value[..8] + .try_into() + .map_err(|_| corrupt("file size is truncated"))?, + ); + catalog.insert_file(hex(&key)?, FileCatalogEntry::new(size, hex(&value[8..])?))?; + } + for (key, value) in scan(descriptor.shards.as_ref(), Kind::CatalogShards)? { + let row: ShardRow = from_json(&value)?; + let hashes = scan(row.xorbs.as_ref(), Kind::CatalogShardXorbs)? + .into_iter() + .map(|(hash, value)| { + if !value.is_empty() { + return Err(corrupt("shard xorb row has unexpected data")); + } + hex(&hash) + }) + .collect::>()?; + catalog.insert_shard(hex(&key)?, ShardCatalogEntry::new(row.encoded_size, hashes))?; + } + for (key, value) in scan(descriptor.xorbs.as_ref(), Kind::CatalogXorbs)? { + let row: XorbRow = from_json(&value)?; + let chunks = scan(Some(&row.chunks), Kind::CatalogXorbChunks)? + .into_iter() + .enumerate() + .map(|(index, (key, value))| { + if key != (index as u32).to_be_bytes() || value.len() != 36 { + return Err(corrupt("xorb chunk row has invalid position or size")); + } + let size = u32::from_be_bytes( + value[32..] + .try_into() + .map_err(|_| corrupt("chunk size is truncated"))?, + ); + Ok(XorbChunkEntry::new(hex(&value[..32])?, size)) + }) + .collect::>()?; + catalog.insert_xorb( + hex(&key)?, + XorbCatalogEntry::new(row.encoded_size, hex(&row.body_digest)?, chunks), + )?; + } + // Rebuild the reverse chunk index from authenticated xorb order. This also + // rejects unclaimed bytes, re-signed routing aliases and noncanonical rows. + if encode(&catalog)?.as_ref() != bytes { + return Err(corrupt("catalog pages are not canonical")); + } + Ok(catalog) +} + +impl Descriptor { + pub(super) fn decode(bytes: &[u8]) -> Result { + if bytes.len() < TRAILER_BYTES + || bytes.len() as u64 > MAX_CATALOG_BYTES + || bytes.get(bytes.len() - 8..) != Some(MAGIC.as_slice()) + { + return Err(corrupt("catalog trailer or encoded size is invalid")); + } + let end = bytes.len() - TRAILER_BYTES; + let length = u32::from_le_bytes( + bytes[end..end + 4] + .try_into() + .map_err(|_| corrupt("catalog descriptor length is truncated"))?, + ) as usize; + if length == 0 || length > MAX_DESCRIPTOR_BYTES { + return Err(corrupt("catalog descriptor exceeds its byte bound")); + } + let start = end + .checked_sub(length) + .ok_or_else(|| corrupt("catalog descriptor span is invalid"))?; + let descriptor: Self = from_json(&bytes[start..end])?; + if descriptor.body_bytes != start as u64 { + return Err(corrupt("catalog body span differs from its descriptor")); + } + descriptor.validate(bytes.len() as u64)?; + Ok(descriptor) + } + + fn validate(&self, length: u64) -> Result<()> { + let header = json(self)?; + if self.version != 1 + || header.len() > MAX_DESCRIPTOR_BYTES + || self + .body_bytes + .checked_add(header.len() as u64) + .and_then(|n| n.checked_add(TRAILER_BYTES as u64)) + != Some(length) + || length > MAX_CATALOG_BYTES + { + return Err(corrupt("catalog descriptor version or span is invalid")); + } + for (root, kind) in [ + (&self.files, Kind::CatalogFiles), + (&self.shards, Kind::CatalogShards), + (&self.xorbs, Kind::CatalogXorbs), + (&self.chunks, Kind::CatalogChunks), + ] { + if let Some(root) = root { + root.validate(kind, self.body_bytes)?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn catalog(xorb_count: usize, chunks_per_xorb: usize) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + let mut xorbs = Vec::new(); + for index in 0..xorb_count { + let xorb = format!("{index:064x}"); + let chunks = (0..chunks_per_xorb) + .map(|chunk| XorbChunkEntry::new(format!("{chunk:064x}"), 7)) + .collect(); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new(100, "a".repeat(64), chunks), + ) + .unwrap(); + xorbs.push(xorb); + } + let shard = "b".repeat(64); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(50, xorbs)) + .unwrap(); + catalog + .insert_file("c".repeat(64), FileCatalogEntry::new(7, shard)) + .unwrap(); + catalog + } + + #[test] + fn nested_chunk_and_shard_directories_round_trip_without_growing_the_descriptor() { + for (xorbs, chunks) in [(1, 2000), (2000, 1)] { + let catalog = catalog(xorbs, chunks); + let bytes = catalog.encode().unwrap(); + let descriptor = Descriptor::decode(&bytes).unwrap(); + assert!(json(&descriptor).unwrap().len() < MAX_DESCRIPTOR_BYTES); + assert_eq!(PointerCatalog::decode(&bytes).unwrap(), catalog); + } + } + + #[test] + fn chunk_index_selects_first_occurrence_in_lowest_xorb() { + let mut catalog = catalog(2, 3); + let xorb = catalog.xorbs.get_mut(&"0".repeat(64)).unwrap(); + xorb.chunks.push(xorb.chunks[0].clone()); + let bytes = catalog.encode().unwrap(); + let descriptor = Descriptor::decode(&bytes).unwrap(); + let body = &bytes[..descriptor.body_bytes as usize]; + let rows = crate::pages::scan( + body, + &descriptor.chunks.unwrap(), + Kind::CatalogChunks, + "chunks", + &mut BTreeMap::new(), + ) + .unwrap(); + assert!(rows.iter().all(|(_, value)| value[..32] == [0; 32])); + assert_eq!(&rows[0].1[32..36], &0u32.to_be_bytes()); + assert_eq!(rows.len(), 3); + } + + #[test] + fn catalogs_reject_tampering_and_truncated_descriptors() { + let bytes = catalog(2, 3).encode().unwrap(); + let mut tampered = bytes.to_vec(); + tampered[14] ^= 1; + assert!(PointerCatalog::decode(&tampered).is_err()); + for length in [0, 7, 11, bytes.len() - 1] { + assert!(PointerCatalog::decode(&bytes[..length]).is_err()); + } + } + + #[test] + fn resigned_chunk_index_cannot_disagree_with_xorb_order() { + let bytes = catalog(2, 3).encode().unwrap(); + let mut descriptor = Descriptor::decode(&bytes).unwrap(); + let root = descriptor.chunks.as_mut().unwrap(); + assert_eq!(root.height, 0); + let mut body = bytes[..descriptor.body_bytes as usize].to_vec(); + let start = root.offset as usize; + // First row: page header, key, value length, then xorb identity. + body[start + 14 + 32 + 4] ^= 1; + root.hash = *blake3::hash(&body[start..start + root.bytes as usize]).as_bytes(); + descriptor.body_hash = *blake3::hash(&body).as_bytes(); + let header = json(&descriptor).unwrap(); + body.extend_from_slice(&header); + crate::pages::put_u32(&mut body, header.len()).unwrap(); + body.extend_from_slice(MAGIC); + assert!(PointerCatalog::decode(&body).is_err()); + } + + #[test] + fn empty_catalog_is_canonical_and_retired_json_is_rejected() { + let catalog = PointerCatalog::new(); + assert_eq!( + PointerCatalog::decode(&catalog.encode().unwrap()).unwrap(), + catalog + ); + assert!(PointerCatalog::decode(&serde_json::to_vec(&catalog).unwrap()).is_err()); + } + + #[test] + fn binary_catalog_keys_reject_case_aliases() { + let mut catalog = catalog(1, 1); + let file = catalog.files.remove(&"c".repeat(64)).unwrap(); + catalog.files.insert("C".repeat(64), file); + assert!(catalog.encode().is_err()); + } + + #[test] + fn later_file_binding_preserves_immutable_descriptor_conflicts() { + let mut base = catalog(1, 1); + let file = "c".repeat(64); + let shard = "d".repeat(64); + let mut delta = PointerCatalog::new(); + delta + .insert_shard( + shard.clone(), + ShardCatalogEntry::new(90, vec!["0".repeat(64)]), + ) + .unwrap(); + delta + .insert_file(file.clone(), FileCatalogEntry::new(7, shard.clone())) + .unwrap(); + let delta = PointerCatalog::decode_delta(&delta.encode_delta().unwrap()).unwrap(); + base.apply(&delta).unwrap(); + let base = PointerCatalog::decode(&base.encode().unwrap()).unwrap(); + assert_eq!(base.files[&file].shard_hash(), shard); + for replace_shard in [false, true] { + let mut conflict = PointerCatalog::new(); + if replace_shard { + conflict + .insert_shard( + shard.clone(), + ShardCatalogEntry::new(91, vec!["0".repeat(64)]), + ) + .unwrap(); + } else { + conflict + .insert_xorb( + "0".repeat(64), + XorbCatalogEntry::new( + 101, + "a".repeat(64), + vec![XorbChunkEntry::new("0".repeat(64), 7)], + ), + ) + .unwrap(); + } + let conflict = PointerCatalog::decode_delta(&conflict.encode_delta().unwrap()).unwrap(); + assert!(base.clone().apply(&conflict).is_err()); + } + } +} diff --git a/crates/crab-metadata/src/error.rs b/crates/crab-metadata/src/error.rs index e3fb0b52d..814919a6d 100644 --- a/crates/crab-metadata/src/error.rs +++ b/crates/crab-metadata/src/error.rs @@ -32,6 +32,12 @@ pub enum MetadataError { #[source] source: serde_json::Error, }, + /// A pointer catalog page descriptor could not be encoded or decoded. + #[error("pointer catalog record encoding is invalid")] + PointerCatalogRecord { + #[source] + source: serde_json::Error, + }, /// A file lookup could not acquire process-wide execution capacity. #[cfg(feature = "file-index-reader")] #[error("file lookup admission closed")] diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 4e4a7d2c5..d0bfc229c 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -63,6 +63,12 @@ pub(crate) enum Kind { CheckpointTransitions = 17, CheckpointRefs = 18, CheckpointSets = 19, + CatalogFiles = 20, + CatalogShards = 21, + CatalogXorbs = 22, + CatalogChunks = 23, + CatalogXorbChunks = 24, + CatalogShardXorbs = 25, } impl Kind { @@ -73,7 +79,8 @@ impl Kind { | Self::CommitRecords | Self::CommitAncestryRoots | Self::RunAdmissionCounts - | Self::CheckpointTransitions => true, + | Self::CheckpointTransitions + | Self::CatalogXorbChunks => true, #[cfg(feature = "remote-index")] Self::PackSlots => true, _ => false, @@ -85,7 +92,12 @@ impl Kind { Self::Names | Self::CommitAncestryNodes | Self::VisibilityRefs - | Self::CheckpointRefs => 32, + | Self::CheckpointRefs + | Self::CatalogFiles + | Self::CatalogShards + | Self::CatalogXorbs + | Self::CatalogChunks + | Self::CatalogShardXorbs => 32, Self::CommitOids | Self::RunAdmission | Self::VisibilityChanges @@ -358,6 +370,7 @@ pub(crate) fn decode( Kind::RunAdmission | Kind::RunAdmissionCounts => "capsule_admission", Kind::VisibilityRefs | Kind::VisibilityChanges => "capsule_visibility", Kind::CheckpointObjects | Kind::CheckpointRefs | Kind::CheckpointSets | Kind::CheckpointTransitions => "checkpoint_visibility_pages", + Kind::CatalogFiles | Kind::CatalogShards | Kind::CatalogXorbs | Kind::CatalogChunks | Kind::CatalogXorbChunks | Kind::CatalogShardXorbs => "pointer_catalog_pages", _ => "path_state", }, decoded_bytes=bytes.len() as u64, "index page decoded"); From f858ab0809767008dff08f1d5bc3a455ac82b248 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 15:36:05 -0700 Subject: [PATCH 036/127] docs: record large-file publication evidence --- .../docs/architecture/indexed-remote-reads.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 104d6638f..cae28d7c8 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1250,6 +1250,26 @@ add and Git clean. It checks pointer staging, incremental upload, fresh-cache historical hydration and integrity after removing source authority. Results are pending; earlier v5 or small-file receipts do not qualify this run. +The four publications in `large-v6-live-r1` have now passed. These are single +release-binary observations sharing RustFS with other qualification/build work, +not 20-sample performance cells: + +| Version / add path | Add seconds | Push seconds | New xorb bytes | +| --- | --- | --- | --- | +| 0 / native | 460.4 | 359.7 | 10,744,916,848 | +| 1 / Git clean | 808.4 | 6.7 | 2,216,724 | +| 2 / native | 156.9 | 6.7 | 2,133,870 | +| 3 / Git clean | 498.5 | 8.9 | 2,215,428 | + +After version 1 staging, segments retain 10,739,857,220 bytes alongside +10,744,916,848 bytes in prepared push plans. This reproduces the duplicate local +payload problem when switching staging paths; efficient incremental remote +uploads do not establish unified local staging. Source authority and source +cache have been removed. Version 0 cold hydration of both 10 GiB files completes +in 604.4 seconds and both SHA-256 checks pass. Versions 1–3 cold reconstruction +and final deep remote integrity remain running. These receipts qualify the +frozen v6 binary, not the new paged pointer-catalog encoding. + ### Blame comparison cache From d5aeef97d14ade337de9f89875f18d57f9b4af00 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 15:38:20 -0700 Subject: [PATCH 037/127] test(metadata): hydrate from catalog checkpoints --- .../e2e/run_add_commit_push_rustfs_smoke.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 82c378e42..d8a27e5c8 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -1685,6 +1685,46 @@ def run_case(self, case_name: str, use_crab_add: bool) -> None: {"original_sha256": original_hash}, ) + # Reopen from a new cache after catalog compaction; the original clone + # can otherwise hide a checkpoint reader that only works with warm data. + self.run_crab( + repo, ["metadb", "owner", "--once", "--jsonl"], + name=f"{case_name} checkpoint catalog", timeout=self.args.push_timeout, + ) + diagnosis = self.run_crab( + repo, ["metadb", "diagnose", "--json"], + name=f"{case_name} checkpoint diagnosis", + ) + self.check( + f"{case_name}-checkpoint-present", + json.loads(self.read_stdout(diagnosis))["data"]["capsule"]["checkpoint_present"], + ) + checkpoint_clone = self.run_root / case_name / "checkpoint-clone" + checkpoint_cache = self.run_root / case_name / "checkpoint-cache" + self.check(f"{case_name}-checkpoint-cache-starts-empty", not checkpoint_cache.exists()) + checkpoint_env = {"CRAB_CACHE_DIR": str(checkpoint_cache)} + self.run_cmd( + f"{case_name} clone checkpoint", + [self.crab_bin, "clone", remote_url, str(checkpoint_clone), "--jsonl"], + self.run_root, extra_env=checkpoint_env, + ) + self.check( + f"{case_name}-checkpoint-clone-tip-matches", + self.rev_parse(checkpoint_clone, "HEAD") == source_tip, + ) + self.run_crab( + checkpoint_clone, ["hydrate", "--all"], + name=f"{case_name} cold checkpoint hydration", extra_env=checkpoint_env, + ) + self.check( + f"{case_name}-checkpoint-model-byte-identical", + sha256_file(checkpoint_clone / "model.bin") == original_hash, + ) + self.check( + f"{case_name}-checkpoint-duplicate-byte-identical", + sha256_file(checkpoint_clone / "duplicate.bin") == original_hash, + ) + sqlite_files = self.sqlite_cache_files() self.check( f"{case_name}-sqlite-cache-used", From 56f32b306f0fd5a1ecd26893238d2ee37ad8de62 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 16:09:16 -0700 Subject: [PATCH 038/127] docs: record complete 10 GiB reconstruction proof --- crab/docs/architecture/indexed-remote-reads.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index cae28d7c8..aa819a4db 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1243,12 +1243,12 @@ rebuild, the same retained OID fails with `Revision { reason: NotReachable }` an zero active backend operations. This checks the reader's visibility boundary, separately from Git-helper upload-pack's default want policy. -The existing large-file harness is running against the isolated Makefile-installed +The existing large-file harness passed against the isolated Makefile-installed v6 binary in a fresh RustFS bucket. It uses two actual 10 GiB files with deterministic entropy, 5,000 code files and four versions, alternating native add and Git clean. It checks pointer staging, incremental upload, fresh-cache -historical hydration and integrity after removing source authority. Results are -pending; earlier v5 or small-file receipts do not qualify this run. +historical hydration and integrity after removing source authority. All 49 checks +pass; earlier v5 or small-file receipts are separate evidence. The four publications in `large-v6-live-r1` have now passed. These are single release-binary observations sharing RustFS with other qualification/build work, @@ -1265,10 +1265,13 @@ After version 1 staging, segments retain 10,739,857,220 bytes alongside 10,744,916,848 bytes in prepared push plans. This reproduces the duplicate local payload problem when switching staging paths; efficient incremental remote uploads do not establish unified local staging. Source authority and source -cache have been removed. Version 0 cold hydration of both 10 GiB files completes -in 604.4 seconds and both SHA-256 checks pass. Versions 1–3 cold reconstruction -and final deep remote integrity remain running. These receipts qualify the -frozen v6 binary, not the new paged pointer-catalog encoding. +cache were removed before reconstruction. Cold hydration of both 10 GiB files +takes 604.4 / 549.4 / 488.0 / 502.8 seconds for versions 0–3. All eight SHA-256 +checks pass. Strict native Git fsck and deep remote integrity pass, with zero +errors and repair failures. The binary fingerprint remains unchanged. Receipts +live under `qualification-large-v6/large-v6-live-r1`; the report records the +binary, source revision, source patch and harness fingerprints. These receipts +qualify the frozen v6 binary, not the new paged pointer-catalog encoding. ### Blame comparison cache From a1b1486f77998a7b45e6276bfd627df60cc94873 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 16:17:51 -0700 Subject: [PATCH 039/127] docs: record live paged catalog qualification --- crab/docs/architecture/indexed-remote-reads.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index aa819a4db..585c8842b 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1344,6 +1344,19 @@ Makefile installation is building for fresh-prefix RustFS qualification; the older installed binary remains pinned to the active 10 GiB run. No full stage-6 or live page-catalog feature claim yet. +The Make-generated full-feature artifact now passes the separate +`pointer-pages-live-fuse-r1` RustFS qualification: 96 commands and 71 checks. +Both native add/push and Git-clean/helper push pass initial publication, +fresh-cache checkpoint clone/hydration, alternate-path edits after checkpointing, +a second checkpoint, exact edited/duplicate hashes and deep remote integrity. +The captured checkpoint bodies contain `CRBPCP01`. The test pins the artifact +SHA-256 `10fa04219b2099cc1940f39225500c8e27c7c52b62e424c936c8df77278504b1` +and verifies unchanged Rust/manifest fingerprints. Its version metadata names +the dirty build's base `0ac1c27d4c8`; this is not a clean-head release receipt. +The final installed no-FUSE binary is still building and needs separate proof. +This establishes the new format's live integration for these workflows; full +catalog scans, run aggregation and the remaining stage-6 gates remain unfinished. + Run aggregation must retain capsule membership, not just merge every physical catalog into one map. `assemble_layered_control_view` trims each ref's run history after its compacted transaction, deduplicates root transactions and applies only From eb8234ff0b5a2d81098b21e72ae45aba8e2b3ebd Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 16:31:40 -0700 Subject: [PATCH 040/127] docs: qualify installed paged catalog binary --- crab/docs/architecture/indexed-remote-reads.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 585c8842b..37fda1af2 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1340,9 +1340,9 @@ catalog JSON is rejected; no fallback or legacy format reader is introduced. Payload-only capsule tests pass 91 cases; storage-enabled capsule tests pass 103 cases, including the new wide/adversarial catalog cases. The CLI compile check passes through read, write, remote and VFS consumers. A separate official -Makefile installation is building for fresh-prefix RustFS qualification; the -older installed binary remains pinned to the active 10 GiB run. No full stage-6 -or live page-catalog feature claim yet. +Makefile installation completed for fresh-prefix RustFS qualification. The +older installed binary remains pinned to its separate 10 GiB receipts. This +format foundation does not establish the complete stage-6 performance gate. The Make-generated full-feature artifact now passes the separate `pointer-pages-live-fuse-r1` RustFS qualification: 96 commands and 71 checks. @@ -1353,7 +1353,8 @@ The captured checkpoint bodies contain `CRBPCP01`. The test pins the artifact SHA-256 `10fa04219b2099cc1940f39225500c8e27c7c52b62e424c936c8df77278504b1` and verifies unchanged Rust/manifest fingerprints. Its version metadata names the dirty build's base `0ac1c27d4c8`; this is not a clean-head release receipt. -The final installed no-FUSE binary is still building and needs separate proof. +The final installed no-FUSE binary separately passes the same 96-command, +71-check suite in `pointer-pages-live-r1`, with its own binary fingerprint. This establishes the new format's live integration for these workflows; full catalog scans, run aggregation and the remaining stage-6 gates remain unfinished. From 2399d9baf3c6613707a7f0778c838cac65e7a326 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 16:47:31 -0700 Subject: [PATCH 041/127] fix(metadata): preserve compacted capsule identity --- .../docs/architecture/indexed-remote-reads.md | 40 +++++ crates/crab-metadata/README.md | 8 + .../crab-metadata/src/capsule_protocol/run.rs | 83 ++++++++- crates/crab-read/README.md | 6 +- crates/crab-read/src/capsule_protocol.rs | 163 ++++++++++++++++++ 5 files changed, 294 insertions(+), 6 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 37fda1af2..e9f4a2238 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1377,3 +1377,43 @@ run, overlapping multi-ref transactions and incomparable ref histories, besides the fixed-recipe/increasing-history request and decode bounds. Root frontiers allow eight runs, ref frontiers 64 and each run 512 capsules; reading each capsule separately would retain excessive lookup amplification. + +### Shared capsule identity across compaction origins + +The catalog-source audit found a prerequisite defect in the unmerged capsule +stack. The same multi-ref capsule can be retained as a leaf for one ref and +inside a compacted run for another. Control construction copied the entire +run's pack directory into every capsule and compared visibility readers using +their physical origins. Complete reads deduplicated identical capsule bytes; +metadata and paged Git views instead reported conflicting transaction content. + +`compacted-origin-live-r1` reproduces this through real Git and RustFS: publish +one atomic main/feature update, then 31 feature updates to trigger the existing +32-way run compaction. The feature tip publishes and deep remote fsck passes, +but `metadb owner --once` fails with `CRAB-E0020` at ref-head reconciliation. +The focused Rust regression fails with the same conflict before the fix. + +The correction assigns only each capsule's own pack members, normalizes their +logical ranges to capsule-relative coordinates and leaves transport ranges on +the run. Control equality retains capsule/transaction/catalog/pack evidence and +the authenticated visibility section hash, length and page roots, excluding +visibility materialization and transport origin. This also removes repeated +whole-run pack-directory clones. No serialized format or migration changes. + +The regression passes after the change, checking complete, metadata and paged +Git views, pack counts/bytes and snapshot parity. A contradictory capsule that +claims the same transaction remains rejected in all three views. All 103 +storage-enabled metadata capsule tests and 48 read capsule tests pass; strict +owner-crate Clippy passes with storage and remote-index enabled. + +`capsule-origin-fixed-live-r1` uses the rebuilt `qualify_remote` executable +(SHA-256 `969b82482c3b1431bce044423e1953624860a33cd57c94d334bebbb13cdc798c`) +against that same real Git/RustFS repository. Canonical browse-index maintenance +now publishes physical and path-attribution readiness. Ten operations, each cold +and warm, match native Git for the captured main tip: revision, commit, tree, +blob, object point/batch, history, path history, tree attribution and every blame +line. All measured backend operations drain before completion. Receipts live +outside the checkout under `capsule-origin-fixed-live-r1/` in the dedicated +qualification directory. The qualifier exercises the shared metadata reader +used by maintenance; a newly installed CLI `metadb owner` invocation and broader +CI remain pending. No indexed-catalog performance claim follows from this fix. diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index e413e110c..07a421e59 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -46,6 +46,14 @@ Catalog readers resolve only ref heads in that root's authority epoch, before loading multi-ref activation records. Restore can leave retired heads in storage; their checkpoint positions and dependencies must not enter the restored catalog. Current-epoch activation and checkpoint-chain errors still fail closed. +Run controls retain each capsule's own pack directory in capsule-relative +coordinates. The run directory keeps physical transport coordinates separately. +Logical capsule equality compares authenticated visibility section identities +and page roots rather than cache state or transport origin. Identical shared +capsules can therefore appear in differently compacted ref frontiers without +inflating pack counts or appearing to conflict; differing payload commitments +remain rejected. + External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and ordered xorb chunk lists are paged too. Keys preserve canonical lowercase Xet hex diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index f9f6f9d59..805e88472 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -110,6 +110,7 @@ pub struct CapsuleRunControl { #[derive(Debug, Clone, PartialEq, Eq)] pub struct CapsuleControlLocation { hash: String, + capsule_range: std::ops::Range, transaction_id: String, base_root_digest: String, transaction: PackRange, @@ -122,24 +123,42 @@ pub struct CapsuleControlLocation { /// Transaction and metadata sections needed to validate a run without reading /// its Git/file payload sections. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct CapsuleControl { hash: String, transaction_id: String, base_root_digest: String, transaction: CapsuleTransaction, visibility: Option, + visibility_identity: Option<(u64, String, super::visibility::VisibilityPages)>, catalog: Option, git_packs: Vec, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] enum ControlVisibility { Complete(std::sync::Arc), #[cfg(feature = "storage")] Paged(std::sync::Arc), } +impl PartialEq for CapsuleControl { + fn eq(&self, other: &Self) -> bool { + // Transport origins and visibility materialization are not capsule + // identity. Compare the authenticated section commitment and roots so + // compaction can relocate bytes without hiding conflicting evidence. + self.hash == other.hash + && self.transaction_id == other.transaction_id + && self.base_root_digest == other.base_root_digest + && self.transaction == other.transaction + && self.visibility_identity == other.visibility_identity + && self.catalog == other.catalog + && self.git_packs == other.git_packs + } +} + +impl Eq for CapsuleControl {} + impl CapsuleRunControl { /// Decode complete control and admission bytes from the authenticated suffix. pub fn decode_suffix( @@ -330,6 +349,11 @@ impl CapsuleRunControl { )?; controls.push(CapsuleControlLocation { hash: location.hash.clone(), + capsule_range: location.offset + ..location + .offset + .checked_add(location.length) + .ok_or_else(|| corrupt("capsule run range overflowed"))?, transaction_id: location.transaction_id.clone(), base_root_digest: location.base_root_digest.clone(), transaction: transaction_range, @@ -580,9 +604,11 @@ impl CapsuleRunControl { external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, visibility: impl Fn(&CapsuleControlLocation) -> Result>, ) -> Result> { + let pack_directories = self.capsule_pack_directories()?; self.capsules .iter() - .map(|location| { + .zip(pack_directories) + .map(|(location, git_packs)| { verify_control_section( Some(&location.transaction), Some(&location.transaction_bytes), @@ -615,12 +641,59 @@ impl CapsuleRunControl { base_root_digest: location.base_root_digest.clone(), transaction, visibility, + visibility_identity: location + .visibility + .as_ref() + .zip(location.visibility_pages.as_ref()) + .map(|(section, pages)| { + (section.length(), section.blake3().to_owned(), pages.clone()) + }), catalog, - git_packs: self.git_packs.clone(), + git_packs, }) }) .collect() } + + #[cfg(any(feature = "storage", test))] + fn capsule_pack_directories(&self) -> Result>> { + let mut directories = vec![Vec::new(); self.capsules.len()]; + for member in &self.git_packs { + let position = self + .capsules + .partition_point(|capsule| capsule.capsule_range.end <= member.pack().offset()); + let capsule = self + .capsules + .get(position) + .ok_or_else(|| corrupt("Git pack has no owning capsule"))?; + let relative = |section: &PackRange| -> Result { + let offset = section + .offset() + .checked_sub(capsule.capsule_range.start) + .ok_or_else(|| corrupt("Git section starts before its owning capsule"))?; + if section + .offset() + .checked_add(section.length()) + .is_none_or(|end| end > capsule.capsule_range.end) + { + return Err(corrupt("Git section escapes its owning capsule")); + } + PackRange::from_parts(offset, section.length(), section.blake3().to_owned()) + }; + // Physical run descriptors stay run-relative. Logical controls own + // only their capsule's members, normalized to its immutable bytes. + directories[position].push(PackMemberDescriptor::new( + relative(member.pack())?, + relative(member.index())?, + relative(member.reverse_index())?, + relative(member.locator())?, + member.git_checksum(), + member.object_count(), + member.external_delta_bases().to_vec(), + )?); + } + Ok(directories) + } } impl CapsuleControl { @@ -692,7 +765,7 @@ impl CapsuleControl { self.catalog.as_ref() } - /// Return the authenticated pack directory for this capsule. + /// Return this capsule's authenticated pack directory with capsule-relative ranges. pub fn git_packs(&self) -> &[PackMemberDescriptor] { &self.git_packs } diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index db66ae725..fe038adf8 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -48,7 +48,11 @@ enumeration are still eager; total bounded opening remains unqualified. Full-capsule and control-only views share borrowed visibility readiness/application checks. Canonical transaction decoding guarantees unique ref names; matched-row counts reject extra evidence without copying the per-object edit lists. -Full and paged controls use one ref-ordering loop. Paged closure layers preserve +Full and paged controls use one ref-ordering loop and preserve capsule identity +across compaction origins: logical pack metadata is +capsule-relative, while transport ranges stay on the captured run. Shared +transactions deduplicate without discarding conflicting content evidence. +Paged closure layers preserve the inherited base when its source ref later moves, and dictionary retention never supplies visibility for a new ref. Initial rooted additions use their authenticated count; differing actual delta additions/removals remain admitted diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 8dfb7066a..5dc4594bc 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -6517,6 +6517,169 @@ mod tests { } } + #[tokio::test] + async fn metadata_views_deduplicate_shared_capsules_across_compacted_origins() { + for conflicting in [false, true] { + let inner = Arc::new(InMemory::new()); + seed_prepared_multi_ref(inner.clone(), true, true).await; + let store = Store::new(inner); + let router = StoreLayout::new(store.clone(), "repositories/test".to_owned()); + let root = load_root(&router).await.unwrap(); + let (heads, active) = capture_ref_heads(&router, root.record().root()) + .await + .unwrap(); + let feature = heads + .iter() + .find(|head| head.ref_name() == "refs/heads/feature") + .unwrap(); + let first_pointer = &feature.visible(&active).frontier()[0]; + let first = crab_metadata::capsule_protocol::load_capsule_run(&router, first_pointer) + .await + .unwrap(); + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/feature", + Some("3".repeat(40)), + Some("6".repeat(40)), + None, + )], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/feature".to_owned(), + GitVisibilityEdit::from_delta_objects( + Some("3".repeat(40)), + "6".repeat(40), + vec!["6".repeat(40)], + vec!["3".repeat(40)], + ), + )])) + .unwrap(); + let second = CapsuleRun::leaf( + Capsule::build( + &transaction, + vec![ + CapsuleGitPack::new( + Bytes::from_static(b"PACK next feature metadata fixture"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "7".repeat(40), + 1, + ) + .unwrap(), + ], + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + )], + ) + .unwrap(), + ) + .unwrap(); + let first = if conflicting { + let capsule = &first.capsules()[0]; + CapsuleRun::leaf( + Capsule::build( + &capsule.transaction().unwrap(), + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + capsule + .visibility_delta() + .unwrap() + .unwrap() + .encode() + .unwrap(), + )], + ) + .unwrap(), + ) + .unwrap() + } else { + first + }; + let compacted = CapsuleRun::compact(vec![first, second]).unwrap(); + store + .put( + &router.capsule_path(compacted.hash()), + compacted.bytes().clone(), + ) + .await + .unwrap(); + let pointer = CapsulePointer::new( + compacted.hash(), + compacted.bytes().len() as u64, + compacted.control_offset(), + compacted.control_size(), + compacted.footer_hash(), + compacted.level(), + compacted.transaction_ids(), + compacted.newest_base_root_digest(), + ) + .unwrap(); + let next = feature + .successor_state( + &active, + Some("6".repeat(40)), + None, + transaction.id().unwrap(), + vec![pointer], + ) + .unwrap(); + store + .put_overwrite( + &router.capsule_ref_head_path( + &crab_metadata::capsule_protocol::capsule_ref_name_key( + "refs/heads/feature", + ), + ), + feature.commit(next).unwrap().encode().unwrap(), + ) + .await + .unwrap(); + + // Main retains the shared leaf; feature now contains that exact capsule + // at a different physical origin. Compaction cannot change its identity. + let complete = + open_view_from_root_with_control(&router, root.clone(), TEST_LIMITS).await; + let others = [ + open_view_from_root_with_metadata(&router, root.clone(), TEST_LIMITS).await, + open_git_view_from_root(&router, root, TEST_LIMITS).await, + ]; + if conflicting { + for view in std::iter::once(complete).chain(others) { + assert!( + matches!(view, Err(ReadError::CorruptObject { path, reason }) + if path == "capsule-protocol ref heads" + && reason == "transaction identity names conflicting capsules") + ); + } + } else { + let complete = complete.unwrap(); + assert_eq!(complete.refs()["refs/heads/feature"], "6".repeat(40)); + for view in others { + let view = view.unwrap(); + assert_eq!( + ( + view.refs(), + view.git_pack_count(), + view.git_pack_bytes().unwrap(), + view.git_snapshot().unwrap() + ), + ( + complete.refs(), + complete.git_pack_count(), + complete.git_pack_bytes().unwrap(), + complete.git_snapshot().unwrap() + ) + ); + } + } + } + } + #[tokio::test] async fn metadata_view_preserves_multi_ref_activation_and_visibility() { for committed in [false, true] { From f7135e6e48b34d5af4f35404ddd9afee6b97a4aa Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 17:39:16 -0700 Subject: [PATCH 042/127] feat(metadata): query checkpoint catalog pages --- .../docs/architecture/indexed-remote-reads.md | 63 +++ .../cmd/history_recovery_v2/layered_tests.rs | 2 +- crab/src/cmd/metadb/capsule_tests.rs | 2 +- crab/src/git/xet_publication.rs | 94 ++-- crates/crab-metadata/README.md | 10 + .../src/capsule_protocol/history.rs | 6 +- .../src/capsule_protocol/layered.rs | 100 ++++- .../crab-metadata/src/capsule_protocol/mod.rs | 2 + .../src/capsule_protocol/pointer.rs | 4 +- .../src/capsule_protocol/pointer/paged.rs | 11 +- .../capsule_protocol/pointer/paged/reader.rs | 400 ++++++++++++++++++ .../pointer/paged/reader/tests.rs | 275 ++++++++++++ .../src/capsule_protocol/root.rs | 12 +- crates/crab-read/README.md | 8 + crates/crab-read/src/capsule_protocol.rs | 31 ++ crates/crab-remote/README.md | 8 + crates/crab-remote/examples/qualify_remote.rs | 58 ++- crates/crab-remote/examples/support/mod.rs | 13 +- crates/crab-write/src/capsule_protocol.rs | 2 +- 19 files changed, 1047 insertions(+), 54 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e9f4a2238..e60a4d861 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1417,3 +1417,66 @@ outside the checkout under `capsule-origin-fixed-live-r1/` in the dedicated qualification directory. The qualifier exercises the shared metadata reader used by maintenance; a newly installed CLI `metadb owner` invocation and broader CI remain pending. No indexed-catalog performance claim follows from this fix. + + +### Selected checkpoint catalog lookup + +Checkpoint format 7 (`CRBCKP07`) authenticates `CRBPCP01` catalog roots in its +footer. The catalog payload format stays unchanged. A fresh audit of all 19 +local release tags found neither affected codec path; format 6 belongs only to +the unmerged stack and is rejected explicitly rather than retained as a reader +fallback. Root/history format tests now identify 7 as the current contract and +reject 6. + +The metadata owner batches requested file hashes, reverse chunk keys and xorb +identities. One page cache and aggregate query byte budget cover file rows, +shard closures, xorb recipes and reverse-placement validation. Missing keys +remain misses; missing pages, invalid ranges, mismatched chunk ordinals and +conflicting immutable descriptors fail. File bindings preserve publication +order and consistent declared sizes. Page cache hits still consume admission. +Pages remain embedded in the existing checkpoint object, so its retention +already retains every catalog page without a new GC object namespace. + +The read view constructs lookup from its captured checkpoint and exactly the +same selected/deduplicated ordered capsule deltas used by full materialization. +CLI `prepare_delta` uses the paged Git view and selects outgoing file and chunk +closures, then checks only its candidate xorb descriptors against the base. +The native and Git remote-helper publication path share this preparation. +Protected receive/shared remote preparation, hydration/file-index wiring and +maintenance migration still need their own consumer proof before the gate can +close. Explicit maintenance retains complete inspection. + +This is a checkpoint lookup slice, not the complete historical-catalog gate. +Deltas are still materialized and their chunk suffix is still scanned. Run-level +key aggregation must retain exact selected-capsule membership and publication +order; whole-run overlays would replay checkpoint-covered capsules or widen +visible state. Until that is implemented and qualified, unrelated uncheckpointed +history can still increase opening and query work. No end-to-end constant-work +or latency claim is justified yet. + +The next run-index slice must batch each requested key once per captured run, +then filter descriptor occurrences by the exact selected capsule set. Immutable +shard/xorb variants need occurrence membership so an unselected conflicting +capsule cannot change the selected catalog; equivalent descriptors can share +one recipe even when their physical page offsets differ. File bindings choose +the last selected occurrence in the view's reconciled publication order. Required +negative fixtures include checkpoint-covered run prefixes, overlapping multi-ref +runs, conflicting selected variants and retained but unselected variants. These +are implementation requirements, not completed qualification. + +The first CLI publication test pass reached eight tests: seven passed, while the +old JSON corruption fixture panicked during UTF-8 conversion before calling +production. The fixture also exists on the paged-format parent branch. It is +retired in favor of known-pointer tests that corrupt an actual chunk-page byte +or truncate the catalog trailer inside an otherwise authenticated capsule. +Both must produce a typed catalog corruption error before staging is consulted; +no JSON reader or production validation bypass is introduced. + +`catalog-lookup-fixed-live-r1` published format 7 and matched selected file, +shard, xorb and reverse-chunk metadata against complete inspection on RustFS. +Opening decoded no catalog pages. Cold lookup fetched six ranges (9,132 bytes), +while warm lookup fetched zero origin bytes. The run exposed duplicate decode +instrumentation in the cache wrapper; the shared page decoder already emits +that event. The duplicate event is removed, and corrected measured qualification +remains pending. This four-MiB fixture is functional evidence, not the required +10-GiB/increasing-history performance gate. diff --git a/crab/src/cmd/history_recovery_v2/layered_tests.rs b/crab/src/cmd/history_recovery_v2/layered_tests.rs index 468da9e87..1df07fc96 100644 --- a/crab/src/cmd/history_recovery_v2/layered_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_tests.rs @@ -129,7 +129,7 @@ async fn layered_history_verification_preserves_historical_refs_and_bytes() { let (layout, root) = fixture().await; let chain = history_chain(&layout, &root).await.unwrap(); let oldest = chain.last().unwrap(); - assert_eq!(oldest.checkpoint().format(), 6); + assert_eq!(oldest.checkpoint().format(), 7); let verified = verify_history( &layout, diff --git a/crab/src/cmd/metadb/capsule_tests.rs b/crab/src/cmd/metadb/capsule_tests.rs index 4bd0aa6ed..5532d045c 100644 --- a/crab/src/cmd/metadb/capsule_tests.rs +++ b/crab/src/cmd/metadb/capsule_tests.rs @@ -109,7 +109,7 @@ async fn layered_diagnosis_and_rebuild_verify_nonempty_repository_without_rewrit .await .unwrap(); let before = root.record().digest().to_owned(); - assert_eq!(root.record().root().checkpoint().unwrap().format(), 6); + assert_eq!(root.record().root().checkpoint().unwrap().format(), 7); let diagnosis = diagnose_capsule(&layout, root.clone(), DbSelector::Both, true, &cancel) .await .unwrap(); diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 4e9a0f68f..e865a5088 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -56,7 +56,7 @@ pub(crate) async fn prepare_delta( return Ok((PointerCatalog::new(), XetPublicationStats::default())); } let concurrency = upload_concurrency.clamp(1, MAX_IN_FLIGHT_XORBS); - let view = crab_read::capsule_protocol::open_view_from_root_with_control( + let view = crab_read::capsule_protocol::open_git_view_from_root( layout, base.clone(), crab_read::capsule_protocol::CapsuleReadLimits { @@ -65,7 +65,8 @@ pub(crate) async fn prepare_delta( }, ) .await?; - let base_catalog = view.pointer_catalog().await?; + let lookup = view.pointer_catalog_lookup(layout, 2 * 1024 * 1024 * 1024)?; + drop(view); let mut pointer_sizes = BTreeMap::::new(); for pointer in pointers { @@ -82,9 +83,14 @@ pub(crate) async fn prepare_delta( } } + let file_hashes = pointer_sizes + .keys() + .map(|hash| hash.hex()) + .collect::>(); + let base_files = lookup.files(layout.store(), &file_hashes).await?; let unresolved = pointer_sizes .iter() - .filter_map(|(hash, size)| match base_catalog.files().get(&hash.hex()) { + .filter_map(|(hash, size)| match base_files.files().get(&hash.hex()) { Some(entry) if entry.size() == *size => None, Some(entry) => Some(Err(CrabError::CorruptObject { path: "capsule-protocol pointer catalog".to_owned(), @@ -97,6 +103,7 @@ pub(crate) async fn prepare_delta( None => Some(Ok((*hash, *size))), }) .collect::>>()?; + drop(base_files); if unresolved.is_empty() { return Ok((PointerCatalog::new(), XetPublicationStats::default())); } @@ -193,14 +200,15 @@ pub(crate) async fn prepare_delta( prepared_files.push((file_hash, file_size, chunks, plan)); } - // Catalog decoding already validates every descriptor. Retain placements - // only for outgoing recipes, so a small edit does not duplicate the entire - // historical chunk index and unrelated xorb descriptors in memory. + // Query only the outgoing recipes. Selected xorb recipes are verified + // against the reverse chunk index before any staged payload is reused. let mut needed_chunks = prepared_files .iter() .flat_map(|(_, _, chunks, _)| chunks.iter().map(|(hash, _)| hash.hex())) .collect::>(); - for (hash, entry) in base_catalog.xorbs() { + let requested_chunks = needed_chunks.iter().cloned().collect::>(); + let base_chunks = lookup.chunks(layout.store(), &requested_chunks).await?; + for (hash, entry) in base_chunks.xorbs() { if needed_chunks.is_empty() { break; } @@ -227,6 +235,7 @@ pub(crate) async fn prepare_delta( xorb_entries.insert(parse_merkle_hash(hash, "base xorb")?, entry.clone()); } } + drop(base_chunks); drop(needed_chunks); let mut external_candidates = external_candidates @@ -503,6 +512,22 @@ pub(crate) async fn prepare_delta( .await?; } + let dependency_hashes = pending_files + .iter() + .flat_map(|(_, _, _, hashes)| hashes.iter().map(|hash| hash.hex())) + .collect::>(); + let base_xorbs = lookup.xorbs(layout.store(), &dependency_hashes).await?; + for (hash, entry) in base_xorbs.xorbs() { + if xorb_entries + .get(&parse_merkle_hash(hash, "base xorb")?) + .is_some_and(|candidate| candidate != entry) + { + return Err(CrabError::CorruptObject { + path: "capsule-protocol pointer catalog".to_owned(), + reason: format!("xorb {hash} has conflicting descriptors"), + }); + } + } let mut shard_closures = vec![HashSet::new(); shards.len()]; for (file_hash, _, shard_index, dependency_hashes) in &pending_files { let closure = shard_closures.get_mut(*shard_index).ok_or_else(|| { @@ -520,7 +545,7 @@ pub(crate) async fn prepare_delta( .collect::>(); closure.sort(); for xorb_hash in &closure { - if base_catalog.xorbs().contains_key(xorb_hash) { + if base_xorbs.xorbs().contains_key(xorb_hash) { continue; } let hash = parse_merkle_hash(xorb_hash, "file xorb")?; @@ -1360,32 +1385,35 @@ mod tests { } #[tokio::test] - async fn known_pointers_do_not_bypass_catalog_validation() { + async fn known_pointers_reject_corrupted_catalog_pages() { let (catalog, pointer) = pointer_catalog(1, 1); - let chunk_hash = catalog.xorbs().values().next().unwrap().chunks()[0].hash(); - let bytes = String::from_utf8(catalog.encode().unwrap().to_vec()) - .unwrap() - .replace( - &format!("\"hash\":\"{chunk_hash}\""), - "\"hash\":\"invalid\"", - ); - let (layout, base) = publish_catalog(Bytes::from(bytes)).await; - let error = prepare_delta( - &layout, - &base, - &[pointer], - None, - None, - None, - 4, - true, - &CancellationToken::new(), - ) - .await - .unwrap_err(); - assert!( - matches!(error, CrabError::CorruptObject { reason, .. } if reason.contains("chunk")) - ); + let encoded = catalog.encode().unwrap(); + for truncate in [false, true] { + let mut bytes = encoded.to_vec(); + if truncate { + bytes.pop(); + } else { + // Corrupt the first chunk's hash while the outer capsule still + // authenticates these bytes. A known file cannot bypass pages. + bytes[22] ^= 1; + } + let (layout, base) = publish_catalog(Bytes::from(bytes)).await; + let error = prepare_delta( + &layout, + &base, + std::slice::from_ref(&pointer), + None, + None, + None, + 4, + true, + &CancellationToken::new(), + ) + .await + .unwrap_err(); + assert!(matches!(error, CrabError::CorruptObject { path, reason } + if path == "capsule-protocol pointer catalog" && reason.contains("catalog"))); + } } #[tokio::test] diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 07a421e59..9f555d53f 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -54,6 +54,16 @@ capsules can therefore appear in differently compacted ref frontiers without inflating pack counts or appearing to conflict; differing payload commitments remain rejected. +Checkpoint format 7 (`CRBCKP07`) carries authenticated catalog page roots in +its footer. `LayeredCheckpoint::pointer_catalog_lookup` captures these roots; +`PointerCatalogLookup` batches file/chunk/xorb keys through one bounded page +cache and byte budget, validates selected dependency closure and keeps immutable +descriptor conflicts distinct from absent keys. Full inspection remains an +explicit maintenance operation. All 19 audited local release tags omit these +capsule checkpoint formats; unshipped format 6 is rejected without a fallback. +Publication deltas are still materialized by the caller's captured view; their +run-level aggregation and lazy capture remain required for the complete gate. + External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and ordered xorb chunk lists are paged too. Keys preserve canonical lowercase Xet hex diff --git a/crates/crab-metadata/src/capsule_protocol/history.rs b/crates/crab-metadata/src/capsule_protocol/history.rs index 96030a2ce..d6b354f26 100644 --- a/crates/crab-metadata/src/capsule_protocol/history.rs +++ b/crates/crab-metadata/src/capsule_protocol/history.rs @@ -441,7 +441,7 @@ mod tests { #[test] fn history_rejects_unsupported_checkpoint_formats() { - for format in [0, 1, 2, 3, 4, 5, 7, u32::MAX] { + for format in [0, 1, 2, 3, 4, 5, 6, 8, u32::MAX] { let segment = history_with_unvalidated_pointers(format, 1); let built = HistorySegment::build( segment.checkpoint().clone(), @@ -467,8 +467,8 @@ mod tests { fn history_rejects_implicit_checkpoint_format() { let segment = HistorySegment::build(checkpoint(), None, state()).unwrap(); let body = std::str::from_utf8(segment.bytes()).unwrap(); - assert_eq!(body.matches("\"format\":6,").count(), 1); - let bytes = Bytes::from(body.replace("\"format\":6,", "")); + assert_eq!(body.matches("\"format\":7,").count(), 1); + let bytes = Bytes::from(body.replace("\"format\":7,", "")); let error = HistorySegment::decode(bytes) .expect_err("an implicit retired checkpoint format must fail"); diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index 0724e0d5c..1003e13db 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -15,8 +15,8 @@ mod visibility_pages; #[cfg(feature = "storage")] pub use visibility_pages::LayeredVisibilityReader; -const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP06"; -pub(super) const CHECKPOINT_VERSION: u32 = 6; +const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP07"; +pub(super) const CHECKPOINT_VERSION: u32 = 7; const LAYER_MAGIC: &[u8; 8] = b"CRBPKL01"; const LAYER_VERSION: u32 = 1; const TRAILER_BYTES: usize = 8 + 32 + 8; @@ -933,6 +933,7 @@ struct CheckpointFooter { #[serde(default)] visibility_transitions: BTreeMap>, visibility_pages: Option, + catalog_pages: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -1046,6 +1047,15 @@ impl LayeredCheckpoint { ) }) .transpose()?; + let catalog_pages = sections + .iter() + .find(|section| section.kind == CapsuleSectionKind::CatalogDelta) + .map(|section| { + super::pointer::paged::Descriptor::decode( + &body[section.offset as usize..(section.offset + section.length) as usize], + ) + }) + .transpose()?; let control_offset = 0_u64; let footer = CheckpointFooter { version: CHECKPOINT_VERSION, @@ -1058,6 +1068,7 @@ impl LayeredCheckpoint { cold_clone_object_count, visibility_transitions, visibility_pages, + catalog_pages, }; validate_source_inventory(&footer.sources)?; for source in &footer.sources { @@ -1188,6 +1199,31 @@ impl LayeredCheckpoint { .map(Some) } + /// Capture catalog page roots without reading the checkpoint catalog body. + #[cfg(feature = "storage")] + pub fn pointer_catalog_lookup( + &self, + layout: &crab_storage::StoreLayout, + maximum: u64, + ) -> Result { + let Some(descriptor) = &self.footer.catalog_pages else { + return Ok(super::PointerCatalogLookup::empty(maximum)); + }; + let section = self + .footer + .sections + .iter() + .find(|section| section.kind == CapsuleSectionKind::CatalogDelta) + .ok_or_else(|| corrupt("checkpoint catalog pages have no section"))?; + let range = PackRange::from_parts(section.offset, section.length, section.blake3.clone())?; + super::PointerCatalogLookup::checkpoint( + layout.capsule_checkpoint_path(&self.hash), + range, + descriptor.clone(), + maximum, + ) + } + /// Return the complete checkpoint bytes. #[must_use] pub fn bytes(&self) -> &Bytes { @@ -1309,6 +1345,13 @@ impl LayeredCheckpoint { let Some(section) = self.section(CapsuleSectionKind::CatalogDelta)? else { return Ok(PointerCatalog::new()); }; + if Some(&super::pointer::paged::Descriptor::decode(§ion)?) + != self.footer.catalog_pages.as_ref() + { + return Err(corrupt( + "checkpoint catalog body roots differ from its authenticated footer", + )); + } PointerCatalog::decode(§ion) } @@ -1549,6 +1592,20 @@ fn validate_checkpoint_footer( )); } } + let catalogs = footer + .sections + .iter() + .filter(|section| section.kind == CapsuleSectionKind::CatalogDelta) + .collect::>(); + match (&footer.catalog_pages, catalogs.as_slice()) { + (Some(descriptor), [section]) => descriptor.validate(section.length)?, + (None, []) => {} + _ => { + return Err(corrupt( + "checkpoint catalog section and footer roots differ", + )); + } + } let mut expected_offset = 0_u64; for section in &footer.sections { if !matches!( @@ -1705,6 +1762,45 @@ mod tests { } } + #[test] + fn checkpoint_control_requires_catalog_roots_for_exactly_one_section() { + let source = PackLayer::build(&pack()) + .unwrap() + .source_descriptor() + .unwrap(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard( + "a".repeat(64), + crate::capsule_protocol::ShardCatalogEntry::new(10, Vec::new()), + ) + .unwrap(); + catalog + .insert_file( + "b".repeat(64), + crate::capsule_protocol::FileCatalogEntry::new(0, "a".repeat(64)), + ) + .unwrap(); + let checkpoint = + LayeredCheckpoint::build(0, &"3".repeat(64), vec![source], catalog, None).unwrap(); + for remove_section in [false, true] { + let mut footer = checkpoint.footer.clone(); + if remove_section { + footer.sections.clear(); + } else { + footer.catalog_pages = None; + } + assert!( + validate_checkpoint_footer( + &footer, + checkpoint.control_offset(), + checkpoint.bytes.len() as u64 + ) + .is_err() + ); + } + } + #[test] fn checkpoint_pages_preserve_ref_membership() { let source = PackLayer::build(&pack()) diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 1e6750fa0..5df613267 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -42,6 +42,8 @@ pub use plan::{ publish_capsule_plan_receipt, publish_capsule_plan_repair_receipt, read_capsule_plan_intent, resolve_capsule_plan_receipt, }; +#[cfg(feature = "storage")] +pub use pointer::PointerCatalogLookup; pub use pointer::{ FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, }; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 0cdf6156b..2e9363aed 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -9,7 +9,9 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; -mod paged; +pub(super) mod paged; +#[cfg(feature = "storage")] +pub use paged::PointerCatalogLookup; /// One file identity and the canonical shard that reconstructs it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index 571d8b833..9eb3489c3 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -2,6 +2,11 @@ use super::*; use crate::pages::{Kind, PageRef, Rows, Writer}; +#[cfg(feature = "storage")] +mod reader; +#[cfg(feature = "storage")] +pub use reader::PointerCatalogLookup; + const MAGIC: &[u8; 8] = b"CRBPCP01"; const TRAILER_BYTES: usize = 12; const MAX_DESCRIPTOR_BYTES: usize = 4096; @@ -9,7 +14,7 @@ const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub(super) struct Descriptor { +pub(in crate::capsule_protocol) struct Descriptor { version: u32, body_bytes: u64, body_hash: [u8; 32], @@ -259,7 +264,7 @@ pub(super) fn decode(bytes: &[u8]) -> Result { } impl Descriptor { - pub(super) fn decode(bytes: &[u8]) -> Result { + pub(in crate::capsule_protocol) fn decode(bytes: &[u8]) -> Result { if bytes.len() < TRAILER_BYTES || bytes.len() as u64 > MAX_CATALOG_BYTES || bytes.get(bytes.len() - 8..) != Some(MAGIC.as_slice()) @@ -286,7 +291,7 @@ impl Descriptor { Ok(descriptor) } - fn validate(&self, length: u64) -> Result<()> { + pub(in crate::capsule_protocol) fn validate(&self, length: u64) -> Result<()> { let header = json(self)?; if self.version != 1 || header.len() > MAX_DESCRIPTOR_BYTES diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs new file mode 100644 index 000000000..954cb1570 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -0,0 +1,400 @@ +//! Selected dependency closure from captured checkpoint roots and ordered deltas. +use super::*; +use crate::capsule_protocol::PackRange; +use crate::pages::{PageCache, PageWork}; +use crab_storage::Store; +use std::collections::BTreeSet; + +#[cfg(test)] +mod tests; + +/// Snapshot-bound catalog queries with one cache and aggregate byte budget. +/// +/// The owner retains the captured objects against GC. Missing keys return an +/// empty selection; missing pages and conflicting dependencies return errors. +#[derive(Debug)] +pub struct PointerCatalogLookup { + checkpoint: Option, + deltas: Vec, + cache: PageCache, + maximum: u64, +} + +#[derive(Debug)] +struct CheckpointSource { + path: object_store::path::Path, + range: PackRange, + descriptor: Descriptor, +} + +impl PointerCatalogLookup { + pub(in crate::capsule_protocol) fn checkpoint( + path: object_store::path::Path, + range: PackRange, + descriptor: Descriptor, + maximum: u64, + ) -> Result { + descriptor.validate(range.length())?; + if range.offset().checked_add(range.length()).is_none() { + return Err(corrupt("catalog source range overflows")); + } + Ok(Self { + checkpoint: Some(CheckpointSource { + path, + range, + descriptor, + }), + deltas: Vec::new(), + cache: PageCache::default(), + maximum, + }) + } + + /// Capture publication-ordered deltas after the checkpoint represented here. + /// + /// Deltas must be the selected, deduplicated capsules of the same view. + /// This performs no mutable-root reads or dependency expansion. + pub fn with_deltas(mut self, deltas: Vec) -> Result { + for delta in &deltas { + delta.validate(false, true)?; + } + self.deltas.extend(deltas); + Ok(self) + } + + /// Start a lookup for a captured view that has no checkpoint catalog. + #[must_use] + pub fn empty(maximum: u64) -> Self { + Self { + checkpoint: None, + deltas: Vec::new(), + cache: PageCache::default(), + maximum, + } + } + + fn work(&self) -> PageWork { + PageWork { + maximum: self.maximum, + bytes: 0, + seen: BTreeSet::new(), + } + } + + async fn rows( + &self, + store: &Store, + root: Option<&PageRef>, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + let Some(source) = &self.checkpoint else { + return Ok(vec![None; keys.len()]); + }; + let Some(root) = root else { + return Ok(vec![None; keys.len()]); + }; + self.cache + .lookup( + store, + &source.path, + 0, + source.range.offset()..source.range.offset() + source.descriptor.body_bytes, + root, + kind, + keys, + work, + ) + .await + } + + async fn scan( + &self, + store: &Store, + root: Option<&PageRef>, + kind: Kind, + work: &mut PageWork, + ) -> Result { + let Some(root) = root else { + return Ok(Vec::new()); + }; + let source = self + .checkpoint + .as_ref() + .ok_or_else(|| corrupt("catalog page has no source"))?; + self.cache + .scan( + store, + &source.path, + 0, + source.range.offset()..source.range.offset() + source.descriptor.body_bytes, + root, + kind, + work, + ) + .await + } + + async fn selected_xorbs( + &self, + store: &Store, + hashes: &BTreeSet, + work: &mut PageWork, + ) -> Result { + let keys = keys(hashes)?; + let root = self + .checkpoint + .as_ref() + .and_then(|s| s.descriptor.xorbs.as_ref()); + let rows = self + .rows(store, root, Kind::CatalogXorbs, &keys, work) + .await?; + let mut catalog = PointerCatalog::new(); + for (hash, row) in hashes.iter().zip(rows) { + if let Some(row) = row { + let row: XorbRow = from_json(&row)?; + let chunks = self + .scan(store, Some(&row.chunks), Kind::CatalogXorbChunks, work) + .await?; + let chunks = chunks + .into_iter() + .enumerate() + .map(|(position, (key, value))| { + if key != (position as u32).to_be_bytes() || value.len() != 36 { + return Err(corrupt("xorb chunk row has invalid position or size")); + } + let size = u32::from_be_bytes( + value[32..] + .try_into() + .map_err(|_| corrupt("chunk size is truncated"))?, + ); + Ok(XorbChunkEntry::new(hex(&value[..32])?, size)) + }) + .collect::>()?; + catalog.insert_xorb( + hash.clone(), + XorbCatalogEntry::new(row.encoded_size, hex(&row.body_digest)?, chunks), + )?; + } + // Every matching immutable descriptor participates, even when a + // later source supplies the placement or file binding selected. + for delta in &self.deltas { + if let Some(entry) = delta.xorbs.get(hash) { + admit( + work, + 72u64.saturating_add((entry.chunks.len() as u64).saturating_mul(36)), + )?; + catalog.insert_xorb(hash.clone(), entry.clone())?; + } + } + } + catalog.validate(true, true)?; + Ok(catalog) + } + + /// Select xorb descriptors without decoding unrelated catalog records. + pub async fn xorbs(&self, store: &Store, hashes: &[String]) -> Result { + let mut work = self.work(); + let hashes = requested(hashes, &mut work)?; + self.selected_xorbs(store, &hashes, &mut work).await + } + + /// Select requested files and their complete shard/xorb dependency closure. + pub async fn files(&self, store: &Store, hashes: &[String]) -> Result { + let mut work = self.work(); + let hashes = requested(hashes, &mut work)?; + let root = self + .checkpoint + .as_ref() + .and_then(|s| s.descriptor.files.as_ref()); + let rows = self + .rows(store, root, Kind::CatalogFiles, &keys(&hashes)?, &mut work) + .await?; + let mut catalog = PointerCatalog::new(); + let mut shards = BTreeSet::new(); + for (hash, row) in hashes.iter().zip(rows) { + let mut selected = row + .map(|value| { + if value.len() != 40 { + return Err(corrupt("catalog file row has an invalid byte length")); + } + let size = u64::from_be_bytes( + value[..8] + .try_into() + .map_err(|_| corrupt("file size is truncated"))?, + ); + Ok(FileCatalogEntry::new(size, hex(&value[8..])?)) + }) + .transpose()?; + for delta in &self.deltas { + if let Some(entry) = delta.files.get(hash) { + if selected.as_ref().is_some_and(|old| old.size != entry.size) { + return Err(corrupt("file has conflicting declared sizes")); + } + admit(&mut work, 40)?; + selected = Some(entry.clone()); + } + } + if let Some(entry) = selected { + shards.insert(entry.shard_hash.clone()); + catalog.insert_file(hash.clone(), entry)?; + } + } + let root = self + .checkpoint + .as_ref() + .and_then(|s| s.descriptor.shards.as_ref()); + let rows = self + .rows(store, root, Kind::CatalogShards, &keys(&shards)?, &mut work) + .await?; + let mut xorbs = BTreeSet::new(); + for (hash, row) in shards.iter().zip(rows) { + if let Some(row) = row { + let row: ShardRow = from_json(&row)?; + let closure = self + .scan( + store, + row.xorbs.as_ref(), + Kind::CatalogShardXorbs, + &mut work, + ) + .await?; + let closure = closure + .into_iter() + .map(|(key, value)| { + if !value.is_empty() { + return Err(corrupt("shard xorb row has unexpected data")); + } + hex(&key) + }) + .collect::>()?; + catalog.insert_shard( + hash.clone(), + ShardCatalogEntry::new(row.encoded_size, closure), + )?; + } + for delta in &self.deltas { + if let Some(entry) = delta.shards.get(hash) { + admit( + &mut work, + 8u64.saturating_add((entry.xorb_hashes.len() as u64).saturating_mul(32)), + )?; + catalog.insert_shard(hash.clone(), entry.clone())?; + } + } + let entry = catalog + .shards + .get(hash) + .ok_or_else(|| corrupt("selected file references an absent shard"))?; + xorbs.extend(entry.xorb_hashes.iter().cloned()); + } + let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + catalog.validate(true, true)?; + Ok(catalog) + } + + /// Select the lowest-xorb placement for each requested chunk and its xorb closure. + /// + /// The selected xorb's ordered recipe must agree with the reverse index. + pub async fn chunks(&self, store: &Store, hashes: &[String]) -> Result { + let mut work = self.work(); + let hashes = requested(hashes, &mut work)?; + let root = self + .checkpoint + .as_ref() + .and_then(|s| s.descriptor.chunks.as_ref()); + let rows = self + .rows(store, root, Kind::CatalogChunks, &keys(&hashes)?, &mut work) + .await?; + let mut selected = BTreeMap::new(); + for (hash, row) in hashes.iter().zip(rows) { + if let Some(value) = row { + if value.len() != 40 { + return Err(corrupt( + "catalog chunk placement has an invalid byte length", + )); + } + let xorb = hex(&value[..32])?; + let index = u32::from_be_bytes( + value[32..36] + .try_into() + .map_err(|_| corrupt("chunk position is truncated"))?, + ); + let size = u32::from_be_bytes( + value[36..] + .try_into() + .map_err(|_| corrupt("chunk size is truncated"))?, + ); + selected.insert(hash.clone(), (xorb, index, size)); + } + } + // Deltas are still materialized by the view owner. This scan is bounded + // to the post-checkpoint suffix; run-level indexing replaces it next. + for delta in &self.deltas { + for (xorb, entry) in &delta.xorbs { + for (index, chunk) in entry.chunks.iter().enumerate() { + if !hashes.contains(&chunk.hash) { + continue; + } + let candidate = (xorb.clone(), index as u32, chunk.uncompressed_size); + let placement = selected + .entry(chunk.hash.clone()) + .or_insert_with(|| candidate.clone()); + if candidate < *placement { + *placement = candidate; + } + } + } + } + let xorbs = selected.values().map(|(xorb, _, _)| xorb.clone()).collect(); + let catalog = self.selected_xorbs(store, &xorbs, &mut work).await?; + for (hash, (xorb, index, size)) in selected { + let entry = catalog + .xorbs + .get(&xorb) + .ok_or_else(|| corrupt("chunk placement references an absent xorb"))?; + let chunk = entry + .chunks + .get(index as usize) + .ok_or_else(|| corrupt("chunk placement references an absent xorb position"))?; + let first = entry.chunks.iter().position(|chunk| chunk.hash == hash); + if chunk.hash != hash + || chunk.uncompressed_size != size + || first != Some(index as usize) + { + return Err(corrupt("chunk placement differs from its xorb recipe")); + } + } + Ok(catalog) + } +} + +fn keys(hashes: &BTreeSet) -> Result>> { + hashes + .iter() + .map(|value| hash(value).map(|key| key.to_vec())) + .collect() +} + +fn admit(work: &mut PageWork, bytes: u64) -> Result<()> { + work.bytes = work + .bytes + .checked_add(bytes) + .ok_or_else(|| corrupt("catalog query byte count overflows"))?; + if work.bytes > work.maximum { + return Err(corrupt("catalog query exceeds its byte limit")); + } + Ok(()) +} + +fn requested(hashes: &[String], work: &mut PageWork) -> Result> { + admit(work, (hashes.len() as u64).saturating_mul(96))?; + for value in hashes { + hash(value)?; + } + Ok(hashes.iter().cloned().collect()) +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs new file mode 100644 index 000000000..e8ed058d7 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs @@ -0,0 +1,275 @@ +use super::*; +use crate::capsule_protocol::{CapsuleGitPack, LayeredCheckpoint, PackLayer}; +use crab_storage::StoreLayout; +use object_store::memory::InMemory; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +fn catalog(count: usize) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + for index in 0..count { + let xorb = format!("{:064x}", index + 1_000_000); + let shard = format!("{:064x}", index + 3_000_000); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 99, + "e".repeat(64), + vec![XorbChunkEntry::new( + format!("{:064x}", index + 2_000_000), + 7, + )], + ), + ) + .unwrap(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + catalog + .insert_file(format!("{index:064x}"), FileCatalogEntry::new(7, shard)) + .unwrap(); + } + catalog +} + +async fn fixture( + catalog: PointerCatalog, + maximum: u64, +) -> ( + StoreLayout, + LayeredCheckpoint, + PointerCatalogLookup, + Arc, +) { + let pack = CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "1".repeat(40), + 1, + ) + .unwrap(); + let source = PackLayer::build(&pack) + .unwrap() + .source_descriptor() + .unwrap(); + let checkpoint = + LayeredCheckpoint::build(1, &"3".repeat(64), vec![source], catalog, None).unwrap(); + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let store = + Store::new(Arc::new(InMemory::new())).with_read_request_observer(Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + })); + let layout = StoreLayout::new(store, "catalog-lookup".to_owned()); + layout + .store() + .put_overwrite( + &layout.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + let control = LayeredCheckpoint::decode_control( + checkpoint + .bytes() + .slice(checkpoint.control_offset() as usize..), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.hash(), + checkpoint.footer_hash(), + ) + .unwrap(); + let lookup = control.pointer_catalog_lookup(&layout, maximum).unwrap(); + (layout, checkpoint, lookup, reads) +} + +#[tokio::test] +async fn fixed_file_and_chunk_queries_read_selected_closure_as_history_grows() { + for count in [100, 20_000] { + let (layout, _, lookup, reads) = fixture(catalog(count), 512 * 1024).await; + let file = format!("{:064x}", 42); + let selected = lookup + .files( + layout.store(), + &[file.clone(), file.clone(), "f".repeat(64)], + ) + .await + .unwrap(); + assert_eq!( + ( + selected.files.len(), + selected.shards.len(), + selected.xorbs.len() + ), + (1, 1, 1) + ); + let chunks = lookup + .chunks(layout.store(), &[format!("{:064x}", 2_000_042)]) + .await + .unwrap(); + assert_eq!(chunks.xorbs, selected.xorbs); + assert!( + reads.load(Ordering::SeqCst) <= 16, + "fixed query exceeded page request bound" + ); + let cold = reads.load(Ordering::SeqCst); + lookup.files(layout.store(), &[file]).await.unwrap(); + assert_eq!(reads.load(Ordering::SeqCst), cold); + } +} + +#[tokio::test] +async fn ordered_deltas_override_file_bindings_but_preserve_descriptor_conflicts() { + let (layout, _, lookup, _) = fixture(catalog(2), 1024 * 1024).await; + let file = format!("{:064x}", 0); + let mut delta = PointerCatalog::new(); + let replacement = format!("{:064x}", 3_000_001); + delta + .insert_file(file.clone(), FileCatalogEntry::new(7, replacement.clone())) + .unwrap(); + let lookup = lookup.with_deltas(vec![delta]).unwrap(); + assert_eq!( + lookup + .files(layout.store(), std::slice::from_ref(&file)) + .await + .unwrap() + .files[&file] + .shard_hash, + replacement + ); + let mut conflict = PointerCatalog::new(); + conflict + .insert_xorb( + format!("{:064x}", 1_000_001), + XorbCatalogEntry::new( + 100, + "e".repeat(64), + vec![XorbChunkEntry::new(format!("{:064x}", 2_000_001), 7)], + ), + ) + .unwrap(); + let lookup = lookup.with_deltas(vec![conflict]).unwrap(); + assert!( + lookup + .files(layout.store(), &[format!("{:064x}", 1)]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn chunk_selection_uses_lowest_xorb_then_first_occurrence_across_sources() { + let (layout, _, lookup, _) = fixture(catalog(1), 1024 * 1024).await; + let chunk = format!("{:064x}", 2_000_000); + let mut delta = PointerCatalog::new(); + delta + .insert_xorb( + "0".repeat(64), + XorbCatalogEntry::new( + 88, + "e".repeat(64), + vec![ + XorbChunkEntry::new(chunk.clone(), 7), + XorbChunkEntry::new(chunk.clone(), 7), + ], + ), + ) + .unwrap(); + let lookup = lookup.with_deltas(vec![delta]).unwrap(); + let selected = lookup.chunks(layout.store(), &[chunk]).await.unwrap(); + assert_eq!( + selected.xorbs.keys().collect::>(), + vec![&"0".repeat(64)] + ); +} + +#[tokio::test] +async fn missing_and_corrupted_pages_are_errors_instead_of_catalog_misses() { + for missing in [false, true] { + let (layout, checkpoint, lookup, _) = fixture(catalog(2), 1024 * 1024).await; + let path = layout.capsule_checkpoint_path(checkpoint.hash()); + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + let mut bytes = checkpoint.bytes().to_vec(); + let source = lookup.checkpoint.as_ref().unwrap(); + let offset = source.range.offset() + source.descriptor.files.as_ref().unwrap().offset; + bytes[offset as usize] ^= 1; + layout + .store() + .put_overwrite(&path, bytes.into()) + .await + .unwrap(); + } + assert!( + lookup + .files(layout.store(), &[format!("{:064x}", 0)]) + .await + .is_err() + ); + } +} + +#[tokio::test] +async fn loaded_dependencies_and_chunk_placements_fail_closed() { + let (layout, _, lookup, _) = fixture(catalog(1), 1024 * 1024).await; + let mut delta = PointerCatalog::new(); + delta + .insert_file("f".repeat(64), FileCatalogEntry::new(7, "d".repeat(64))) + .unwrap(); + let lookup = lookup.with_deltas(vec![delta]).unwrap(); + assert!( + lookup + .files(layout.store(), &["f".repeat(64)]) + .await + .is_err() + ); + // A re-signed reverse index must still agree with its authenticated xorb. + let bytes = catalog(1).encode().unwrap(); + let mut descriptor = Descriptor::decode(&bytes).unwrap(); + let root = descriptor.chunks.as_mut().unwrap(); + let mut body = bytes[..descriptor.body_bytes as usize].to_vec(); + let size = root.offset as usize + 14 + 32 + 4 + 36; + body[size] ^= 1; + root.hash = + *blake3::hash(&body[root.offset as usize..root.offset as usize + root.bytes as usize]) + .as_bytes(); + descriptor.body_hash = *blake3::hash(&body).as_bytes(); + let header = json(&descriptor).unwrap(); + body.extend_from_slice(&header); + crate::pages::put_u32(&mut body, header.len()).unwrap(); + body.extend_from_slice(MAGIC); + let descriptor = Descriptor::decode(&body).unwrap(); + let range = PackRange::new(0, &body).unwrap(); + let path = object_store::path::Path::from("resigned-catalog"); + layout + .store() + .put_overwrite(&path, body.into()) + .await + .unwrap(); + let lookup = PointerCatalogLookup::checkpoint(path, range, descriptor, 1024 * 1024).unwrap(); + assert!( + lookup + .chunks(layout.store(), &[format!("{:064x}", 2_000_000)]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn query_budget_applies_to_warm_pages_and_materialized_delta_dependencies() { + let (layout, _, mut lookup, _) = fixture(catalog(2), 1024 * 1024).await; + let files = [format!("{:064x}", 0)]; + lookup.files(layout.store(), &files).await.unwrap(); + lookup.maximum = 1; + assert!(lookup.files(layout.store(), &files).await.is_err()); + let lookup = PointerCatalogLookup::empty(1) + .with_deltas(vec![catalog(2)]) + .unwrap(); + assert!(lookup.files(layout.store(), &files).await.is_err()); +} diff --git a/crates/crab-metadata/src/capsule_protocol/root.rs b/crates/crab-metadata/src/capsule_protocol/root.rs index 71484ac45..624c8d170 100644 --- a/crates/crab-metadata/src/capsule_protocol/root.rs +++ b/crates/crab-metadata/src/capsule_protocol/root.rs @@ -1330,9 +1330,15 @@ mod tests { fn root_rejects_retired_or_implicit_checkpoint_formats() { let record = checkpointed_record(); let payload = serde_json::to_string(record.root()).unwrap(); - assert_eq!(payload.matches("\"format\":6,").count(), 1); - for replacement in ["", "\"format\":3,", "\"format\":4,", "\"format\":5,"] { - let payload = payload.replace("\"format\":6,", replacement); + assert_eq!(payload.matches("\"format\":7,").count(), 1); + for replacement in [ + "", + "\"format\":3,", + "\"format\":4,", + "\"format\":5,", + "\"format\":6,", + ] { + let payload = payload.replace("\"format\":7,", replacement); // Recompute the envelope digest so only format admission can fail. let mut bytes = ROOT_MAGIC.to_vec(); bytes.extend_from_slice(&ROOT_VERSION.to_be_bytes()); diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index fe038adf8..bcc80a908 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -453,3 +453,11 @@ inspection is an explicit async operation. `CapsuleRepositoryView::pointer_catal is also async: hydration, publication and maintenance materialize the complete catalog from the captured checkpoint when its body was omitted during opening. Catalog indexing and bounded historical Xet lookup remain separate work. + +`CapsuleRepositoryView::pointer_catalog_lookup` captures the checkpoint's paged +catalog together with exactly the selected publication-ordered deltas. File +bindings retain later-wins precedence with equal sizes; selected shard/xorb +identities retain conflict checks across sources. Queries do not read a newer +root. Checkpoint catalogs stay cold during lookup construction; visible deltas +still materialize and need run-level indexed aggregation. The complete catalog +API remains for explicit maintenance and integrity inspection. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 5dc4594bc..4fdbe86b8 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -738,6 +738,37 @@ impl CapsuleRepositoryView { }) } + /// Capture batched catalog lookup against this view's checkpoint and selected deltas. + /// + /// Checkpoint pages stay cold. Publication deltas still use the view's + /// authenticated materialization; unrelated checkpoint history is never decoded. + pub fn pointer_catalog_lookup( + &self, + layout: &StoreLayout, + maximum: u64, + ) -> Result { + let lookup = self + .checkpoint + .as_ref() + .map(|checkpoint| checkpoint.pointer_catalog_lookup(layout, maximum)) + .transpose()? + .unwrap_or_else(|| { + crab_metadata::capsule_protocol::PointerCatalogLookup::empty(maximum) + }); + let mut deltas = Vec::new(); + for capsule in &self.capsules { + if let Some(delta) = capsule.pointer_catalog_delta()? { + deltas.push(delta); + } + } + deltas.extend( + self.capsule_controls + .iter() + .filter_map(|capsule| capsule.pointer_catalog_delta().cloned()), + ); + Ok(lookup.with_deltas(deltas)?) + } + /// Materialize the complete generation-pinned external pointer catalog. pub async fn pointer_catalog(&self) -> Result { let mut catalog = diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index aa2735df6..07fabe32e 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -103,3 +103,11 @@ chosen repository; use a dedicated qualification prefix. Run `build-indexes` afterward, then repeat the same native-Git query oracles. This separates checkpoint maintenance costs from foreground opening and prevents a no-op maintenance command from standing in for checkpoint qualification. + +`qualify_remote catalog` measures selected file +and chunk dependency lookup against a captured paged Git view, cold and warm. +The hash argument uses Xet's word-ordered catalog-key hex, as produced by +`crab_xet::hash::merkle_hex_from_bytes` from the pointer's raw hash bytes. +Its complete-catalog maintenance oracle runs outside measured lookup work. +`pointer_catalog_pages` reports parsed checkpoint catalog pages separately; +opening still includes eager publication deltas and ref/visibility metadata. diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index c28b959e5..07142244b 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -36,12 +36,13 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes|checkpoint|ref-point|resolve-oid|blame-history|blame-history-uncached]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|ref-point|resolve-oid|blame-history|blame-history-uncached]", ) .into()), }; if selection != "all" && selection != "build-indexes" + && selection != "catalog" && selection != "checkpoint" && selection != "graph-pages" && selection != "graph-ancestry" @@ -100,6 +101,61 @@ async fn main() -> Result<()> { "{}", json!({"kind":"result", "operation":"ref-point", "value":previous}) ); + } else if selection == "catalog" { + let view = counters + .measure("open", "cold", async { + let root = crab_metadata::capsule_protocol::load_root(&layout).await?; + Ok(crab_read::capsule_protocol::open_git_view_from_root( + &layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 2 * 1024 * 1024 * 1024, + max_frontier_bytes: 2 * 1024 * 1024 * 1024, + }, + ) + .await?) + }) + .await?; + // Like resolve-oid, this selection interprets the path argument as an + // immutable identity. The bulk maintenance oracle stays outside timing. + let hash = std::str::from_utf8(path.as_bytes())?.to_owned(); + let lookup = view.pointer_catalog_lookup(&layout, 128 * 1024 * 1024)?; + let full = view.pointer_catalog().await?; + let expected_file = full + .files() + .get(&hash) + .ok_or_else(|| io::Error::other("requested catalog file is absent"))?; + let mut previous = None; + for cache in ["cold", "warm"] { + let selected = counters.measure("catalog", cache, async { + let files = lookup.files(layout.store(), std::slice::from_ref(&hash)).await?; + if files.files().get(&hash) != Some(expected_file) { + return Err(io::Error::other("file binding differs from complete catalog").into()); + } + for (key, entry) in files.shards() { + if full.shards().get(key) != Some(entry) { return Err(io::Error::other("selected shard differs").into()); } + } + for (key, entry) in files.xorbs() { + if full.xorbs().get(key) != Some(entry) { return Err(io::Error::other("selected xorb differs").into()); } + } + let chunks = files.xorbs().values().flat_map(|entry| entry.chunks().iter().map(|chunk| chunk.hash().to_owned())).collect::>(); + let placements = lookup.chunks(layout.store(), &chunks).await?; + for chunk in &chunks { + let expected = full.xorbs().iter().find(|(_, entry)| entry.chunks().iter().any(|item| item.hash() == chunk)); + let selected = placements.xorbs().iter().find(|(_, entry)| entry.chunks().iter().any(|item| item.hash() == chunk)); + if expected != selected { return Err(io::Error::other("chunk placement differs from complete catalog").into()); } + } + Ok(json!({"files":files.files().len(), "shards":files.shards().len(), "xorbs":files.xorbs().len(), "chunks":chunks.len()})) + }).await?; + if previous.as_ref().is_some_and(|value| value != &selected) { + return Err(io::Error::other("cold/warm catalog results differ").into()); + } + previous = Some(selected); + } + println!( + "{}", + json!({"kind":"result", "operation":"catalog", "value":previous}) + ); } else if selection == "checkpoint" { counters .measure("checkpoint", "maintenance", async { diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index 2abfd31d2..221add4c4 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -26,8 +26,8 @@ pub struct Measurements { written_bytes: AtomicU64, active: AtomicU64, comparison_work: [AtomicU64; 2], - decoded_blocks: [AtomicU64; 7], - decoded_bytes: [AtomicU64; 7], + decoded_blocks: [AtomicU64; 8], + decoded_bytes: [AtomicU64; 8], } #[derive(Clone, Copy)] @@ -37,8 +37,8 @@ struct Snapshot { read_bytes: u64, written_bytes: u64, comparison_work: [u64; 2], - decoded_blocks: [u64; 7], - decoded_bytes: [u64; 7], + decoded_blocks: [u64; 8], + decoded_bytes: [u64; 8], } impl StorageObserver for Measurements { @@ -78,6 +78,7 @@ impl tracing::field::Visit for DecodeEvent { "capsule_visibility" => Some(4), "checkpoint_visibility" => Some(5), "checkpoint_visibility_pages" => Some(6), + "pointer_catalog_pages" => Some(7), _ => None, }; } @@ -192,6 +193,7 @@ impl Measurements { "capsule_visibility", "checkpoint_visibility", "checkpoint_visibility_pages", + "pointer_catalog_pages", ] .into_iter() .enumerate() @@ -277,13 +279,14 @@ mod tests { tracing::debug!(target:"crab_metadata::index_decode", index_format="checkpoint_visibility_pages", decoded_bytes=43_u64); tracing::debug!(target:"crab_remote_git::blame_comparison", cache_hit=false); tracing::debug!(target:"crab_remote_git::blame_comparison", cache_hit=true); + tracing::debug!(target:"crab_metadata::index_decode", index_format="pointer_catalog_pages", decoded_bytes=47_u64); tracing::debug!(target:"unrelated", index_format="path_state", decoded_bytes=999_u64); }); let snapshot = counters.snapshot(); assert_eq!(snapshot.comparison_work, [1, 1]); assert_eq!( (snapshot.decoded_blocks, snapshot.decoded_bytes), - ([1, 1, 1, 1, 1, 1, 1], [41, 123, 67, 29, 31, 37, 43]) + ([1, 1, 1, 1, 1, 1, 1, 1], [41, 123, 67, 29, 31, 37, 43, 47]) ); } diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index abfac50a9..fbe55b854 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -3253,7 +3253,7 @@ mod tests { ) .await .unwrap(); - assert_eq!(restored.record().root().checkpoint().unwrap().format(), 6); + assert_eq!(restored.record().root().checkpoint().unwrap().format(), 7); assert_eq!( restored.record().root().generation(), fenced.record().root().generation() + 1 From 4fb0e381984644a52625e728ece7d46c5ee0418d Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 17:44:17 -0700 Subject: [PATCH 043/127] docs: record selected catalog lookup proof --- .../docs/architecture/indexed-remote-reads.md | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e60a4d861..9f22d46a8 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1477,6 +1477,21 @@ shard, xorb and reverse-chunk metadata against complete inspection on RustFS. Opening decoded no catalog pages. Cold lookup fetched six ranges (9,132 bytes), while warm lookup fetched zero origin bytes. The run exposed duplicate decode instrumentation in the cache wrapper; the shared page decoder already emits -that event. The duplicate event is removed, and corrected measured qualification -remains pending. This four-MiB fixture is functional evidence, not the required -10-GiB/increasing-history performance gate. +that event. `catalog-lookup-fixed-live-r2` passes after removal: exactly six +cold page decodes (9,132 bytes) and zero warm decodes, with the same transport +counts and zero active backend work at completion. The corrected executable's +SHA-256 is `22157c5a665a9a063b1aa81c2fdc1b10c339e9827e66ff524e376aa0a98500ca`. +The raw root/checkpoint footer and object fingerprints confirm format 7 and its +catalog roots; frozen Rust/manifest fingerprints remain unchanged. Receipts are +retained outside the checkout in `catalog-lookup-fixed-live-r2/` under the +dedicated qualification directory. This four-MiB fixture is functional evidence, +not the required 10-GiB/increasing-history performance gate. + +Validation passes: 110 storage-enabled metadata capsule tests, 92 payload-only +metadata capsule tests, 48 read capsule tests, 22 writer capsule tests, all eight +CLI publication tests, the six final lookup regressions and both qualification +counter tests. Strict metadata/read/remote Clippy with all targets and publication +features passes, as do CLI compilation, formatting and diff checks. The CLI test +binary emits an Apple linker warning about unwind-table size; the eight tests +still complete successfully. Installed CLI push/hydration and representative S3 +latency qualification remain separate required proof. From 4d59cbef9a83e21863d74ab04cd268d93f683d12 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 18:13:07 -0700 Subject: [PATCH 044/127] feat(metadata): index run catalog occurrences Bind paged catalog variants to exact logical capsule occurrences so future selected lookups can preserve visibility and file precedence. Run footer v4 rejects unshipped v3. Reader wiring and CLI boundary qualification remain incomplete; keep this producer slice in draft. --- .../docs/architecture/indexed-remote-reads.md | 27 ++ crates/crab-metadata/README.md | 10 + .../src/capsule_protocol/pointer/paged.rs | 60 ++++ .../crab-metadata/src/capsule_protocol/run.rs | 51 ++- .../src/capsule_protocol/run/catalog.rs | 311 ++++++++++++++++++ .../src/capsule_protocol/run/catalog/tests.rs | 208 ++++++++++++ crates/crab-metadata/src/pages.rs | 12 +- 7 files changed, 673 insertions(+), 6 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/run/catalog.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 9f22d46a8..20529a11e 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1495,3 +1495,30 @@ features passes, as do CLI compilation, formatting and diff checks. The CLI test binary emits an Apple linker warning about unwind-table size; the eight tests still complete successfully. Installed CLI push/hydration and representative S3 latency qualification remain separate required proof. +### Run catalog occurrence index under implementation + +Run footer version 4 stores one embedded catalog index after capsule/index-pool +bytes and before admission pages. Its descriptor commits four paged key roots, +the exact capsule source-body extents, byte count and body hash. File, shard, +xorb and reverse-chunk records retain disjoint capsule occurrence lists for each +logical value variant. Equal shard/xorb values share a recipe even when their +original catalog pages occupy different offsets. Distinct values keep separate +membership, including conflicts in unselected capsules. + +Full verification reconstructs the canonical index from the original nested +catalogs and checks exact bytes and roots; footer-only validation checks region +coverage and source bindings without fetching index pages. An absent index must +correspond to absent or canonically empty capsule catalogs. Tests cover relocated +equal descriptors, conflicting file bindings, empty sections, compacted control +parity, omitted roots and re-signed occurrence changes. This is a format/producer +slice still under validation. Foreground lookup remains eager for deltas until +selected run sources and their reconciled ordering are wired into the reader. + +Producer validation: 117 storage-enabled capsule tests, 98 payload-only capsule +tests, 48 read-side capsule tests and 22 publication tests pass. Strict metadata +Clippy with storage/remote-index features passes. These checks establish encoding, +compaction and existing consumer integration; they do not establish lazy run +lookups or the fixed-edit history-scaling gate. CLI publication qualification is +pending. Its corrupt-catalog fixture currently attempts to publish malformed +catalog bytes, which the new producer validates earlier; that boundary needs +explicit qualification before this format change is ready to land. diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 9f555d53f..30fc67733 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -462,3 +462,13 @@ must decide whether that candidate actually changed the requested path. Full loads, resumable construction, compaction and maintenance scans retain markers. The retired unreleased version 5 is rebuilt by existing maintenance, never read through an alternate codec. + +Run footer version 4 adds an embedded `CRBRCG01` catalog index. Its four paged +key directories retain exact capsule occurrences and group equal immutable +values by a semantic commitment that excludes physical page offsets. Full run +verification rebuilds the index from nested capsules, so re-signed membership +changes cannot invent catalog authority. Footer-only reads admit the index +region and source spans without fetching its body. Foreground catalog lookup +still uses materialized deltas; selected run-index reads remain unfinished. +The affected run format is absent from the audited release tags; unshipped +version 3 is rejected rather than retained through a fallback. diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index 9eb3489c3..4af634053 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -47,6 +47,66 @@ fn from_json(bytes: &[u8]) -> Result { serde_json::from_slice(bytes).map_err(|source| MetadataError::PointerCatalogRecord { source }) } +pub(in crate::capsule_protocol) struct CatalogIndexRow { + pub key: Vec, + pub value: Vec, + pub identity: [u8; 32], +} + +// Run aggregation retains physical page references but groups immutable values +// by logical content. Page offsets can change without changing a descriptor. +pub(in crate::capsule_protocol) fn index_rows( + bytes: &[u8], +) -> Result<(u64, [Vec; 4])> { + let catalog = PointerCatalog::decode_delta(bytes)?; + let descriptor = Descriptor::decode(bytes)?; + let body = &bytes[..descriptor.body_bytes as usize]; + let mut output = std::array::from_fn(|_| Vec::new()); + let mut visited = BTreeMap::new(); + for (position, (root, kind)) in [ + (descriptor.files.as_ref(), Kind::CatalogFiles), + (descriptor.shards.as_ref(), Kind::CatalogShards), + (descriptor.xorbs.as_ref(), Kind::CatalogXorbs), + (descriptor.chunks.as_ref(), Kind::CatalogChunks), + ] + .into_iter() + .enumerate() + { + let Some(root) = root else { + continue; + }; + for (key, value) in + crate::pages::scan(body, root, kind, "pointer catalog pages", &mut visited)? + { + let normalized = match kind { + Kind::CatalogShards => json( + catalog + .shards + .get(&hex(&key)?) + .ok_or_else(|| corrupt("indexed shard is absent"))?, + )?, + Kind::CatalogXorbs => json( + catalog + .xorbs + .get(&hex(&key)?) + .ok_or_else(|| corrupt("indexed xorb is absent"))?, + )?, + _ => value.clone(), + }; + let mut identity = blake3::Hasher::new(); + identity.update(b"crab.catalog.value.v1\0"); + identity.update(&[kind as u8]); + identity.update(&normalized); + output[position].push(CatalogIndexRow { + key, + value, + identity: *identity.finalize().as_bytes(), + }); + } + } + Ok((descriptor.body_bytes, output)) +} + fn hash(value: &str) -> Result<[u8; 32]> { let bytes = value.as_bytes(); // Binary keys cannot distinguish hex aliases. Reject them instead of silently diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index 805e88472..66ac974e3 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -11,7 +11,7 @@ use crate::error::{MetadataError, Result}; use crate::validation::validate_content_hash; const RUN_MAGIC: &[u8; 8] = b"CRBRUN01"; -const RUN_VERSION: u32 = 3; +const RUN_VERSION: u32 = 4; const RUN_TRAILER_BYTES: usize = 8 + 32 + RUN_MAGIC.len(); pub(super) const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_INLINE_RUN_CONTROL_SECTION_BYTES: usize = 512 * 1024; @@ -70,13 +70,17 @@ struct CapsuleRunFooter { #[serde(default)] admission: Option, admission_pages: Option, + catalog: Option, + catalog_pages: Option, } mod admission; +mod catalog; use admission::AdmissionPages; pub use admission::CapsuleRunAdmission; #[cfg(feature = "storage")] pub use admission::CapsuleRunAdmissionReader; +use catalog::CatalogPages; /// Immutable bounded run of complete push capsules. #[derive(Debug, Clone, PartialEq, Eq)] @@ -369,6 +373,7 @@ impl CapsuleRunControl { .ok_or_else(|| corrupt("capsule run range overflowed"))?; } expected_offset = validate_index_pool(&footer, expected_offset, control_offset)?; + expected_offset = validate_catalog(&footer, expected_offset, control_offset)?; if expected_offset != control_offset { return Err(corrupt( "capsule run controls do not cover its complete body", @@ -960,6 +965,15 @@ impl CapsuleRun { } else { None }; + let catalog = CatalogPages::build(&capsules)?.map(|(bytes, descriptor)| { + let range = RunSectionLocation { + offset: body.len() as u64, + length: bytes.len() as u64, + hash: blake3::hash(&bytes).to_hex().to_string(), + }; + body.extend_from_slice(&bytes); + (range, descriptor) + }); let admission = admission .filter(|admission| !admission.members.is_empty()) .map(|admission| { @@ -995,6 +1009,8 @@ impl CapsuleRun { capsules: locations.clone(), git_packs: git_packs.clone(), index_pool: index_pool.clone(), + catalog: catalog.as_ref().map(|(range, _)| range.clone()), + catalog_pages: catalog.as_ref().map(|(_, descriptor)| descriptor.clone()), admission: admission.as_ref().map(|(_, range)| range.clone()), admission_pages: admission .as_ref() @@ -1125,7 +1141,16 @@ impl CapsuleRun { } } } - expected_offset = pool_end; + expected_offset = validate_catalog(&footer, pool_end, footer_start as u64)?; + if let (Some(range), Some(pages)) = (&footer.catalog, &footer.catalog_pages) { + let body = bytes + .get(range.offset as usize..expected_offset as usize) + .ok_or_else(|| corrupt("run catalog is outside its object"))?; + if blake3::hash(body).to_hex().as_str() != range.hash { + return Err(corrupt("run catalog hash differs")); + } + pages.verify(body, &capsules)?; + } if footer.admission.is_some() != footer.admission_pages.is_some() { return Err(corrupt( "capsule run admission descriptor presence differs from its body", @@ -1569,8 +1594,9 @@ fn validate_index_pool( return Ok(capsule_end); }; let end = footer - .admission + .catalog .as_ref() + .or(footer.admission.as_ref()) .map_or(control_offset, |range| range.offset); validate_body_range(pool, capsule_end, end)?; let length = footer.git_packs.iter().try_fold(0_u64, |total, member| { @@ -1586,6 +1612,23 @@ fn validate_index_pool( Ok(end) } +fn validate_catalog(footer: &CapsuleRunFooter, offset: u64, control_offset: u64) -> Result { + CatalogPages::validate_presence(footer.catalog_pages.as_ref(), &footer.capsules)?; + match (&footer.catalog, &footer.catalog_pages) { + (None, None) => Ok(offset), + (Some(range), Some(pages)) => { + let end = footer + .admission + .as_ref() + .map_or(control_offset, |range| range.offset); + validate_body_range(range, offset, end)?; + pages.validate_region(range.length, footer.capsules.len())?; + Ok(end) + } + _ => Err(corrupt("run catalog section and page roots differ")), + } +} + fn validate_body_range( range: &RunSectionLocation, expected_offset: u64, @@ -1909,7 +1952,7 @@ mod tests { #[test] fn unsupported_run_footer_versions_are_rejected() { let run = CapsuleRun::leaf(capsule('1', '2')).unwrap(); - for version in [0, 1, 2, 4, 5, 6, 7] { + for version in [0, 1, 2, 3, 5, 6, 7] { let mut footer = run.footer.clone(); footer.version = version; let footer = serde_json::to_vec(&footer).unwrap(); diff --git a/crates/crab-metadata/src/capsule_protocol/run/catalog.rs b/crates/crab-metadata/src/capsule_protocol/run/catalog.rs new file mode 100644 index 000000000..c1f9e393e --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/run/catalog.rs @@ -0,0 +1,311 @@ +//! Run-wide catalog keys with exact logical-capsule occurrence membership. +use super::*; +use crate::pages::{Input, Kind, PageRef, Rows, Writer}; +use std::collections::{BTreeMap, BTreeSet}; + +#[cfg(test)] +mod tests; + +const MAGIC: &[u8; 8] = b"CRBRCG01"; +const MAX_DESCRIPTOR_BYTES: usize = 16 * 1024; +const MAX_BYTES: u64 = 2 * 1024 * 1024 * 1024; +// Values contain fixed-width file/placement rows or one bounded PageRef row. +const MAX_VALUE_BYTES: usize = 2048; +const KINDS: [Kind; 4] = [ + Kind::RunCatalogFiles, + Kind::RunCatalogShards, + Kind::RunCatalogXorbs, + Kind::RunCatalogChunks, +]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct CatalogPages { + version: u32, + capsule_bodies: Vec>, + body_bytes: u64, + body_hash: [u8; 32], + roots: [Option; 4], +} + +struct Variant { + identity: [u8; 32], + source: u16, + members: Vec, + value: Vec, +} + +fn json(value: &T) -> Result> { + serde_json::to_vec(value).map_err(|source| MetadataError::PointerCatalogRecord { source }) +} + +impl CatalogPages { + pub(super) fn build(capsules: &[Capsule]) -> Result> { + let mut directories = KINDS.map(|_| BTreeMap::<_, Variant>::new()); + let mut capsule_bodies = Vec::with_capacity(capsules.len()); + let mut admitted = 0_u64; + for (position, capsule) in capsules.iter().enumerate() { + let position = u16::try_from(position) + .map_err(|_| corrupt("catalog capsule position overflows"))?; + let Some(bytes) = section_bytes_for_kind(capsule, CapsuleSectionKind::CatalogDelta)? + else { + capsule_bodies.push(None); + continue; + }; + let (body_bytes, rows) = super::super::pointer::paged::index_rows(&bytes)?; + capsule_bodies.push(Some(body_bytes)); + for (directory, rows) in directories.iter_mut().zip(rows) { + for row in rows { + if row.value.len() > MAX_VALUE_BYTES { + return Err(corrupt("run catalog value exceeds its bound")); + } + let key = (row.key, row.identity); + match directory.entry(key) { + std::collections::btree_map::Entry::Vacant(entry) => { + admitted = admitted.saturating_add(row.value.len() as u64 + 128); + entry.insert(Variant { + identity: row.identity, + source: position, + members: vec![position], + value: row.value, + }); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + admitted = admitted.saturating_add(2); + entry.get_mut().members.push(position); + } + } + if admitted > MAX_BYTES { + return Err(corrupt("run catalog exceeds its byte bound")); + } + } + } + } + if directories.iter().all(BTreeMap::is_empty) { + return Ok(None); + } + let mut writer = Writer::default(); + let mut roots = std::array::from_fn(|_| None); + for (position, directory) in directories.into_iter().enumerate() { + let mut rows: Rows = Vec::new(); + let mut key = None; + let mut variants = Vec::new(); + for ((next, _), variant) in directory { + if key.as_ref().is_some_and(|key| key != &next) { + if let Some(key) = key.take() { + rows.push((key, encode_variants(&variants)?)); + } + variants.clear(); + } + key = Some(next); + variants.push(variant); + } + if let Some(key) = key { + rows.push((key, encode_variants(&variants)?)); + } + if !rows.is_empty() { + roots[position] = Some(writer.tree(KINDS[position], rows)?); + } + if writer.bytes.len() as u64 > MAX_BYTES { + return Err(corrupt("run catalog exceeds its byte bound")); + } + } + let descriptor = Self { + version: 1, + capsule_bodies, + body_bytes: writer.bytes.len() as u64, + body_hash: *blake3::hash(&writer.bytes).as_bytes(), + roots, + }; + let header = json(&descriptor)?; + if header.len() > MAX_DESCRIPTOR_BYTES { + return Err(corrupt("run catalog descriptor exceeds its byte bound")); + } + writer.bytes.extend_from_slice(&header); + crate::pages::put_u32(&mut writer.bytes, header.len())?; + writer.bytes.extend_from_slice(MAGIC); + descriptor.validate_region(writer.bytes.len() as u64, capsules.len())?; + Ok(Some((writer.bytes.into(), descriptor))) + } + + pub(super) fn validate_presence( + pages: Option<&Self>, + capsules: &[RunCapsuleLocation], + ) -> Result<()> { + let empty = PointerCatalog::new().encode_delta()?; + let empty_hash = blake3::hash(&empty).to_hex().to_string(); + if pages.is_some_and(|pages| pages.capsule_bodies.len() != capsules.len()) { + return Err(corrupt( + "run catalog source count differs from its capsules", + )); + } + for (position, capsule) in capsules.iter().enumerate() { + let body = pages.and_then(|pages| pages.capsule_bodies[position]); + match (&capsule.catalog, body) { + (None, None) => {} + (Some(range), None) + if pages.is_none() + && range.hash == empty_hash + && range.length == empty.len() as u64 => {} + (Some(range), Some(body)) + if body + .checked_add(12) + .is_some_and(|length| length < range.length) + && (body != 0 + || (range.hash == empty_hash + && range.length == empty.len() as u64)) => {} + _ => { + return Err(corrupt( + "run catalog sources differ from their capsule sections", + )); + } + } + } + Ok(()) + } + + pub(super) fn validate_region(&self, length: u64, capsule_count: usize) -> Result<()> { + if self.version != 1 + || capsule_count == 0 + || capsule_count > MAX_CAPSULES_PER_RUN + || self.capsule_bodies.len() != capsule_count + || self.body_bytes == 0 + || self.roots.iter().all(Option::is_none) + || json(self)?.len() > MAX_DESCRIPTOR_BYTES + || self + .body_bytes + .checked_add(json(self)?.len() as u64) + .and_then(|n| n.checked_add(12)) + != Some(length) + || length > MAX_BYTES + || self + .capsule_bodies + .iter() + .flatten() + .any(|bytes| *bytes > MAX_BYTES) + { + return Err(corrupt("run catalog descriptor has an invalid shape")); + } + for (root, kind) in self.roots.iter().zip(KINDS) { + if let Some(root) = root { + root.validate(kind, self.body_bytes)?; + } + } + Ok(()) + } + + pub(super) fn verify(&self, bytes: &[u8], capsules: &[Capsule]) -> Result<()> { + self.validate_region(bytes.len() as u64, capsules.len())?; + let header = json(self)?; + let start = self.body_bytes as usize; + if bytes.get(start..start + header.len()) != Some(header.as_slice()) + || bytes.get(bytes.len() - 8..) != Some(MAGIC.as_slice()) + || u32::from_le_bytes( + bytes[bytes.len() - 12..bytes.len() - 8] + .try_into() + .map_err(|_| corrupt("run catalog trailer is invalid"))?, + ) as usize + != header.len() + || blake3::hash(&bytes[..start]).as_bytes() != &self.body_hash + { + return Err(corrupt("run catalog body differs from its descriptor")); + } + let mut visited = BTreeMap::new(); + for (root, kind) in self.roots.iter().zip(KINDS) { + if let Some(root) = root { + for (_, value) in crate::pages::scan( + &bytes[..start], + root, + kind, + "run catalog pages", + &mut visited, + )? { + decode_variants(&value, capsules.len())?; + } + } + } + // Footer roots cannot invent occurrence membership. Full verification + // ties every variant and physical reference back to the nested capsules. + let expected = Self::build(capsules)? + .ok_or_else(|| corrupt("run catalog has no nested catalog contribution"))?; + if expected.0.as_ref() != bytes || &expected.1 != self { + return Err(corrupt("run catalog does not match its nested capsules")); + } + Ok(()) + } +} + +fn encode_variants(variants: &[Variant]) -> Result> { + let mut bytes = Vec::new(); + put_u16(&mut bytes, variants.len())?; + for variant in variants { + bytes.extend_from_slice(&variant.identity); + bytes.extend_from_slice(&variant.source.to_le_bytes()); + put_u16(&mut bytes, variant.members.len())?; + for member in &variant.members { + bytes.extend_from_slice(&member.to_le_bytes()); + } + put_u16(&mut bytes, variant.value.len())?; + bytes.extend_from_slice(&variant.value); + } + Ok(bytes) +} + +fn decode_variants(bytes: &[u8], capsule_count: usize) -> Result> { + let mut input = Input::new(bytes, "run catalog variants"); + let count = u16::from_le_bytes(input.array()?) as usize; + if count == 0 || count > capsule_count { + return Err(corrupt("run catalog variant count is invalid")); + } + let mut output: Vec = Vec::with_capacity(count); + let mut seen = BTreeSet::new(); + for _ in 0..count { + let identity = input.array()?; + if output + .last() + .is_some_and(|previous| previous.identity >= identity) + { + return Err(corrupt("run catalog variants are not sorted and unique")); + } + let source = u16::from_le_bytes(input.array()?); + let count = u16::from_le_bytes(input.array()?) as usize; + if count == 0 || count > capsule_count { + return Err(corrupt("run catalog occurrence count is invalid")); + } + let mut members = Vec::with_capacity(count); + for _ in 0..count { + let member = u16::from_le_bytes(input.array()?); + if member as usize >= capsule_count + || members.last().is_some_and(|previous| *previous >= member) + || !seen.insert(member) + { + return Err(corrupt("run catalog occurrences are invalid or overlap")); + } + members.push(member); + } + if members.first() != Some(&source) { + return Err(corrupt("run catalog source is not its first occurrence")); + } + let length = u16::from_le_bytes(input.array()?) as usize; + if length == 0 || length > MAX_VALUE_BYTES { + return Err(corrupt("run catalog value has an invalid length")); + } + output.push(Variant { + identity, + source, + members, + value: input.take(length)?.to_vec(), + }); + } + input.finish()?; + Ok(output) +} + +fn put_u16(bytes: &mut Vec, value: usize) -> Result<()> { + bytes.extend_from_slice( + &u16::try_from(value) + .map_err(|_| corrupt("run catalog scalar overflows"))? + .to_le_bytes(), + ); + Ok(()) +} diff --git a/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs b/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs new file mode 100644 index 000000000..5bd4a4686 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs @@ -0,0 +1,208 @@ +use super::*; +use crate::capsule_protocol::{ + CapsuleRefEdit, CapsuleSection, FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, + XorbChunkEntry, +}; + +fn catalog(shard: &str, extra: bool) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + "8".repeat(64), + XorbCatalogEntry::new( + 99, + "a".repeat(64), + vec![XorbChunkEntry::new("9".repeat(64), 7)], + ), + ) + .unwrap(); + catalog + .insert_shard(shard, ShardCatalogEntry::new(80, vec!["8".repeat(64)])) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(7, shard)) + .unwrap(); + if extra { + catalog + .insert_xorb( + "1".repeat(64), + XorbCatalogEntry::new( + 100, + "c".repeat(64), + vec![XorbChunkEntry::new("2".repeat(64), 8)], + ), + ) + .unwrap(); + } + catalog +} + +fn capsule(position: usize, catalog: Option) -> Capsule { + let transaction = CapsuleTransaction::new( + &"e".repeat(64), + vec![CapsuleRefEdit::new( + format!("refs/tags/{position}"), + None, + Some("f".repeat(40)), + None, + )], + ) + .unwrap(); + let sections = catalog + .into_iter() + .map(|catalog| { + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + ) + }) + .collect(); + Capsule::build(&transaction, Vec::new(), sections).unwrap() +} + +fn rows(bytes: &Bytes, descriptor: &CatalogPages, position: usize) -> Rows { + crate::pages::scan( + &bytes[..descriptor.body_bytes as usize], + descriptor.roots[position].as_ref().unwrap(), + KINDS[position], + "run catalog", + &mut BTreeMap::new(), + ) + .unwrap() +} + +#[test] +fn relocated_equal_descriptors_share_one_variant_and_retain_every_occurrence() { + let capsules = vec![ + capsule(0, Some(catalog(&"d".repeat(64), false))), + capsule(1, Some(catalog(&"d".repeat(64), true))), + ]; + let (bytes, descriptor) = CatalogPages::build(&capsules).unwrap().unwrap(); + descriptor.verify(&bytes, &capsules).unwrap(); + let values = rows(&bytes, &descriptor, 2); + let value = &values.iter().find(|(key, _)| key == &[0x88; 32]).unwrap().1; + let variants = decode_variants(value, 2).unwrap(); + assert_eq!( + variants + .iter() + .map(|variant| (variant.source, variant.members.as_slice())) + .collect::>(), + vec![(0, &[0, 1][..])] + ); +} + +#[test] +fn conflicting_file_bindings_keep_disjoint_capsule_membership() { + let capsules = vec![ + capsule(0, Some(catalog(&"d".repeat(64), false))), + capsule(1, Some(catalog(&"e".repeat(64), false))), + ]; + let (bytes, descriptor) = CatalogPages::build(&capsules).unwrap().unwrap(); + descriptor.verify(&bytes, &capsules).unwrap(); + let values = rows(&bytes, &descriptor, 0); + let variants = decode_variants(&values[0].1, 2).unwrap(); + let members = variants + .into_iter() + .map(|variant| variant.members) + .collect::>(); + assert_eq!(members, BTreeSet::from([vec![0], vec![1]])); +} + +#[test] +fn catalog_page_roots_and_full_run_agree_after_compaction() { + let runs = (0..3) + .map(|position| { + CapsuleRun::leaf(capsule( + position, + Some(catalog(&"d".repeat(64), position > 0)), + )) + .unwrap() + }) + .collect(); + let run = CapsuleRun::compact(runs).unwrap(); + let control = CapsuleRunControl::decode_suffix( + run.bytes().slice(run.control_offset() as usize..), + run.bytes().len() as u64, + run.hash(), + run.level(), + &run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + assert_eq!(control.capsule_locations().len(), 3); +} + +#[test] +fn catalog_regions_preserve_index_pool_and_admission_boundaries() { + let runs = (0..2) + .map(|position| { + let metadata = catalog(&"d".repeat(64), false); + let transaction = capsule(position, None).transaction().unwrap(); + let pack = crate::capsule_protocol::CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "1".repeat(40), + 1, + ) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![pack], + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + metadata.encode_delta().unwrap(), + )], + ) + .unwrap(); + CapsuleRun::leaf_with_member_oids(capsule, vec![vec![[position as u8; 20]]]).unwrap() + }) + .collect(); + let run = CapsuleRun::compact(runs).unwrap(); + let (control, _) = run.verified_controls().unwrap(); + assert_eq!(control.admission().unwrap().entries().count(), 2); +} + +#[test] +fn omitted_catalog_roots_cannot_hide_nonempty_capsule_data() { + let run = CapsuleRun::leaf(capsule(0, Some(catalog(&"d".repeat(64), false)))).unwrap(); + assert!(CatalogPages::validate_presence(None, &run.footer.capsules).is_err()); +} + +#[test] +fn empty_catalog_sections_do_not_require_a_run_index() { + let run = CapsuleRun::compact(vec![ + CapsuleRun::leaf(capsule(0, Some(PointerCatalog::new()))).unwrap(), + CapsuleRun::leaf(capsule(1, None)).unwrap(), + ]) + .unwrap(); + assert!(run.footer.catalog.is_none()); +} + +#[test] +fn authenticated_occurrence_changes_still_fail_full_capsule_binding() { + let capsules = vec![ + capsule(0, Some(catalog(&"d".repeat(64), false))), + capsule(1, None), + ]; + let (bytes, mut descriptor) = CatalogPages::build(&capsules).unwrap().unwrap(); + let mut body = bytes[..descriptor.body_bytes as usize].to_vec(); + let root = descriptor.roots[0].as_mut().unwrap(); + // Keep the row structurally valid while falsely attributing it to capsule 1. + let value = root.offset as usize + 14 + 32 + 4; + body[value + 2 + 32] = 1; + body[value + 2 + 32 + 2 + 2] = 1; + root.hash = + *blake3::hash(&body[root.offset as usize..root.offset as usize + root.bytes as usize]) + .as_bytes(); + descriptor.body_hash = *blake3::hash(&body).as_bytes(); + let header = json(&descriptor).unwrap(); + body.extend_from_slice(&header); + crate::pages::put_u32(&mut body, header.len()).unwrap(); + body.extend_from_slice(MAGIC); + let error = descriptor.verify(&body, &capsules).unwrap_err(); + assert!( + matches!(error, MetadataError::CorruptObject { reason, .. } if reason.contains("nested capsules")) + ); +} diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index d0bfc229c..63a1f40cb 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -69,6 +69,10 @@ pub(crate) enum Kind { CatalogChunks = 23, CatalogXorbChunks = 24, CatalogShardXorbs = 25, + RunCatalogFiles = 26, + RunCatalogShards = 27, + RunCatalogXorbs = 28, + RunCatalogChunks = 29, } impl Kind { @@ -97,7 +101,11 @@ impl Kind { | Self::CatalogShards | Self::CatalogXorbs | Self::CatalogChunks - | Self::CatalogShardXorbs => 32, + | Self::CatalogShardXorbs + | Self::RunCatalogFiles + | Self::RunCatalogShards + | Self::RunCatalogXorbs + | Self::RunCatalogChunks => 32, Self::CommitOids | Self::RunAdmission | Self::VisibilityChanges @@ -370,7 +378,7 @@ pub(crate) fn decode( Kind::RunAdmission | Kind::RunAdmissionCounts => "capsule_admission", Kind::VisibilityRefs | Kind::VisibilityChanges => "capsule_visibility", Kind::CheckpointObjects | Kind::CheckpointRefs | Kind::CheckpointSets | Kind::CheckpointTransitions => "checkpoint_visibility_pages", - Kind::CatalogFiles | Kind::CatalogShards | Kind::CatalogXorbs | Kind::CatalogChunks | Kind::CatalogXorbChunks | Kind::CatalogShardXorbs => "pointer_catalog_pages", + Kind::CatalogFiles | Kind::CatalogShards | Kind::CatalogXorbs | Kind::CatalogChunks | Kind::CatalogXorbChunks | Kind::CatalogShardXorbs | Kind::RunCatalogFiles | Kind::RunCatalogShards | Kind::RunCatalogXorbs | Kind::RunCatalogChunks => "pointer_catalog_pages", _ => "path_state", }, decoded_bytes=bytes.len() as u64, "index page decoded"); From 3c04c1fc1d0afa0fb7527a0084966db87bde11fd Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 18:59:37 -0700 Subject: [PATCH 045/127] feat(metadata): read selected run catalog pages Capture exact capsule occurrence order across checkpoint and run sources. Remove eager delta lookups and inline catalog bodies while retaining explicit maintenance inspection and catalog commitment identity. Run-v4 live qualification, shared receive/hydration wiring and large-file lifecycle gates remain incomplete. --- .../docs/architecture/indexed-remote-reads.md | 83 +++++ crab/src/git/xet_publication.rs | 60 ++-- crates/crab-metadata/README.md | 13 +- .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/pointer/paged.rs | 14 +- .../capsule_protocol/pointer/paged/reader.rs | 254 +++++++++------- .../pointer/paged/reader/tests.rs | 231 +++++++++++++- .../crab-metadata/src/capsule_protocol/run.rs | 284 ++++++++++-------- .../src/capsule_protocol/run/catalog.rs | 6 + .../capsule_protocol/run/catalog/reader.rs | 149 +++++++++ .../src/capsule_protocol/run/catalog/tests.rs | 74 +++++ .../src/capsule_protocol/store.rs | 7 +- crates/crab-read/src/capsule_protocol.rs | 193 ++++++++++-- 13 files changed, 1074 insertions(+), 298 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 20529a11e..683137cab 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1522,3 +1522,86 @@ lookups or the fixed-edit history-scaling gate. CLI publication qualification is pending. Its corrupt-catalog fixture currently attempts to publish malformed catalog bytes, which the new producer validates earlier; that boundary needs explicit qualification before this format change is ready to land. + +The next reader slice captures run catalog roots and original source extents +from already authenticated run footers. Selected queries retain exact logical +capsule order across physical sources. File bindings select the latest visible +occurrence with consistent size; shard/xorb conflicts include every selected +variant; reverse chunks select the lowest xorb and its first occurrence. +All selected pages, including nested original recipes and warm cache hits, +share one query byte budget. Nested shard/xorb values must match the aggregate +semantic commitment after expansion. + +The Git metadata view now captures these sources without decoding delta +catalogs. Run-v4 footers no longer inline catalog bodies, and the materialized +delta lookup API is removed. Bulk maintenance retains explicit materialization +of captured sections with their length/hash checks. Complete and paged capsule +controls compare the same catalog section commitment, preserving logical +identity across compaction. + +Reader tests cover selected/unselected conflicts, logical order independent of +physical order, equal variants reusing an unselected source, repeated chunks, +page corruption, and fixed requests against 100 versus 20,000 catalog entries. +A read-view integration test holds one file query fixed against 100 versus +10,000 entries and checks bounded opening/query bytes, warm origin silence and +the full maintenance oracle. The storage-enabled metadata capsule suite passes +123 tests, including selected-run queries and corruption. A separately added +regression proves one aggregate page budget applies across two runs even when +the first run is warm. Metadata/read compilation and strict all-target Clippy +with storage/remote-index features pass. The additional control-identity case +also passes from the newly compiled metadata test binary: complete and paged +controls compare equally, explicit materialization matches the catalog, and a +changed catalog commitment is unequal. These establish 125 distinct metadata +checks. The 49 read-side and 22 writer capsule tests pass, including the new +read-view integration. The payload-only suite passes 99 tests; it exposed one +new unused catalog-field warning. That internal field is now storage-gated, +while pure decoding still validates the catalog before discarding it. The +minimal-feature suite passes all 99 tests again after that correction; strict +metadata/read all-target Clippy with storage/remote-index also passes. Four +other payload-only warnings predate this slice. A source guard covers 445 Rust +files in the CLI, metadata and read crates; the feature-gating correction is +the only intentional source change after the initial capture. + +The producer-only CLI run completed with seven passing tests and one setup +failure: the corrupt-catalog fixture attempted publication, and the producer +rejected it before `prepare_delta`. Its replacement exercises publication +directly and checks both the root digest and visible refs remain unchanged. +Selected-reader corruption remains covered independently. The replacement CLI +test passes in the fresh CLI run: all eight publication tests pass. The old +producer-only failure remains a rejected receipt, not a passing gate. The fresh +test binary reports the existing Apple unwind-table-size linker warning. + +### Installed checkpoint-reader qualification + +Normal `make install` completed from frozen revision `8090eb68df1` with a +private installation prefix. It built the no-FUSE CLI, FUSE helper, cache +server and remote-helper symlink. Both installed CLI variants pass the 4-MiB +native/Git managed-file workflows: 74 commands and 58 checks each. Coverage +includes staging/push, clone/hydration, Git/remote fsck, checkpoint publication +and independent cold checkpoint hydration with byte-identical payloads. Raw +stored root/checkpoint inspection confirms format 7 and nonempty catalog roots +for both workflows. All 1,215 guarded Rust/manifest files remain unchanged. + +The no-FUSE CLI SHA-256 is +`ce23afdd3054a8c95ac5d117a2a216a9f76baeeedd4709c866246b7ee2f17794`; +the FUSE helper SHA-256 is +`a92a7bcd74322300d4b51f76da73ddceb27936ada8267d60d3e08b616b111b25`. +External receipts are `catalog-lookup-installed-live-r1/receipt.json` and +`catalog-lookup-fuse-managed-live-r1/receipt.json`. These qualify the frozen +checkpoint-reader slice; they do not qualify the newer run-v4 reader, the +10-GiB matrix or representative S3 latency. + +The separate default broad harness stopped after 50 commands and 38 checks at +`missing-manifest-list-fails-closed`: it deletes `/manifest`, while the current +initializer publishes only the capsule `/v2/root`. The initializer contract is +explicitly tested by `remote_initialization_publishes_only_the_capsule_root`; +listing still succeeds through the intact v2 authority. The broad run remains +failed and requires canonical-root corruption fixtures. Its expectation has +not been weakened or suppressed, and the focused receipts do not replace it. +A separate canonical-root live probe passes 34 commands and 18 checks. Listing +and clone reject missing roots with `CRAB-E0032` and corrupt roots with +`CRAB-E0020`; restoring each exact root restores the original branch tip. Its +receipt is `catalog-root-failures-live-r2/receipt.json`. The first probe stopped +on a driver assertion against unborn `HEAD`; the corrected probe commits and +pushes a real fixture before fault injection. Neither probe changes the broad +harness's failed status. diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index e865a5088..251ded49e 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -1101,9 +1101,14 @@ mod tests { (catalog, pointer) } - async fn publish_catalog( + async fn catalog_publication( bytes: Bytes, - ) -> (crab_storage::StoreLayout, RootSnapshot) { + ) -> ( + crab_storage::StoreLayout, + RootSnapshot, + CapsuleTransaction, + Capsule, + ) { let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())); let layout = crab_storage::StoreLayout::new(store, "repo".to_owned()); let base = @@ -1126,6 +1131,13 @@ mod tests { vec![CapsuleSection::new(CapsuleSectionKind::CatalogDelta, bytes)], ) .unwrap(); + (layout, base, transaction, capsule) + } + + async fn publish_catalog( + bytes: Bytes, + ) -> (crab_storage::StoreLayout, RootSnapshot) { + let (layout, base, transaction, capsule) = catalog_publication(bytes).await; let base = crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) .await .unwrap(); @@ -1385,34 +1397,38 @@ mod tests { } #[tokio::test] - async fn known_pointers_reject_corrupted_catalog_pages() { - let (catalog, pointer) = pointer_catalog(1, 1); + async fn corrupted_catalog_pages_are_rejected_before_publication() { + let (catalog, _) = pointer_catalog(1, 1); let encoded = catalog.encode().unwrap(); for truncate in [false, true] { let mut bytes = encoded.to_vec(); if truncate { bytes.pop(); } else { - // Corrupt the first chunk's hash while the outer capsule still - // authenticates these bytes. A known file cannot bypass pages. bytes[22] ^= 1; } - let (layout, base) = publish_catalog(Bytes::from(bytes)).await; - let error = prepare_delta( - &layout, - &base, - std::slice::from_ref(&pointer), - None, - None, - None, - 4, - true, - &CancellationToken::new(), - ) - .await - .unwrap_err(); - assert!(matches!(error, CrabError::CorruptObject { path, reason } - if path == "capsule-protocol pointer catalog" && reason.contains("catalog"))); + let (layout, base, transaction, capsule) = + catalog_publication(Bytes::from(bytes)).await; + let expected_root = base.record().digest().to_owned(); + let error = + crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) + .await + .unwrap_err(); + assert!(matches!(error, crab_write::WriteError::Metadata( + crab_metadata::error::MetadataError::CorruptObject { path, .. }) + if path == "capsule-protocol pointer catalog")); + // Reject before a ref becomes visible: root identity alone cannot + // prove this because per-ref heads publish independently of the root. + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + let refs = crab_read::capsule_protocol::read_visible_refs(&layout) + .await + .unwrap(); + assert_eq!( + (root.record().digest(), refs.is_empty()), + (expected_root.as_str(), true) + ); } } diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 30fc67733..c7752c5a2 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -468,7 +468,16 @@ key directories retain exact capsule occurrences and group equal immutable values by a semantic commitment that excludes physical page offsets. Full run verification rebuilds the index from nested capsules, so re-signed membership changes cannot invent catalog authority. Footer-only reads admit the index -region and source spans without fetching its body. Foreground catalog lookup -still uses materialized deltas; selected run-index reads remain unfinished. +region and source spans without fetching its body. Captured run sources feed +batched file, shard, xorb and reverse-chunk lookup through the checkpoint +reader's shared cache and aggregate budget. Exact selected capsule ordering +controls file precedence; unselected occurrences cannot introduce conflicts. +Nested recipes retain authenticated page references into their original catalog +sections and must match the aggregate's logical-value commitment. +Git controls retain catalog section commitments and captured origins rather +than loading catalog bytes. Full maintenance inspection materializes those +captured sections explicitly. Run footers contain catalog roots, never inline +catalog bodies. This reader wiring is under qualification; it does not establish +the complete history-scaling, lifecycle or 10 GiB acceptance gates. The affected run format is absent from the audited release tags; unshipped version 3 is rejected rather than retained through a fallback. diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 5df613267..a93cdb337 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -56,13 +56,13 @@ pub use root::{ CapsulePointer, CheckpointPointer, GcFence, MAX_CAPSULE_FRONTIER, MAX_ROOT_BYTES, RepositoryRoot, RootRecord, }; -#[cfg(feature = "storage")] -pub use run::CapsuleRunAdmissionReader; pub use run::{ CapsuleControl, CapsuleControlLocation, CapsuleRun, CapsuleRunAdmission, CapsuleRunControl, MAX_CAPSULES_PER_RUN, }; #[cfg(feature = "storage")] +pub use run::{CapsuleRunAdmissionReader, RunCatalogSource}; +#[cfg(feature = "storage")] pub use store::{ RootSnapshot, create_root, load_capsule_run, load_capsule_run_control, load_capsule_run_git_control, load_history_chain, load_history_segment, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index 4af634053..61346bb59 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -93,20 +93,24 @@ pub(in crate::capsule_protocol) fn index_rows( )?, _ => value.clone(), }; - let mut identity = blake3::Hasher::new(); - identity.update(b"crab.catalog.value.v1\0"); - identity.update(&[kind as u8]); - identity.update(&normalized); output[position].push(CatalogIndexRow { key, value, - identity: *identity.finalize().as_bytes(), + identity: semantic_identity(kind, &normalized), }); } } Ok((descriptor.body_bytes, output)) } +pub(in crate::capsule_protocol) fn semantic_identity(kind: Kind, bytes: &[u8]) -> [u8; 32] { + let mut hash = blake3::Hasher::new(); + hash.update(b"crab.catalog.value.v1\0"); + hash.update(&[kind as u8]); + hash.update(bytes); + *hash.finalize().as_bytes() +} + fn hash(value: &str) -> Result<[u8; 32]> { let bytes = value.as_bytes(); // Binary keys cannot distinguish hex aliases. Reject them instead of silently diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index 954cb1570..1f96eb221 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -1,9 +1,11 @@ -//! Selected dependency closure from captured checkpoint roots and ordered deltas. +//! Selected dependency closure from captured checkpoint and run roots. use super::*; -use crate::capsule_protocol::PackRange; +use crate::capsule_protocol::run::catalog::{CatalogOrigin, CatalogValue}; +use crate::capsule_protocol::{PackRange, RunCatalogSource}; use crate::pages::{PageCache, PageWork}; use crab_storage::Store; use std::collections::BTreeSet; +use std::sync::Arc; #[cfg(test)] mod tests; @@ -15,7 +17,8 @@ mod tests; #[derive(Debug)] pub struct PointerCatalogLookup { checkpoint: Option, - deltas: Vec, + runs: Vec, + ranks: BTreeMap, cache: PageCache, maximum: u64, } @@ -44,35 +47,50 @@ impl PointerCatalogLookup { range, descriptor, }), - deltas: Vec::new(), + runs: Vec::new(), + ranks: BTreeMap::new(), cache: PageCache::default(), maximum, }) } - /// Capture publication-ordered deltas after the checkpoint represented here. - /// - /// Deltas must be the selected, deduplicated capsules of the same view. - /// This performs no mutable-root reads or dependency expansion. - pub fn with_deltas(mut self, deltas: Vec) -> Result { - for delta in &deltas { - delta.validate(false, true)?; - } - self.deltas.extend(deltas); - Ok(self) - } - /// Start a lookup for a captured view that has no checkpoint catalog. #[must_use] pub fn empty(maximum: u64) -> Self { Self { checkpoint: None, - deltas: Vec::new(), + runs: Vec::new(), + ranks: BTreeMap::new(), cache: PageCache::default(), maximum, } } + /// Capture physical runs and the exact selected logical-capsule publication order. + /// + /// Unselected occurrences contribute neither file precedence nor conflicts. + pub fn with_run_sources( + mut self, + runs: Vec, + order: &[String], + ) -> Result { + if !self.runs.is_empty() || !self.ranks.is_empty() { + return Err(corrupt("run catalog sources have already been captured")); + } + for (index, hash) in order.iter().enumerate() { + crate::validation::validate_content_hash( + hash, + "catalog capsule", + "capsule-protocol pointer catalog", + )?; + if self.ranks.insert(hash.clone(), index + 1).is_some() { + return Err(corrupt("selected catalog capsules are not unique")); + } + } + self.runs = runs; + Ok(self) + } + fn work(&self) -> PageWork { PageWork { maximum: self.maximum, @@ -88,30 +106,53 @@ impl PointerCatalogLookup { kind: Kind, keys: &[Vec], work: &mut PageWork, - ) -> Result>>> { - let Some(source) = &self.checkpoint else { - return Ok(vec![None; keys.len()]); - }; - let Some(root) = root else { - return Ok(vec![None; keys.len()]); - }; - self.cache - .lookup( - store, - &source.path, - 0, - source.range.offset()..source.range.offset() + source.descriptor.body_bytes, - root, - kind, - keys, - work, - ) - .await + ) -> Result>> { + let mut output = (0..keys.len()).map(|_| Vec::new()).collect::>(); + if let (Some(source), Some(root)) = (&self.checkpoint, root) { + let origin = Arc::new(CatalogOrigin { + path: source.path.clone(), + range: source.range.offset()..source.range.offset() + source.descriptor.body_bytes, + layer: 0, + }); + let rows = self + .cache + .lookup( + store, + &origin.path, + origin.layer, + origin.range.clone(), + root, + kind, + keys, + work, + ) + .await?; + for (result, row) in output.iter_mut().zip(rows) { + if let Some(bytes) = row { + result.push(CatalogValue { + origin: origin.clone(), + bytes, + identity: None, + rank: 0, + }); + } + } + } + for (index, run) in self.runs.iter().enumerate() { + let rows = run + .lookup(store, &self.cache, index + 1, &self.ranks, kind, keys, work) + .await?; + for (result, rows) in output.iter_mut().zip(rows) { + result.extend(rows); + } + } + Ok(output) } async fn scan( &self, store: &Store, + origin: &CatalogOrigin, root: Option<&PageRef>, kind: Kind, work: &mut PageWork, @@ -119,16 +160,12 @@ impl PointerCatalogLookup { let Some(root) = root else { return Ok(Vec::new()); }; - let source = self - .checkpoint - .as_ref() - .ok_or_else(|| corrupt("catalog page has no source"))?; self.cache .scan( store, - &source.path, - 0, - source.range.offset()..source.range.offset() + source.descriptor.body_bytes, + &origin.path, + origin.layer, + origin.range.clone(), root, kind, work, @@ -152,10 +189,16 @@ impl PointerCatalogLookup { .await?; let mut catalog = PointerCatalog::new(); for (hash, row) in hashes.iter().zip(rows) { - if let Some(row) = row { - let row: XorbRow = from_json(&row)?; + for value in row { + let row: XorbRow = from_json(&value.bytes)?; let chunks = self - .scan(store, Some(&row.chunks), Kind::CatalogXorbChunks, work) + .scan( + store, + &value.origin, + Some(&row.chunks), + Kind::CatalogXorbChunks, + work, + ) .await?; let chunks = chunks .into_iter() @@ -172,21 +215,9 @@ impl PointerCatalogLookup { Ok(XorbChunkEntry::new(hex(&value[..32])?, size)) }) .collect::>()?; - catalog.insert_xorb( - hash.clone(), - XorbCatalogEntry::new(row.encoded_size, hex(&row.body_digest)?, chunks), - )?; - } - // Every matching immutable descriptor participates, even when a - // later source supplies the placement or file binding selected. - for delta in &self.deltas { - if let Some(entry) = delta.xorbs.get(hash) { - admit( - work, - 72u64.saturating_add((entry.chunks.len() as u64).saturating_mul(36)), - )?; - catalog.insert_xorb(hash.clone(), entry.clone())?; - } + let entry = XorbCatalogEntry::new(row.encoded_size, hex(&row.body_digest)?, chunks); + verify_identity(&value, Kind::CatalogXorbs, &json(&entry)?)?; + catalog.insert_xorb(hash.clone(), entry)?; } } catalog.validate(true, true)?; @@ -214,28 +245,28 @@ impl PointerCatalogLookup { let mut catalog = PointerCatalog::new(); let mut shards = BTreeSet::new(); for (hash, row) in hashes.iter().zip(rows) { - let mut selected = row - .map(|value| { - if value.len() != 40 { - return Err(corrupt("catalog file row has an invalid byte length")); - } - let size = u64::from_be_bytes( - value[..8] - .try_into() - .map_err(|_| corrupt("file size is truncated"))?, - ); - Ok(FileCatalogEntry::new(size, hex(&value[8..])?)) - }) - .transpose()?; - for delta in &self.deltas { - if let Some(entry) = delta.files.get(hash) { - if selected.as_ref().is_some_and(|old| old.size != entry.size) { - return Err(corrupt("file has conflicting declared sizes")); - } - admit(&mut work, 40)?; - selected = Some(entry.clone()); + let mut selected: Option<(usize, FileCatalogEntry)> = None; + for value in row { + verify_identity(&value, Kind::CatalogFiles, &value.bytes)?; + if value.bytes.len() != 40 { + return Err(corrupt("catalog file row has an invalid byte length")); + } + let size = u64::from_be_bytes( + value.bytes[..8] + .try_into() + .map_err(|_| corrupt("file size is truncated"))?, + ); + let entry = FileCatalogEntry::new(size, hex(&value.bytes[8..])?); + if let Some((rank, old)) = &selected + && (old.size != entry.size || (*rank == value.rank && old != &entry)) + { + return Err(corrupt("file has conflicting declared sizes or ordering")); + } + if selected.as_ref().is_none_or(|(rank, _)| *rank < value.rank) { + selected = Some((value.rank, entry)); } } + let selected = selected.map(|(_, entry)| entry); if let Some(entry) = selected { shards.insert(entry.shard_hash.clone()); catalog.insert_file(hash.clone(), entry)?; @@ -250,11 +281,12 @@ impl PointerCatalogLookup { .await?; let mut xorbs = BTreeSet::new(); for (hash, row) in shards.iter().zip(rows) { - if let Some(row) = row { - let row: ShardRow = from_json(&row)?; + for value in row { + let row: ShardRow = from_json(&value.bytes)?; let closure = self .scan( store, + &value.origin, row.xorbs.as_ref(), Kind::CatalogShardXorbs, &mut work, @@ -269,19 +301,9 @@ impl PointerCatalogLookup { hex(&key) }) .collect::>()?; - catalog.insert_shard( - hash.clone(), - ShardCatalogEntry::new(row.encoded_size, closure), - )?; - } - for delta in &self.deltas { - if let Some(entry) = delta.shards.get(hash) { - admit( - &mut work, - 8u64.saturating_add((entry.xorb_hashes.len() as u64).saturating_mul(32)), - )?; - catalog.insert_shard(hash.clone(), entry.clone())?; - } + let entry = ShardCatalogEntry::new(row.encoded_size, closure); + verify_identity(&value, Kind::CatalogShards, &json(&entry)?)?; + catalog.insert_shard(hash.clone(), entry)?; } let entry = catalog .shards @@ -312,7 +334,9 @@ impl PointerCatalogLookup { .await?; let mut selected = BTreeMap::new(); for (hash, row) in hashes.iter().zip(rows) { - if let Some(value) = row { + for row in row { + verify_identity(&row, Kind::CatalogChunks, &row.bytes)?; + let value = row.bytes; if value.len() != 40 { return Err(corrupt( "catalog chunk placement has an invalid byte length", @@ -329,24 +353,12 @@ impl PointerCatalogLookup { .try_into() .map_err(|_| corrupt("chunk size is truncated"))?, ); - selected.insert(hash.clone(), (xorb, index, size)); - } - } - // Deltas are still materialized by the view owner. This scan is bounded - // to the post-checkpoint suffix; run-level indexing replaces it next. - for delta in &self.deltas { - for (xorb, entry) in &delta.xorbs { - for (index, chunk) in entry.chunks.iter().enumerate() { - if !hashes.contains(&chunk.hash) { - continue; - } - let candidate = (xorb.clone(), index as u32, chunk.uncompressed_size); - let placement = selected - .entry(chunk.hash.clone()) - .or_insert_with(|| candidate.clone()); - if candidate < *placement { - *placement = candidate; - } + let candidate = (xorb, index, size); + let placement = selected + .entry(hash.clone()) + .or_insert_with(|| candidate.clone()); + if candidate < *placement { + *placement = candidate; } } } @@ -398,3 +410,15 @@ fn requested(hashes: &[String], work: &mut PageWork) -> Result> } Ok(hashes.iter().cloned().collect()) } + +fn verify_identity(value: &CatalogValue, kind: Kind, normalized: &[u8]) -> Result<()> { + if value + .identity + .is_some_and(|identity| identity != semantic_identity(kind, normalized)) + { + return Err(corrupt( + "run catalog value differs from its semantic commitment", + )); + } + Ok(()) +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs index e8ed058d7..610dce41b 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs @@ -124,7 +124,7 @@ async fn fixed_file_and_chunk_queries_read_selected_closure_as_history_grows() { } #[tokio::test] -async fn ordered_deltas_override_file_bindings_but_preserve_descriptor_conflicts() { +async fn ordered_runs_override_file_bindings_but_preserve_descriptor_conflicts() { let (layout, _, lookup, _) = fixture(catalog(2), 1024 * 1024).await; let file = format!("{:064x}", 0); let mut delta = PointerCatalog::new(); @@ -132,7 +132,7 @@ async fn ordered_deltas_override_file_bindings_but_preserve_descriptor_conflicts delta .insert_file(file.clone(), FileCatalogEntry::new(7, replacement.clone())) .unwrap(); - let lookup = lookup.with_deltas(vec![delta]).unwrap(); + let lookup = append_run(&layout, lookup, vec![delta]).await; assert_eq!( lookup .files(layout.store(), std::slice::from_ref(&file)) @@ -153,7 +153,7 @@ async fn ordered_deltas_override_file_bindings_but_preserve_descriptor_conflicts ), ) .unwrap(); - let lookup = lookup.with_deltas(vec![conflict]).unwrap(); + let lookup = append_run(&layout, lookup, vec![conflict]).await; assert!( lookup .files(layout.store(), &[format!("{:064x}", 1)]) @@ -180,7 +180,7 @@ async fn chunk_selection_uses_lowest_xorb_then_first_occurrence_across_sources() ), ) .unwrap(); - let lookup = lookup.with_deltas(vec![delta]).unwrap(); + let lookup = append_run(&layout, lookup, vec![delta]).await; let selected = lookup.chunks(layout.store(), &[chunk]).await.unwrap(); assert_eq!( selected.xorbs.keys().collect::>(), @@ -222,7 +222,7 @@ async fn loaded_dependencies_and_chunk_placements_fail_closed() { delta .insert_file("f".repeat(64), FileCatalogEntry::new(7, "d".repeat(64))) .unwrap(); - let lookup = lookup.with_deltas(vec![delta]).unwrap(); + let lookup = append_run(&layout, lookup, vec![delta]).await; assert!( lookup .files(layout.store(), &["f".repeat(64)]) @@ -262,14 +262,227 @@ async fn loaded_dependencies_and_chunk_placements_fail_closed() { } #[tokio::test] -async fn query_budget_applies_to_warm_pages_and_materialized_delta_dependencies() { +async fn query_budget_applies_to_warm_pages_and_run_dependencies() { let (layout, _, mut lookup, _) = fixture(catalog(2), 1024 * 1024).await; let files = [format!("{:064x}", 0)]; lookup.files(layout.store(), &files).await.unwrap(); lookup.maximum = 1; assert!(lookup.files(layout.store(), &files).await.is_err()); - let lookup = PointerCatalogLookup::empty(1) - .with_deltas(vec![catalog(2)]) - .unwrap(); + let lookup = append_run(&layout, PointerCatalogLookup::empty(1), vec![catalog(2)]).await; assert!(lookup.files(layout.store(), &files).await.is_err()); } + +async fn run_source( + layout: &StoreLayout, + catalogs: Vec, +) -> (RunCatalogSource, Vec) { + use crate::capsule_protocol::{ + Capsule, CapsuleRefEdit, CapsuleRun, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, + }; + let mut runs: Vec<_> = catalogs + .into_iter() + .enumerate() + .map(|(position, catalog)| { + let transaction = CapsuleTransaction::new( + &"e".repeat(64), + vec![CapsuleRefEdit::new( + format!("refs/tags/{position}"), + None, + Some("f".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )], + ) + .unwrap(); + CapsuleRun::leaf(capsule).unwrap() + }) + .collect(); + let run = if runs.len() == 1 { + runs.pop().unwrap() + } else { + CapsuleRun::compact(runs).unwrap() + }; + layout + .store() + .put_overwrite(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let order = run + .capsules() + .iter() + .map(|capsule| capsule.hash().to_owned()) + .collect(); + let source = run.catalog_source(layout).unwrap().unwrap(); + (source, order) +} + +#[tokio::test] +async fn selected_run_queries_keep_unselected_conflicts_out_of_the_view() { + let (layout, _, lookup, _) = fixture(catalog(2), 1024 * 1024).await; + let mut conflict = catalog(1); + conflict + .xorbs + .get_mut(&format!("{:064x}", 1_000_000)) + .unwrap() + .encoded_size = 100; + let (source, order) = run_source(&layout, vec![catalog(1), conflict]).await; + let file = format!("{:064x}", 0); + let lookup = lookup + .with_run_sources(vec![source.clone()], &order[..1]) + .unwrap(); + assert_eq!( + lookup + .files(layout.store(), std::slice::from_ref(&file)) + .await + .unwrap(), + catalog(1) + ); + let lookup = PointerCatalogLookup::empty(1024 * 1024) + .with_run_sources(vec![source], &order) + .unwrap(); + assert!(lookup.files(layout.store(), &[file]).await.is_err()); +} + +#[tokio::test] +async fn selected_run_file_precedence_uses_logical_order_instead_of_physical_position() { + let (layout, _, lookup, _) = fixture(catalog(2), 1024 * 1024).await; + let file = format!("{:064x}", 0); + let mut older = PointerCatalog::new(); + older + .insert_file( + file.clone(), + FileCatalogEntry::new(7, format!("{:064x}", 3_000_000)), + ) + .unwrap(); + let mut newer = PointerCatalog::new(); + let replacement = format!("{:064x}", 3_000_001); + newer + .insert_file(file.clone(), FileCatalogEntry::new(7, replacement.clone())) + .unwrap(); + let (source, order) = run_source(&layout, vec![newer, older]).await; + let lookup = lookup + .with_run_sources(vec![source], &[order[1].clone(), order[0].clone()]) + .unwrap(); + assert_eq!( + lookup + .files(layout.store(), std::slice::from_ref(&file)) + .await + .unwrap() + .files[&file] + .shard_hash, + replacement + ); +} + +#[tokio::test] +async fn run_lookup_reads_selected_pages_with_one_budget_and_warm_cache() { + for count in [100, 20_000] { + let (layout, _, _, reads) = fixture(PointerCatalog::new(), 512 * 1024).await; + let expected = catalog(count); + let (source, order) = run_source(&layout, vec![expected.clone()]).await; + let lookup = PointerCatalogLookup::empty(512 * 1024) + .with_run_sources(vec![source], &order) + .unwrap(); + reads.store(0, Ordering::SeqCst); + let file = format!("{:064x}", 42); + let selected = lookup + .files(layout.store(), std::slice::from_ref(&file)) + .await + .unwrap(); + assert_eq!(selected.files.get(&file), expected.files.get(&file)); + let chunks = lookup + .chunks(layout.store(), &[format!("{:064x}", 2_000_042)]) + .await + .unwrap(); + assert_eq!(chunks.xorbs, selected.xorbs); + let cold = reads.load(Ordering::SeqCst); + assert!(cold <= 16, "selected run query exceeded its page bound"); + lookup.files(layout.store(), &[file]).await.unwrap(); + assert_eq!(reads.load(Ordering::SeqCst), cold); + } +} + +#[tokio::test] +async fn run_chunks_choose_the_first_occurrence_in_the_lowest_selected_xorb() { + let (layout, _, lookup, _) = fixture(catalog(1), 1024 * 1024).await; + let chunk = format!("{:064x}", 2_000_000); + let mut delta = PointerCatalog::new(); + delta + .insert_xorb( + "0".repeat(64), + XorbCatalogEntry::new( + 88, + "e".repeat(64), + vec![ + XorbChunkEntry::new(chunk.clone(), 7), + XorbChunkEntry::new(chunk.clone(), 7), + ], + ), + ) + .unwrap(); + let (source, order) = run_source(&layout, vec![delta]).await; + let lookup = lookup.with_run_sources(vec![source], &order).unwrap(); + let selected = lookup.chunks(layout.store(), &[chunk]).await.unwrap(); + assert_eq!( + selected.xorbs.keys().collect::>(), + vec![&"0".repeat(64)] + ); +} + +async fn append_run( + layout: &StoreLayout, + mut lookup: PointerCatalogLookup, + catalogs: Vec, +) -> PointerCatalogLookup { + let (source, additional) = run_source(layout, catalogs).await; + let mut ranked = std::mem::take(&mut lookup.ranks) + .into_iter() + .collect::>(); + ranked.sort_by_key(|(_, rank)| *rank); + let mut order = ranked.into_iter().map(|(hash, _)| hash).collect::>(); + order.extend(additional); + let mut sources = std::mem::take(&mut lookup.runs); + sources.push(source); + lookup.with_run_sources(sources, &order).unwrap() +} + +#[tokio::test] +async fn equal_run_variants_reuse_unselected_physical_sources_without_losing_authority() { + let (layout, _, _, _) = fixture(PointerCatalog::new(), 1024 * 1024).await; + let (source, order) = run_source(&layout, vec![catalog(2), catalog(3)]).await; + let lookup = PointerCatalogLookup::empty(1024 * 1024) + .with_run_sources(vec![source], &order[1..]) + .unwrap(); + let selected = lookup + .files(layout.store(), &[format!("{:064x}", 0)]) + .await + .unwrap(); + assert_eq!(selected, catalog(1)); +} + +#[tokio::test] +async fn query_page_budget_is_shared_across_runs_including_warm_sources() { + let (layout, _, _, _) = fixture(PointerCatalog::new(), 128 * 1024).await; + let mut lookup = append_run( + &layout, + PointerCatalogLookup::empty(128 * 1024), + vec![catalog(1000)], + ) + .await; + let file = [format!("{:064x}", 42)]; + lookup.files(layout.store(), &file).await.unwrap(); + lookup = append_run(&layout, lookup, vec![catalog(1001)]).await; + let error = lookup.files(layout.store(), &file).await.unwrap_err(); + assert!( + matches!(error, crate::error::MetadataError::CorruptObject { reason, .. } + if reason.contains("byte limit")) + ); +} diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index 66ac974e3..e20d5eef9 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -14,7 +14,6 @@ const RUN_MAGIC: &[u8; 8] = b"CRBRUN01"; const RUN_VERSION: u32 = 4; const RUN_TRAILER_BYTES: usize = 8 + 32 + RUN_MAGIC.len(); pub(super) const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; -const MAX_INLINE_RUN_CONTROL_SECTION_BYTES: usize = 512 * 1024; /// Largest number of push capsules coalesced before a repository checkpoint. pub const MAX_CAPSULES_PER_RUN: usize = 512; @@ -55,8 +54,6 @@ struct RunSectionLocation { struct RunControlSections { transaction: Vec, visibility_pages: Option, - #[serde(default)] - catalog: Option>, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -75,12 +72,14 @@ struct CapsuleRunFooter { } mod admission; -mod catalog; +pub(in crate::capsule_protocol) mod catalog; use admission::AdmissionPages; pub use admission::CapsuleRunAdmission; #[cfg(feature = "storage")] pub use admission::CapsuleRunAdmissionReader; use catalog::CatalogPages; +#[cfg(feature = "storage")] +pub use catalog::RunCatalogSource; /// Immutable bounded run of complete push capsules. #[derive(Debug, Clone, PartialEq, Eq)] @@ -108,6 +107,7 @@ pub struct CapsuleRunControl { index_pool: Option, admission: Option, admission_info: Option<(RunSectionLocation, AdmissionPages)>, + catalog_info: Option<(RunSectionLocation, CatalogPages)>, } /// Ranges for the transaction and optional metadata sections of one capsule. @@ -122,7 +122,6 @@ pub struct CapsuleControlLocation { catalog: Option, transaction_bytes: Bytes, visibility_pages: Option, - catalog_bytes: Option, } /// Transaction and metadata sections needed to validate a run without reading @@ -135,7 +134,11 @@ pub struct CapsuleControl { transaction: CapsuleTransaction, visibility: Option, visibility_identity: Option<(u64, String, super::visibility::VisibilityPages)>, + #[cfg(feature = "storage")] catalog: Option, + catalog_identity: Option<(u64, String)>, + #[cfg(feature = "storage")] + catalog_origin: Option<(object_store::path::Path, PackRange)>, git_packs: Vec, } @@ -149,14 +152,14 @@ enum ControlVisibility { impl PartialEq for CapsuleControl { fn eq(&self, other: &Self) -> bool { // Transport origins and visibility materialization are not capsule - // identity. Compare the authenticated section commitment and roots so + // identity. Compare catalog/visibility commitments and roots so // compaction can relocate bytes without hiding conflicting evidence. self.hash == other.hash && self.transaction_id == other.transaction_id && self.base_root_digest == other.base_root_digest && self.transaction == other.transaction && self.visibility_identity == other.visibility_identity - && self.catalog == other.catalog + && self.catalog_identity == other.catalog_identity && self.git_packs == other.git_packs } } @@ -340,17 +343,11 @@ impl CapsuleRunControl { )); } } - let catalog_bytes = location.control.catalog.clone().map(Bytes::from); let catalog_range = location .catalog .as_ref() .map(|range| PackRange::from_parts(range.offset, range.length, range.hash.clone())) .transpose()?; - verify_embedded_control_section( - catalog_range.as_ref(), - catalog_bytes.as_ref(), - "catalog", - )?; controls.push(CapsuleControlLocation { hash: location.hash.clone(), capsule_range: location.offset @@ -365,7 +362,6 @@ impl CapsuleRunControl { catalog: catalog_range, transaction_bytes, visibility_pages: location.control.visibility_pages.clone(), - catalog_bytes, }); expected_offset = location .offset @@ -447,9 +443,37 @@ impl CapsuleRunControl { index_pool: footer.index_pool, admission, admission_info, + catalog_info: footer.catalog.zip(footer.catalog_pages), }) } + #[cfg(feature = "storage")] + /// Capture run catalog roots and capsule sources without loading their pages. + pub fn catalog_source( + &self, + layout: &crab_storage::StoreLayout, + ) -> Result> { + self.catalog_info + .as_ref() + .map(|(range, pages)| { + let capsules = self + .capsules + .iter() + .map(|capsule| { + ( + capsule.hash.clone(), + capsule + .catalog + .as_ref() + .map(|range| range.offset()..range.offset() + range.length()), + ) + }) + .collect(); + RunCatalogSource::new(layout, &self.hash, range, pages.clone(), capsules) + }) + .transpose() + } + #[cfg(feature = "storage")] /// Open bounded admission lookup from this run's authenticated footer. pub fn admission_reader( @@ -558,56 +582,75 @@ impl CapsuleRunControl { &self, external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, ) -> Result> { - self.materialize_controls(external_controls, |location| { - let visibility_bytes = external_controls - .get(&(location.hash.clone(), CapsuleSectionKind::VisibilityDelta)) - .cloned(); - if visibility_bytes - .as_ref() - .map(|bytes| super::visibility::VisibilityPages::decode(bytes)) - .transpose()? - != location.visibility_pages - { - return Err(corrupt( - "visibility page descriptor differs from its run footer", - )); - } - let delta = visibility_bytes - .as_ref() - .map(|bytes| CapsuleVisibilityDelta::decode(bytes)) - .transpose()?; - verify_control_section( - location.visibility.as_ref(), - visibility_bytes.as_ref(), - "visibility", - )?; - Ok(delta.map(|delta| ControlVisibility::Complete(std::sync::Arc::new(delta)))) - }) + self.materialize_controls( + |location| { + let visibility_bytes = external_controls + .get(&(location.hash.clone(), CapsuleSectionKind::VisibilityDelta)) + .cloned(); + if visibility_bytes + .as_ref() + .map(|bytes| super::visibility::VisibilityPages::decode(bytes)) + .transpose()? + != location.visibility_pages + { + return Err(corrupt( + "visibility page descriptor differs from its run footer", + )); + } + let delta = visibility_bytes + .as_ref() + .map(|bytes| CapsuleVisibilityDelta::decode(bytes)) + .transpose()?; + verify_control_section( + location.visibility.as_ref(), + visibility_bytes.as_ref(), + "visibility", + )?; + Ok(delta.map(|delta| ControlVisibility::Complete(std::sync::Arc::new(delta)))) + }, + |location| { + let bytes = external_controls + .get(&(location.hash.clone(), CapsuleSectionKind::CatalogDelta)); + verify_control_section(location.catalog.as_ref(), bytes, "catalog")?; + bytes + .map(|bytes| PointerCatalog::decode_delta(bytes)) + .transpose() + }, + ) } #[cfg(feature = "storage")] pub(crate) fn materialize_capsules_with_paged_visibility( &self, layout: &crab_storage::StoreLayout, - external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, ) -> Result> { - self.materialize_controls(external_controls, |location| { - self.visibility_reader( - layout, - &location.hash, - crate::git_visibility::MAX_GIT_VISIBILITY_INDEX_BYTES, - ) - .map(|reader| { - reader.map(|reader| ControlVisibility::Paged(std::sync::Arc::new(reader))) - }) - }) + let mut controls = self.materialize_controls( + |location| { + self.visibility_reader( + layout, + &location.hash, + crate::git_visibility::MAX_GIT_VISIBILITY_INDEX_BYTES, + ) + .map(|reader| { + reader.map(|reader| ControlVisibility::Paged(std::sync::Arc::new(reader))) + }) + }, + |_| Ok(None), + )?; + for (control, location) in controls.iter_mut().zip(&self.capsules) { + control.catalog_origin = location + .catalog + .as_ref() + .map(|range| (layout.capsule_path(&self.hash), range.clone())); + } + Ok(controls) } #[cfg(any(feature = "storage", test))] fn materialize_controls( &self, - external_controls: &std::collections::BTreeMap<(String, CapsuleSectionKind), Bytes>, visibility: impl Fn(&CapsuleControlLocation) -> Result>, + catalog: impl Fn(&CapsuleControlLocation) -> Result>, ) -> Result> { let pack_directories = self.capsule_pack_directories()?; self.capsules @@ -620,26 +663,15 @@ impl CapsuleRunControl { "transaction", )?; let visibility = visibility(location)?; - let catalog_bytes = location.catalog_bytes.clone().or_else(|| { - external_controls - .get(&(location.hash.clone(), CapsuleSectionKind::CatalogDelta)) - .cloned() - }); - verify_control_section( - location.catalog.as_ref(), - catalog_bytes.as_ref(), - "catalog", - )?; let transaction = CapsuleTransaction::decode(&location.transaction_bytes)?; if transaction.id()? != location.transaction_id || transaction.base_root_digest() != location.base_root_digest { return Err(corrupt("capsule run transaction does not match its footer")); } - let catalog = catalog_bytes - .as_ref() - .map(|bytes| PointerCatalog::decode_delta(bytes)) - .transpose()?; + let catalog = catalog(location)?; + #[cfg(not(feature = "storage"))] + drop(catalog); Ok(CapsuleControl { hash: location.hash.clone(), transaction_id: location.transaction_id.clone(), @@ -653,7 +685,14 @@ impl CapsuleRunControl { .map(|(section, pages)| { (section.length(), section.blake3().to_owned(), pages.clone()) }), + #[cfg(feature = "storage")] catalog, + catalog_identity: location + .catalog + .as_ref() + .map(|range| (range.length(), range.blake3().to_owned())), + #[cfg(feature = "storage")] + catalog_origin: None, git_packs, }) }) @@ -765,9 +804,25 @@ impl CapsuleControl { Ok(control) } - /// Return the pointer catalog delta, when present. - pub fn pointer_catalog_delta(&self) -> Option<&PointerCatalog> { - self.catalog.as_ref() + #[cfg(feature = "storage")] + /// Materialize catalog bytes only for explicit bulk inspection of this captured control. + pub async fn materialize_catalog( + &self, + store: &crab_storage::Store, + ) -> Result> { + if let Some(catalog) = &self.catalog { + return Ok(Some(catalog.clone())); + } + let Some((path, range)) = &self.catalog_origin else { + return Ok(None); + }; + let end = range + .offset() + .checked_add(range.length()) + .ok_or_else(|| corrupt("catalog materialization range overflows"))?; + let bytes = store.range_get(path, range.offset()..end).await?; + verify_control_section(Some(range), Some(&bytes), "catalog")?; + PointerCatalog::decode_delta(&bytes).map(Some) } /// Return this capsule's authenticated pack directory with capsule-relative ranges. @@ -827,18 +882,13 @@ impl CapsuleControlLocation { ) }) .into_iter() - .chain( - self.catalog - .as_ref() - .filter(|_| self.catalog_bytes.is_none()) - .map(|range| { - ( - self.hash.clone(), - CapsuleSectionKind::CatalogDelta, - range.clone(), - ) - }), - ) + .chain(self.catalog.as_ref().map(|range| { + ( + self.hash.clone(), + CapsuleSectionKind::CatalogDelta, + range.clone(), + ) + })) } } @@ -921,9 +971,6 @@ impl CapsuleRun { section_bytes_for_kind(capsule, CapsuleSectionKind::VisibilityDelta)? .map(|bytes| super::visibility::VisibilityPages::decode(&bytes)) .transpose()?; - let inline_catalog = section_bytes_for_kind(capsule, CapsuleSectionKind::CatalogDelta)? - .filter(|bytes| bytes.len() <= MAX_INLINE_RUN_CONTROL_SECTION_BYTES) - .map(|bytes| bytes.to_vec()); locations.push(RunCapsuleLocation { offset, length, @@ -936,7 +983,6 @@ impl CapsuleRun { control: RunControlSections { transaction: capsule.section_bytes(0)?.to_vec(), visibility_pages, - catalog: inline_catalog, }, }); } @@ -1002,7 +1048,7 @@ impl CapsuleRun { )) }) .transpose()?; - let footer_bytes = loop { + let footer_bytes = { let footer = CapsuleRunFooter { version: RUN_VERSION, level, @@ -1023,31 +1069,12 @@ impl CapsuleRun { let footer_bytes = serde_json::to_vec(&footer).map_err(|source| { MetadataError::Internal(format!("capsule run serialization failed: {source}")) })?; - if footer_bytes.len() <= MAX_RUN_FOOTER_BYTES { - break footer_bytes; - } - - let largest = locations - .iter() - .enumerate() - .filter_map(|(index, location)| { - location - .control - .catalog - .as_ref() - .map(|bytes| (index, bytes.len())) - }) - .max_by_key(|(_, length)| *length); - let Some((index, _)) = largest else { + if footer_bytes.len() > MAX_RUN_FOOTER_BYTES { return Err(contract_error(format!( "capsule run footer exceeds {MAX_RUN_FOOTER_BYTES} bytes" ))); - }; - locations - .get_mut(index) - .ok_or_else(|| contract_error("capsule run control disappeared"))? - .control - .catalog = None; + } + footer_bytes }; let footer_length = u64::try_from(footer_bytes.len()) .map_err(|_| contract_error("capsule run footer length cannot be represented"))?; @@ -1244,6 +1271,36 @@ impl CapsuleRun { Ok((control, capsules)) } + #[cfg(feature = "storage")] + /// Capture catalog lookup sources from this already verified run. + pub fn catalog_source( + &self, + layout: &crab_storage::StoreLayout, + ) -> Result> { + self.footer + .catalog + .as_ref() + .zip(self.footer.catalog_pages.as_ref()) + .map(|(range, pages)| { + let capsules = self + .footer + .capsules + .iter() + .map(|capsule| { + ( + capsule.hash.clone(), + capsule + .catalog + .as_ref() + .map(|range| range.offset..range.offset + range.length), + ) + }) + .collect(); + RunCatalogSource::new(layout, &self.hash, range, pages.clone(), capsules) + }) + .transpose() + } + /// Return the BLAKE3 object identity of the run. #[must_use] pub fn hash(&self) -> &str { @@ -1530,8 +1587,6 @@ fn validate_control_bundle(capsule: &Capsule, location: &RunCapsuleLocation) -> "visibility page descriptor differs from its capsule", )); } - let embedded_catalog = location.control.catalog.clone().map(Bytes::from); - verify_embedded_control_section(catalog_range.as_ref(), embedded_catalog.as_ref(), "catalog")?; Ok(()) } @@ -1677,17 +1732,6 @@ fn verify_control_section( } } -fn verify_embedded_control_section( - expected: Option<&PackRange>, - actual: Option<&Bytes>, - label: &str, -) -> Result<()> { - match (expected, actual) { - (Some(_), None) => Ok(()), - _ => verify_control_section(expected, actual, label), - } -} - fn contract_error(reason: impl Into) -> MetadataError { MetadataError::CapsuleContract { record: "capsule run", diff --git a/crates/crab-metadata/src/capsule_protocol/run/catalog.rs b/crates/crab-metadata/src/capsule_protocol/run/catalog.rs index c1f9e393e..fad6d5dd4 100644 --- a/crates/crab-metadata/src/capsule_protocol/run/catalog.rs +++ b/crates/crab-metadata/src/capsule_protocol/run/catalog.rs @@ -3,8 +3,14 @@ use super::*; use crate::pages::{Input, Kind, PageRef, Rows, Writer}; use std::collections::{BTreeMap, BTreeSet}; +#[cfg(feature = "storage")] +mod reader; #[cfg(test)] mod tests; +#[cfg(feature = "storage")] +pub use reader::RunCatalogSource; +#[cfg(feature = "storage")] +pub(in crate::capsule_protocol) use reader::{CatalogOrigin, CatalogValue}; const MAGIC: &[u8; 8] = b"CRBRCG01"; const MAX_DESCRIPTOR_BYTES: usize = 16 * 1024; diff --git a/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs b/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs new file mode 100644 index 000000000..e5c65fb7d --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs @@ -0,0 +1,149 @@ +//! Captured run roots and exact selected logical-capsule membership. +use super::*; +use crate::pages::{PageCache, PageWork}; +use crab_storage::{Store, StoreLayout}; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub(in crate::capsule_protocol) struct CatalogOrigin { + pub path: object_store::path::Path, + pub range: std::ops::Range, + pub layer: usize, +} + +pub(in crate::capsule_protocol) struct CatalogValue { + pub origin: Arc, + pub bytes: Vec, + pub identity: Option<[u8; 32]>, + pub rank: usize, +} + +/// Authenticated catalog roots and source ranges captured from one immutable run. +/// +/// Construction is restricted to verified run footers. The owning repository +/// view must retain this run against GC for the lifetime of its queries. +#[derive(Debug, Clone)] +pub struct RunCatalogSource { + path: object_store::path::Path, + range: std::ops::Range, + pages: CatalogPages, + capsules: Vec<(String, Option>)>, +} + +impl RunCatalogSource { + pub(in crate::capsule_protocol::run) fn new( + layout: &StoreLayout, + hash: &str, + range: &RunSectionLocation, + pages: CatalogPages, + capsules: Vec<(String, Option>)>, + ) -> Result { + pages.validate_region(range.length, capsules.len())?; + let end = range + .offset + .checked_add(pages.body_bytes) + .ok_or_else(|| corrupt("run catalog body range overflows"))?; + let capsules = capsules + .into_iter() + .zip(&pages.capsule_bodies) + .map(|((hash, range), body)| { + let range = match (range, body) { + (Some(range), Some(body)) => { + let end = range + .start + .checked_add(*body) + .filter(|end| *end <= range.end) + .ok_or_else(|| { + corrupt("run catalog source body escapes its section") + })?; + Some(range.start..end) + } + (None, None) => None, + _ => return Err(corrupt("run catalog source presence differs")), + }; + Ok((hash, range)) + }) + .collect::>()?; + Ok(Self { + path: layout.capsule_path(hash), + range: range.offset..end, + pages, + capsules, + }) + } + + pub(in crate::capsule_protocol) async fn lookup( + &self, + store: &Store, + cache: &PageCache, + layer: usize, + ranks: &BTreeMap, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>> { + let position = match kind { + Kind::CatalogFiles => 0, + Kind::CatalogShards => 1, + Kind::CatalogXorbs => 2, + Kind::CatalogChunks => 3, + _ => return Err(corrupt("unsupported run catalog directory")), + }; + let mut output = (0..keys.len()).map(|_| Vec::new()).collect::>(); + if !self + .capsules + .iter() + .any(|(hash, _)| ranks.contains_key(hash)) + { + return Ok(output); + } + let Some(root) = &self.pages.roots[position] else { + return Ok(output); + }; + let rows = cache + .lookup( + store, + &self.path, + layer, + self.range.clone(), + root, + KINDS[position], + keys, + work, + ) + .await?; + for (result, row) in output.iter_mut().zip(rows) { + let Some(row) = row else { + continue; + }; + for variant in decode_variants(&row, self.capsules.len())? { + let rank = variant + .members + .iter() + .filter_map(|member| ranks.get(&self.capsules[*member as usize].0)) + .max() + .copied(); + let Some(rank) = rank else { + continue; + }; + let range = self.capsules[variant.source as usize] + .1 + .clone() + .ok_or_else(|| corrupt("run catalog variant has no physical source"))?; + // Membership admits logical values, not all capsules in the + // physical run. Equal variants may reuse an unselected source. + result.push(CatalogValue { + origin: Arc::new(CatalogOrigin { + path: self.path.clone(), + range, + layer, + }), + bytes: variant.value, + identity: Some(variant.identity), + rank, + }); + } + } + Ok(output) + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs b/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs index 5bd4a4686..f6d28b7ad 100644 --- a/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/run/catalog/tests.rs @@ -206,3 +206,77 @@ fn authenticated_occurrence_changes_still_fail_full_capsule_binding() { matches!(error, MetadataError::CorruptObject { reason, .. } if reason.contains("nested capsules")) ); } + +#[cfg(feature = "storage")] +#[tokio::test] +async fn selected_run_reads_reject_corrupt_index_pages_and_source_recipes() { + use crate::capsule_protocol::PointerCatalogLookup; + use crab_storage::{Store, StoreLayout}; + use std::sync::Arc; + for index_page in [false, true] { + let run = CapsuleRun::leaf(capsule(0, Some(catalog(&"d".repeat(64), false)))).unwrap(); + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "run-corruption".into(), + ); + let source = run.catalog_source(&layout).unwrap().unwrap(); + let lookup = PointerCatalogLookup::empty(1024 * 1024) + .with_run_sources(vec![source], &[run.capsules()[0].hash().to_owned()]) + .unwrap(); + let mut bytes = run.bytes().to_vec(); + let offset = if index_page { + run.footer.catalog.as_ref().unwrap().offset + + run.footer.catalog_pages.as_ref().unwrap().roots[0] + .as_ref() + .unwrap() + .offset + } else { + // Nested chunk recipes remain range-addressed in their original + // catalog body; the aggregate cannot bypass their page hashes. + let section = run.footer.capsules[0].catalog.as_ref().unwrap(); + section.offset + }; + bytes[offset as usize] ^= 1; + layout + .store() + .put_overwrite(&layout.capsule_path(run.hash()), bytes.into()) + .await + .unwrap(); + assert!( + lookup + .files(layout.store(), &["b".repeat(64)]) + .await + .is_err() + ); + } +} + +#[cfg(feature = "storage")] +#[tokio::test] +async fn catalog_control_identity_is_independent_of_materialization() { + use crab_storage::{Store, StoreLayout}; + use std::sync::Arc; + let expected = catalog(&"d".repeat(64), false); + let run = CapsuleRun::leaf(capsule(0, Some(expected.clone()))).unwrap(); + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "catalog-identity".into(), + ); + layout + .store() + .put_overwrite(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let (control, full) = run.verified_controls().unwrap(); + let paged = control + .materialize_capsules_with_paged_visibility(&layout) + .unwrap(); + assert_eq!(paged, full); + assert_eq!( + paged[0].materialize_catalog(layout.store()).await.unwrap(), + Some(expected) + ); + let mut conflicting = paged[0].clone(); + conflicting.catalog_identity.as_mut().unwrap().1 = "f".repeat(64); + assert_ne!(conflicting, full[0]); +} diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index 5bd2de359..c5c35d069 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -393,7 +393,7 @@ pub async fn load_capsule_run_control( "capsule pointer footer hash does not match the authenticated run suffix", )); } - let detached = load_detached_control_sections(router, &path, &control, true).await?; + let detached = load_detached_control_sections(router, &path, &control).await?; let capsules = control.materialize_capsules_with_external_controls(&detached)?; Ok((control, capsules)) } @@ -434,8 +434,7 @@ pub async fn load_capsule_run_git_control( )); } } - let detached = load_detached_control_sections(router, &path, &control, false).await?; - let capsules = control.materialize_capsules_with_paged_visibility(router, &detached)?; + let capsules = control.materialize_capsules_with_paged_visibility(router)?; Ok((control, capsules)) } @@ -443,13 +442,11 @@ async fn load_detached_control_sections( router: &StoreLayout, path: &object_store::path::Path, control: &CapsuleRunControl, - complete_visibility: bool, ) -> Result> { let ranges = control .capsule_locations() .iter() .flat_map(|location| location.detached_controls()) - .filter(|(_, kind, _)| complete_visibility || *kind != CapsuleSectionKind::VisibilityDelta) .collect::>(); futures_util::stream::iter(ranges.into_iter().map(|(hash, kind, range)| async move { let end = range diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 4fdbe86b8..a51f38db5 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -88,6 +88,7 @@ pub struct CapsuleRepositoryView { ref_capsule_counts: BTreeMap, capsule_run_pointers: Vec, capsule_run_sources: Vec, + catalog_sources: Vec, capsule_run_bytes: BTreeMap, capsule_run_indexes: BTreeMap>, capsule_run_member_oids: BTreeMap>>, @@ -740,8 +741,8 @@ impl CapsuleRepositoryView { /// Capture batched catalog lookup against this view's checkpoint and selected deltas. /// - /// Checkpoint pages stay cold. Publication deltas still use the view's - /// authenticated materialization; unrelated checkpoint history is never decoded. + /// Checkpoint and run pages stay cold until selected dependency queries; + /// exact logical-capsule order excludes unrelated physical run occurrences. pub fn pointer_catalog_lookup( &self, layout: &StoreLayout, @@ -755,18 +756,17 @@ impl CapsuleRepositoryView { .unwrap_or_else(|| { crab_metadata::capsule_protocol::PointerCatalogLookup::empty(maximum) }); - let mut deltas = Vec::new(); - for capsule in &self.capsules { - if let Some(delta) = capsule.pointer_catalog_delta()? { - deltas.push(delta); - } - } - deltas.extend( - self.capsule_controls - .iter() - .filter_map(|capsule| capsule.pointer_catalog_delta().cloned()), - ); - Ok(lookup.with_deltas(deltas)?) + let order = self + .capsules + .iter() + .map(|capsule| capsule.hash().to_owned()) + .chain( + self.capsule_controls + .iter() + .map(|capsule| capsule.hash().to_owned()), + ) + .collect::>(); + Ok(lookup.with_run_sources(self.catalog_sources.clone(), &order)?) } /// Materialize the complete generation-pinned external pointer catalog. @@ -794,8 +794,11 @@ impl CapsuleRepositoryView { } } for capsule in &self.capsule_controls { - if let Some(delta) = capsule.pointer_catalog_delta() { - catalog.apply(delta)?; + let layout = self.metadata_layout.as_ref().ok_or_else(|| { + ReadError::internal("catalog materialization has no captured origin") + })?; + if let Some(delta) = capsule.materialize_catalog(layout.store()).await? { + catalog.apply(&delta)?; } } Ok(catalog) @@ -4321,6 +4324,7 @@ pub fn compacted_view_from_checkpoint( ref_capsule_counts: BTreeMap::new(), capsule_run_pointers: Vec::new(), capsule_run_sources: Vec::new(), + catalog_sources: Vec::new(), capsule_run_bytes: BTreeMap::new(), capsule_run_indexes: BTreeMap::new(), capsule_run_member_oids: BTreeMap::new(), @@ -4456,6 +4460,13 @@ async fn assemble_view( .into_iter() .map(|run| (run.hash().to_owned(), run)) .collect::>(); + let catalog_sources = runs + .values() + .map(|run| run.catalog_source(router)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(); let mut capsule_run_sources = Vec::new(); let mut capsule_run_indexes = BTreeMap::new(); let mut capsule_run_member_oids = BTreeMap::new(); @@ -4628,7 +4639,7 @@ async fn assemble_view( capsules: root_capsules, capsule_controls: Vec::new(), tip_bound_transitions, - metadata_layout: None, + metadata_layout: Some(router.clone()), tip_bound_controls: Vec::new(), refs, peeled_refs, @@ -4636,6 +4647,7 @@ async fn assemble_view( ref_capsule_counts, capsule_run_pointers: pointers, capsule_run_sources, + catalog_sources, capsule_run_bytes: BTreeMap::new(), capsule_run_indexes, capsule_run_member_oids, @@ -4709,6 +4721,13 @@ async fn assemble_layered_control_view( } runs.insert(hash, (control, capsules)); } + let catalog_sources = runs + .values() + .map(|(control, _)| control.catalog_source(router)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(); let all_capsule_controls = runs .values() .flat_map(|(_, capsules)| capsules.iter().cloned()) @@ -4885,6 +4904,7 @@ async fn assemble_layered_control_view( ref_capsule_counts, capsule_run_pointers: pointers, capsule_run_sources, + catalog_sources, capsule_run_bytes, capsule_run_indexes, capsule_run_member_oids, @@ -4973,7 +4993,7 @@ async fn assemble_layered_control_view( capsules: Vec::new(), capsule_controls: root_capsules, tip_bound_transitions, - metadata_layout: matches!(read, ControlRead::Git).then(|| router.clone()), + metadata_layout: Some(router.clone()), tip_bound_controls: if matches!(read, ControlRead::Git) { all_capsule_controls } else { @@ -4985,6 +5005,7 @@ async fn assemble_layered_control_view( ref_capsule_counts, capsule_run_pointers: pointers, capsule_run_sources, + catalog_sources, capsule_run_bytes, capsule_run_indexes, capsule_run_member_oids, @@ -6548,6 +6569,142 @@ mod tests { } } + #[tokio::test] + async fn git_catalog_queries_leave_unrelated_run_history_cold() { + use crab_metadata::capsule_protocol::{ + FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + }; + for count in [100, 10_000] { + let observer = Arc::new(RecordingObserver::default()); + let store = + Store::new(Arc::new(InMemory::new())).with_storage_observer(observer.clone()); + let router = StoreLayout::new(store.clone(), "repositories/catalog-query".into()); + let initial = RootRecord::encode( + RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(), + ) + .unwrap(); + let transaction = CapsuleTransaction::new( + initial.digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let mut catalog = PointerCatalog::new(); + for index in 0..count { + let xorb = format!("{:064x}", index + 1_000_000); + let shard = format!("{:064x}", index + 3_000_000); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 99, + "e".repeat(64), + vec![XorbChunkEntry::new( + format!("{:064x}", index + 2_000_000), + 7, + )], + ), + ) + .unwrap(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + catalog + .insert_file(format!("{index:064x}"), FileCatalogEntry::new(7, shard)) + .unwrap(); + } + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + store + .put(&router.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let next = initial + .root() + .advance( + initial.digest(), + BTreeMap::from([("refs/heads/main".into(), "2".repeat(40))]), + BTreeMap::new(), + vec![pointer], + &transaction.id().unwrap(), + ) + .unwrap(); + let root = RootRecord::encode(next).unwrap(); + store + .create_strict(&router.capsule_root_path(), root.bytes().clone()) + .await + .unwrap(); + observer.observations.lock().unwrap().clear(); + let root = load_root(&router).await.unwrap(); + let view = open_git_view_from_root( + &router, + root, + CapsuleReadLimits { + max_capsule_bytes: 64 * 1024 * 1024, + max_frontier_bytes: 64 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let opening = observer + .observations + .lock() + .unwrap() + .iter() + .map(|read| read.bytes_read) + .sum::(); + assert!( + opening <= 64 * 1024, + "opening decoded catalog history: {opening}" + ); + let lookup = view.pointer_catalog_lookup(&router, 256 * 1024).unwrap(); + observer.observations.lock().unwrap().clear(); + let file = format!("{:064x}", 42); + let selected = lookup + .files(router.store(), std::slice::from_ref(&file)) + .await + .unwrap(); + assert_eq!(selected.files().get(&file), catalog.files().get(&file)); + assert!( + observer + .observations + .lock() + .unwrap() + .iter() + .map(|read| read.bytes_read) + .sum::() + <= 256 * 1024 + ); + observer.observations.lock().unwrap().clear(); + lookup.files(router.store(), &[file]).await.unwrap(); + assert!(observer.observations.lock().unwrap().is_empty()); + assert_eq!(view.pointer_catalog().await.unwrap(), catalog); + } + } + #[tokio::test] async fn metadata_views_deduplicate_shared_capsules_across_compacted_origins() { for conflicting in [false, true] { From ed2a8ad826a5f54c18b635b861fcc1e9206bf36f Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 20:51:20 -0700 Subject: [PATCH 046/127] fix(read): reject incomplete catalog captures --- .../docs/architecture/indexed-remote-reads.md | 83 ++++++++++++++ crab/src/core/error.rs | 17 ++- crates/crab-read/README.md | 20 ++-- crates/crab-read/src/capsule_protocol.rs | 103 +++++++++++++++--- crates/crab-read/src/error.rs | 3 + 5 files changed, 201 insertions(+), 25 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 683137cab..d86c5c6a1 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1605,3 +1605,86 @@ receipt is `catalog-root-failures-live-r2/receipt.json`. The first probe stopped on a driver assertion against unborn `HEAD`; the corrected probe commits and pushes a real fixture before fault injection. Neither probe changes the broad harness's failed status. + +### Reject incomplete catalog captures + +Fetch-only views retain physical pack/visibility evidence while intentionally +omitting reconciled logical delta controls. Treating their missing controls as +an empty suffix would let either catalog API return a checkpoint-only result. +The guard represents catalog capture as an optional source set: +`None` means incomplete; `Some(empty)` means a complete capture with no indexed +delta contribution. Both selected lookup and full inspection reject incomplete +captures before origin I/O with `ReadError::IncompleteCatalogView`. CLI +conversion retains this typed failure in its standard source chain. + +The regression covers unborn and populated fetch-only views, verifies no I/O +on rejection, and contrasts both with an authenticated miss from a complete +Git metadata view. All 50 read-side capsule tests pass, including this guard, +and strict read all-target Clippy passes. The CLI source-chain regression +and CLI compile check also pass. This guard is not included in the frozen run-v4 +installation build at commit `2d78b341010`. + +### Run-v4 native and Git workflow qualification + +The full-feature release binary at `2d78b341010` passes the existing managed +RustFS workflow: 74 commands and 58 checks across native add/push and Git +add/push. Each uses an actual 4 MiB file and duplicate, then verifies lazy +clone, byte-identical hydration, dehydrate/rehydrate, cold checkpoint clone, +Git connectivity and remote integrity. The input is the existing read-only +100-commit fixture; this is correctness smoke, not history-scaling evidence. + +Independent inspection confirms both stored publication runs use footer v4 +and contain `CRBRCG01` indexes with nonempty roots. Both resulting checkpoints +use format 7 with file-catalog roots. The binary SHA-256 remains +`5171cfee26d393c0121cc3fb02891312f5825b8f22b4434db8abbc7b37cd07c8`, +and all 1,218 guarded source/manifest files are unchanged. External receipt: +`run-catalog-fuse-live-r1/receipt.json`. The incomplete-capture guard above is +not part of this frozen binary. Normal installation, fixed edits against +increasing history, new 10 GiB qualification, consumer/lifecycle integration +and representative S3 latency remain outstanding. + +### Published run-reader CI coverage + +The protocol workflow at published head `2d78b341010` completed successfully: +[Git protocol qualification run 38669011348](https://github.com/crabbuild/crab/actions/runs/38669011348). +Its unit logs include the catalog-history growth regression, read-view lazy +catalog regression, shared cross-run warm budget regression and complete/paged +control identity regression. The workflow also passes native Git compatibility +for 2.30, 2.40, 2.45 and current Git, macOS/Windows protocol contracts, and the +released-shape RustFS partial-clone lifecycle. Architecture Gates at the same +head also pass. These results cover the published revision; they do not cover +the local incomplete-capture guard or replace the remaining 10 GiB, +history-scaling, consumer and lifecycle acceptance gates. The external log and +revision-bound receipt are `run-catalog-ci-protocol-unit.log` and +`run-catalog-ci-proof.json`. + +### Fixed-edit qualification exposed cache close cost + +`run-catalog-fixed-edit-r1` holds a 4 MiB file and 4 KiB edit fixed against +16 versus 1,024 unrelated 64 KiB managed files. The 16-file case passes cold +byte-identical reconstruction, strict Git integrity and remote fsck with zero +errors. The larger case fails during its history push at the 900-second +command deadline; it never reaches the fixed edit. This is a failed scaling +qualification, not evidence that the new catalog passes its acceptance gate. + +Two stack samples and offline symbolization identify the observed hot path: +`CacheCatalog::reserve_sync` drops its private database, rusqlite closes the +connection, and SQLite checkpoints the last WAL connection. The second sample +records 753 of 788 blocking-worker samples in `fcntl`; disassembly identifies +`File::sync_all` using macOS `F_FULLFSYNC`. The pinned SQLite source documents +this default close/checkpoint behavior. These observations identify cache +close cost during the failure; a bounded session/checkpoint fix still needs a +deterministic regression, security/lifetime proof and a full rerun. No sync +or payload durability policy has been relaxed. + +A fresh read after timeout confirms the 1,024-file repository's published ref +still names its initial commit (`run-catalog-timeout-inspection-r1`, eight +commands/fourteen checks). Failed logs, samples, symbols and dependency proof +remain in `run-catalog-fixed-push-diagnostic.json` and the original run report. +The separate `codex/cache-catalog-io` change retains WAL at all three catalog +writer boundaries. Its regression fails before the fix and passes afterward; +46 catalog and 128 private-filesystem/lifecycle/local-cache tests pass, as does +strict local-cache all-target Clippy. The full failed workload still needs a +rerun. The pinned SQLite 3.49.1 is also affected by the upstream WAL-reset +race; a matched Crab/Cellule dependency upgrade is required before release +qualification. Remaining consumer, lifecycle and 10 GiB gates still apply. diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index d1f6ec727..24cdac161 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -1134,7 +1134,8 @@ impl From for CrabError { | crab_read::ReadError::Runtime(_) | crab_read::ReadError::ResolutionTask(_) | crab_read::ReadError::ReadinessTask(_) - | crab_read::ReadError::Reconstruction { .. }) => Self::Read(ReadFailure(error)), + | crab_read::ReadError::Reconstruction { .. } + | crab_read::ReadError::IncompleteCatalogView) => Self::Read(ReadFailure(error)), crab_read::ReadError::UnauthorizedObject => { Self::Protocol("requested object is outside the visible generation".to_owned()) } @@ -3987,6 +3988,20 @@ mod tests { } } + #[test] + fn incomplete_catalog_capture_survives_cli_conversion() { + use std::error::Error; + let error = CrabError::from(crab_read::ReadError::IncompleteCatalogView); + assert!( + std::iter::successors(error.source(), |source| (*source).source()).any( + |source| matches!( + source.downcast_ref::(), + Some(crab_read::ReadError::IncompleteCatalogView) + ) + ) + ); + } + #[tokio::test] async fn term_resolution_task_source_survives_cli_conversion() { use std::error::Error; diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index bcc80a908..6fe09c3ad 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -37,8 +37,9 @@ Replica selection reports readiness, generations, fallbacks, and routing choices so callers can distinguish an unavailable replica from a corrupt object. -`capsule_protocol::open_git_view_from_root` captures complete refs and pointer -catalogs while keeping frontier visibility and run OID admission paged. It validates +`capsule_protocol::open_git_view_from_root` captures complete refs and catalog +lookup roots while keeping catalog bodies, frontier visibility and run OID +admission paged. It validates authenticated member counts and omits placement-hint dictionaries. CLI reads, HTTP browse/projection and S3 gateway reads use this view. Physical lookup uses the captured index; unindexed reads retain their existing pack-index path. @@ -452,12 +453,17 @@ membership pages separately from retained-object presence. Complete visibility inspection is an explicit async operation. `CapsuleRepositoryView::pointer_catalog` is also async: hydration, publication and maintenance materialize the complete catalog from the captured checkpoint when its body was omitted during opening. -Catalog indexing and bounded historical Xet lookup remain separate work. +Shared preparation and hydration still need selected-lookup integration. `CapsuleRepositoryView::pointer_catalog_lookup` captures the checkpoint's paged -catalog together with exactly the selected publication-ordered deltas. File +catalog together with authenticated run roots and exactly the selected +publication-ordered capsule occurrences. File bindings retain later-wins precedence with equal sizes; selected shard/xorb identities retain conflict checks across sources. Queries do not read a newer -root. Checkpoint catalogs stay cold during lookup construction; visible deltas -still materialize and need run-level indexed aggregation. The complete catalog -API remains for explicit maintenance and integrity inspection. +root. Checkpoint and run catalog pages stay cold during lookup construction and +share one cache and aggregate query budget. The complete catalog API remains +for explicit maintenance and integrity inspection. Fetch-only captures lack +reconciled logical delta state; both catalog APIs return +`ReadError::IncompleteCatalogView` before I/O rather than treating that state +as an authenticated miss. A complete capture with an empty suffix can still +return a legitimate miss. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index a51f38db5..edad84b9d 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -88,7 +88,9 @@ pub struct CapsuleRepositoryView { ref_capsule_counts: BTreeMap, capsule_run_pointers: Vec, capsule_run_sources: Vec, - catalog_sources: Vec, + // Fetch-only captures omit reconciled logical deltas. Distinguish that + // state from a proven empty suffix so catalog absence cannot hide data. + catalog_sources: Option>, capsule_run_bytes: BTreeMap, capsule_run_indexes: BTreeMap>, capsule_run_member_oids: BTreeMap>>, @@ -743,11 +745,16 @@ impl CapsuleRepositoryView { /// /// Checkpoint and run pages stay cold until selected dependency queries; /// exact logical-capsule order excludes unrelated physical run occurrences. + /// Fetch-only captures return `ReadError::IncompleteCatalogView`. pub fn pointer_catalog_lookup( &self, layout: &StoreLayout, maximum: u64, ) -> Result { + let sources = self + .catalog_sources + .as_ref() + .ok_or(ReadError::IncompleteCatalogView)?; let lookup = self .checkpoint .as_ref() @@ -766,11 +773,16 @@ impl CapsuleRepositoryView { .map(|capsule| capsule.hash().to_owned()), ) .collect::>(); - Ok(lookup.with_run_sources(self.catalog_sources.clone(), &order)?) + Ok(lookup.with_run_sources(sources.clone(), &order)?) } /// Materialize the complete generation-pinned external pointer catalog. + /// + /// Fetch-only captures return `ReadError::IncompleteCatalogView`. pub async fn pointer_catalog(&self) -> Result { + self.catalog_sources + .as_ref() + .ok_or(ReadError::IncompleteCatalogView)?; let mut catalog = match self.checkpoint.as_ref() { Some(checkpoint) if checkpoint.is_control_only() => { @@ -4324,7 +4336,7 @@ pub fn compacted_view_from_checkpoint( ref_capsule_counts: BTreeMap::new(), capsule_run_pointers: Vec::new(), capsule_run_sources: Vec::new(), - catalog_sources: Vec::new(), + catalog_sources: Some(Vec::new()), capsule_run_bytes: BTreeMap::new(), capsule_run_indexes: BTreeMap::new(), capsule_run_member_oids: BTreeMap::new(), @@ -4460,13 +4472,14 @@ async fn assemble_view( .into_iter() .map(|run| (run.hash().to_owned(), run)) .collect::>(); - let catalog_sources = runs - .values() - .map(|run| run.catalog_source(router)) - .collect::, _>>()? - .into_iter() - .flatten() - .collect(); + let catalog_sources = Some( + runs.values() + .map(|run| run.catalog_source(router)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(), + ); let mut capsule_run_sources = Vec::new(); let mut capsule_run_indexes = BTreeMap::new(); let mut capsule_run_member_oids = BTreeMap::new(); @@ -4721,13 +4734,18 @@ async fn assemble_layered_control_view( } runs.insert(hash, (control, capsules)); } - let catalog_sources = runs - .values() - .map(|(control, _)| control.catalog_source(router)) - .collect::, _>>()? - .into_iter() - .flatten() - .collect(); + let catalog_sources = if footer_only { + None + } else { + Some( + runs.values() + .map(|(control, _)| control.catalog_source(router)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(), + ) + }; let all_capsule_controls = runs .values() .flat_map(|(_, capsules)| capsules.iter().cloned()) @@ -6502,6 +6520,57 @@ mod tests { )); } + #[tokio::test] + async fn fetch_only_catalog_capture_is_distinct_from_an_authenticated_miss() { + for frontier in [false, true] { + let inner = Arc::new(InMemory::new()); + let observer = Arc::new(RecordingObserver::default()); + let store = Store::new(inner.clone()).with_storage_observer(observer.clone()); + let router = StoreLayout::new(store, "repositories/test".into()); + if frontier { + seed_one_capsule(inner, None).await; + } else { + let root = RootRecord::encode( + RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(), + ) + .unwrap(); + router + .store() + .create_strict(&router.capsule_root_path(), root.bytes().clone()) + .await + .unwrap(); + } + let root = load_root(&router).await.unwrap(); + let fetch = + open_view_from_root_with_layered_control(&router, root.clone(), TEST_LIMITS) + .await + .unwrap(); + observer.observations.lock().unwrap().clear(); + assert!(matches!( + fetch.pointer_catalog_lookup(&router, 1024 * 1024), + Err(ReadError::IncompleteCatalogView) + )); + assert!(matches!( + fetch.pointer_catalog().await, + Err(ReadError::IncompleteCatalogView) + )); + assert!(observer.observations.lock().unwrap().is_empty()); + let complete = open_git_view_from_root(&router, root, TEST_LIMITS) + .await + .unwrap(); + let lookup = complete + .pointer_catalog_lookup(&router, 1024 * 1024) + .unwrap(); + assert!( + lookup + .files(router.store(), &["f".repeat(64)]) + .await + .unwrap() + .is_empty() + ); + } + } + #[tokio::test] async fn metadata_view_preserves_complete_state_without_frontier_body_reads() { let inner = Arc::new(InMemory::new()); diff --git a/crates/crab-read/src/error.rs b/crates/crab-read/src/error.rs index 94c0836b2..36fc3793c 100644 --- a/crates/crab-read/src/error.rs +++ b/crates/crab-read/src/error.rs @@ -91,6 +91,9 @@ pub enum ReadError { #[error("requested object is outside the visible generation")] UnauthorizedObject, + #[error("catalog access requires a reconciled metadata view; fetch-only capture is incomplete")] + IncompleteCatalogView, + #[error("capsule-protocol read exceeds {resource} limit ({maximum} bytes)")] CapsuleReadLimit { resource: &'static str, From b27131360cfd0c7ba4d88115836e41ccb20fcc50 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 20:52:14 -0700 Subject: [PATCH 047/127] perf(cache): retain WAL across catalog operations --- crates/crab-cache/README.md | 5 +++ crates/crab-cache/REFERENCE.md | 31 +++++++++++++++ crates/crab-cache/src/catalog.rs | 20 +++++++++- crates/crab-cache/src/catalog/removal.rs | 15 +++++--- crates/crab-cache/src/catalog/tests.rs | 49 ++++++++++++++++++++---- 5 files changed, 106 insertions(+), 14 deletions(-) diff --git a/crates/crab-cache/README.md b/crates/crab-cache/README.md index 92dc624b8..1ed835340 100644 --- a/crates/crab-cache/README.md +++ b/crates/crab-cache/README.md @@ -138,3 +138,8 @@ or mapping: a newly created side file can behave differently from a reopened main file. The check protects live WAL mappings from another opener's reset. The probe uses a byte beyond SQLite's lock region and releases its locks on both success and failure; it does not modify database contents. + +Catalog connections retain WAL between operations to avoid a close-time device +flush for every small reservation or owner release. Existing sync and payload +publication guarantees remain in force; see [checkpoint cadence](REFERENCE.md#catalog-checkpoint-cadence) +for recovery, snapshot and qualification boundaries. diff --git a/crates/crab-cache/REFERENCE.md b/crates/crab-cache/REFERENCE.md index 4ebab1f51..294fcfe69 100644 --- a/crates/crab-cache/REFERENCE.md +++ b/crates/crab-cache/REFERENCE.md @@ -11,6 +11,7 @@ their outstanding qualification limits. - [Health inspection and accounting](#health-inspection-and-accounting) - [Object maintenance and recency](#object-maintenance-and-recency) - [Catalog eviction](#catalog-eviction) +- [Catalog checkpoint cadence](#catalog-checkpoint-cadence) - [Private SQLite ownership](#private-sqlite-ownership) - [Database generation binding](#database-generation-binding) - [Read-only SQLite inspection](#read-only-sqlite-inspection) @@ -303,3 +304,33 @@ closing their file handles. On Unix, a descriptor inherited by a concurrent fork must not prolong ownership until that unrelated child executes or exits. Callers still must join their own cache workers before dropping the guard; explicit unlock does not make a detached writer safe. + +## Catalog checkpoint cadence + +Catalog writers retain their generation-bound WAL when short connections +close. Reservation creation, owner release and payload removal use the same +close policy; closing one SQL operation no longer requests a full checkpoint +of the catalog. SQLite's existing automatic checkpoint policy remains enabled +(the pinned dependency defaults to 1,000 pages), and synchronization settings, +payload flushes, private filesystem checks and accounting transactions remain +unchanged. A long active reader can still defer an automatic checkpoint under +SQLite's existing WAL rules. + +Committed catalog state may remain in WAL after all catalog handles close. +Inspection reads that WAL through the existing read-only private VFS. Copying +the main database alone is not a coherent snapshot; fixture builders and repair +operations must use a complete SQLite snapshot or checkpoint under its locks. +Generation replacement still rejects stale owners instead of adopting recovery +files from a different main/owner identity. This changes checkpoint cadence, +not the schema or public cache configuration. + +The pre-fix regression fails for repeated reservations and immediate release. +Post-fix qualification passes 46 catalog tests and 128 private-filesystem, +lifecycle and local-cache tests, plus strict local-cache all-target Clippy. +Native-platform coverage and the failed 1,024-file push rerun remain required. + +The pinned SQLite 3.49.1 also falls in the affected range of the upstream +[WAL-reset race](https://www.sqlite.org/wal.html#walreset). A matched Crab and +Cellule dependency upgrade to a fixed SQLite release is a prerequisite for +release qualification; these focused tests do not establish immunity to that +race. No dependency or lockfile change is included in this optimization. diff --git a/crates/crab-cache/src/catalog.rs b/crates/crab-cache/src/catalog.rs index 12e31342a..3575b0ab5 100644 --- a/crates/crab-cache/src/catalog.rs +++ b/crates/crab-cache/src/catalog.rs @@ -719,6 +719,7 @@ fn open_catalog(root: &PinnedRoot, path: &Path) -> Result { } fn configure_catalog(mut connection: Database, path: &Path) -> Result { + retain_catalog_wal(&connection).map_err(|source| index_error(path, source))?; connection .pragma_update(None, "synchronous", "NORMAL") .map_err(|source| index_error(path, source))?; @@ -744,6 +745,18 @@ fn configure_catalog(mut connection: Database, path: &Path) -> Result } } +fn retain_catalog_wal(connection: &Database) -> rusqlite::Result<()> { + // Short catalog connections must not checkpoint and flush the device on + // every reservation or owner release. SQLite's automatic checkpoints and + // existing sync policy still govern the retained, generation-bound WAL. + connection + .set_db_config( + rusqlite::config::DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + true, + ) + .map(|_| ()) +} + fn catalog_schema_ready(connection: &Database, path: &Path) -> Result { let journal_mode = connection .pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0)) @@ -935,7 +948,12 @@ fn remove_owner_row( ) else { return; }; - let _ = connection.execute(sql, params![first, second]); + if let Err(error) = retain_catalog_wal(&connection) + .and_then(|()| connection.execute(sql, params![first, second]).map(|_| ())) + { + tracing::warn!(family = "catalog", operation = "release-owner", %error, + "cache owner cleanup failed"); + } } fn remove_stale_owners(transaction: &Transaction<'_>, catalog_path: &Path) -> Result<()> { diff --git a/crates/crab-cache/src/catalog/removal.rs b/crates/crab-cache/src/catalog/removal.rs index 83fe91b63..8b91df63d 100644 --- a/crates/crab-cache/src/catalog/removal.rs +++ b/crates/crab-cache/src/catalog/removal.rs @@ -165,11 +165,16 @@ impl PayloadRemoval { // A standalone range directory need not have a private catalog parent. // Never inspect that ambient parent or open SQLite during a dry run. let connection = if let Some(root) = root.filter(|_| !dry_run) { - match root.open_database( - Path::new(CATALOG_FILE), - DatabaseMode::ReadWrite, - std::time::Duration::ZERO, - ) { + match root + .open_database( + Path::new(CATALOG_FILE), + DatabaseMode::ReadWrite, + std::time::Duration::ZERO, + ) + .and_then(|connection| { + retain_catalog_wal(&connection).map_err(|source| index_error(&path, source))?; + Ok(connection) + }) { Ok(connection) => Some(connection), Err(CacheError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => None, Err(error) if busy(&error) => return Err(error), diff --git a/crates/crab-cache/src/catalog/tests.rs b/crates/crab-cache/src/catalog/tests.rs index e89821927..eecdd2ce2 100644 --- a/crates/crab-cache/src/catalog/tests.rs +++ b/crates/crab-cache/src/catalog/tests.rs @@ -5,6 +5,15 @@ fn open_catalog(root: &Path) -> Result { super::open_catalog(&PinnedRoot::create(root)?, &root.join(CATALOG_FILE)) } +fn checkpoint_catalog_snapshot(root: &Path) { + // These fixtures copy or replace only the main file. Let SQLite fold its + // WAL and remove recovery side files before constructing that snapshot. + open_catalog(root) + .unwrap() + .execute_batch("PRAGMA journal_mode = DELETE;") + .unwrap(); +} + async fn payload(root: &Path, byte: u8, size: usize) -> PathBuf { let data = vec![byte; size]; let hash = crab_xet::hash::compute_data_hash(&data).hex(); @@ -21,6 +30,28 @@ async fn payload(root: &Path, byte: u8, size: usize) -> PathBuf { path } +#[tokio::test] +async fn reservations_reuse_uncheckpointed_wal_between_operations() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("cache"); + let catalog = CacheCatalog::new(root.clone(), 1024 * 1024); + for index in 0..8 { + drop( + catalog + .reserve(&root.join(format!("incoming-{index}")), 7) + .await + .unwrap() + .unwrap(), + ); + } + // A checkpoint on every final connection close makes small cache fills + // issue a device flush per reservation, even under WAL/NORMAL policy. + assert!( + std::fs::metadata(root.join(format!("{CATALOG_FILE}-wal"))) + .is_ok_and(|metadata| metadata.len() > 32) + ); +} + #[tokio::test] async fn unlimited_admission_and_maintenance_retain_payloads() { let tmp = tempfile::tempdir().unwrap(); @@ -215,14 +246,12 @@ fn read_only_stats_preserves_quiet_and_uncheckpointed_catalogs() { "INSERT INTO cache_entries VALUES ('entry', 'temporary', 'key', 17, 0, 0)", ) .unwrap(); - if retained_wal { - connection - .set_db_config( - rusqlite::config::DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, - true, - ) - .unwrap(); - } + connection + .set_db_config( + rusqlite::config::DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, + retained_wal, + ) + .unwrap(); drop(connection); assert_eq!( root.join(format!("{CATALOG_FILE}-wal")).exists(), @@ -507,6 +536,7 @@ async fn owner_cleanup_releases_only_the_original_root_after_replacement() { let path = root.join("incoming"); let lease = catalog.lease(&path).await.unwrap(); let reservation = catalog.reserve(&path, 7).await.unwrap().unwrap(); + checkpoint_catalog_snapshot(&root); let moved = tmp.path().join("moved"); std::fs::rename(&root, &moved).unwrap(); ensure_private_cache_directory(&root).unwrap(); @@ -584,6 +614,7 @@ fn reservation_keeps_database_generation_leased_after_connection_close() { .reserve_sync(&root.join("incoming"), 7) .unwrap() .unwrap(); + checkpoint_catalog_snapshot(&root); let main = root.join(CATALOG_FILE); let retired = root.join("retired.sqlite"); std::fs::rename(&main, &retired).unwrap(); @@ -654,6 +685,7 @@ fn windows_pid_liveness_distinguishes_current_and_missing_processes() { #[cfg(unix)] fn replace_catalog_generation(root: &Path, replace_main: bool) { + checkpoint_catalog_snapshot(root); let main = root.join(CATALOG_FILE); let retired = root.join("retired.sqlite"); if replace_main { @@ -1137,6 +1169,7 @@ async fn dropping_owners_never_recreates_or_follows_a_replaced_catalog() { let catalog = CacheCatalog::new(root.clone(), 1024 * 1024); let lease = catalog.lease(&path).await.unwrap(); let reservation = catalog.reserve(&path, 7).await.unwrap().unwrap(); + checkpoint_catalog_snapshot(&root); let old = tmp.path().join("retired.sqlite"); std::fs::rename(root.join(CATALOG_FILE), &old).unwrap(); if link { From b503652764fbff684b070101f15dd6425467e96d Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 20:57:25 -0700 Subject: [PATCH 048/127] test(cache): corrupt checkpointed catalog fixtures --- crates/crab-cache/REFERENCE.md | 9 ++++++--- crates/crab-cache/src/health/tests.rs | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/crab-cache/REFERENCE.md b/crates/crab-cache/REFERENCE.md index 294fcfe69..fc0e4edcd 100644 --- a/crates/crab-cache/REFERENCE.md +++ b/crates/crab-cache/REFERENCE.md @@ -325,9 +325,12 @@ files from a different main/owner identity. This changes checkpoint cadence, not the schema or public cache configuration. The pre-fix regression fails for repeated reservations and immediate release. -Post-fix qualification passes 46 catalog tests and 128 private-filesystem, -lifecycle and local-cache tests, plus strict local-cache all-target Clippy. -Native-platform coverage and the failed 1,024-file push rerun remain required. +Post-fix qualification passes all 309 library tests and three credential +integration tests with local-cache, Xet chunk cache and remote-client enabled. +The corrupt-main fixture checkpoints before damage so committed WAL cannot +supply its supposedly corrupted header; its original assertions remain. +Strict combined-feature all-target Clippy also passes. Native-platform coverage +and the failed 1,024-file push rerun remain required. The pinned SQLite 3.49.1 also falls in the affected range of the upstream [WAL-reset race](https://www.sqlite.org/wal.html#walreset). A matched Crab and diff --git a/crates/crab-cache/src/health/tests.rs b/crates/crab-cache/src/health/tests.rs index b4fc80666..0b2325561 100644 --- a/crates/crab-cache/src/health/tests.rs +++ b/crates/crab-cache/src/health/tests.rs @@ -422,6 +422,18 @@ async fn corrupt_and_orphaned_catalogs_are_unavailable_not_empty_or_repaired() { .maintain() .await .unwrap(); + // Corrupt the authoritative image, not a main file whose header + // can still be supplied by committed WAL pages during inspection. + PinnedRoot::open(&root) + .unwrap() + .open_database( + Path::new(".catalog.sqlite"), + crate::private_fs::DatabaseMode::ReadWrite, + std::time::Duration::ZERO, + ) + .unwrap() + .execute_batch("PRAGMA journal_mode = DELETE;") + .unwrap(); } // Overwrite the existing main inode so its real generation owner is // valid and the probe reaches SQLite's corrupt-header check. From 9c3e277a144a3f9f4a86b829b7b428d784fb5c48 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 21:13:56 -0700 Subject: [PATCH 049/127] fix(sqlite): align Crab with fixed WAL runtime Select the minimal Cellule backport and bundled SQLite 3.51.3. Preserve checked unsigned bindings and propagate inspection hook installation errors before SQL. Keep the unrelated Cellule 0.2 storage migration out of this dependency correction. --- Cargo.lock | 49 ++++++++++++++++++++-------- Cargo.toml | 12 +++---- crates/crab-cache/REFERENCE.md | 16 ++++++--- crates/crab-cache/src/shard_hints.rs | 10 +++--- 4 files changed, 59 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad69dadea..aa7fddfa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1527,7 +1527,7 @@ dependencies = [ [[package]] name = "cellule-app" version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule?rev=6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6#6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6" +source = "git+https://github.com/crabbuild/cellule?rev=25809dc7a3acc37411776d79f526512088e466c3#25809dc7a3acc37411776d79f526512088e466c3" dependencies = [ "blake3", "cellule-runtime", @@ -1536,7 +1536,7 @@ dependencies = [ [[package]] name = "cellule-host" version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule?rev=6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6#6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6" +source = "git+https://github.com/crabbuild/cellule?rev=25809dc7a3acc37411776d79f526512088e466c3#25809dc7a3acc37411776d79f526512088e466c3" dependencies = [ "cellule-app", "cellule-runtime", @@ -1550,7 +1550,7 @@ dependencies = [ [[package]] name = "cellule-ltx" version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule?rev=6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6#6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6" +source = "git+https://github.com/crabbuild/cellule?rev=25809dc7a3acc37411776d79f526512088e466c3#25809dc7a3acc37411776d79f526512088e466c3" dependencies = [ "async-trait", "blake3", @@ -1572,7 +1572,7 @@ dependencies = [ [[package]] name = "cellule-runtime" version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule?rev=6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6#6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6" +source = "git+https://github.com/crabbuild/cellule?rev=25809dc7a3acc37411776d79f526512088e466c3#25809dc7a3acc37411776d79f526512088e466c3" dependencies = [ "async-trait", "blake3", @@ -1600,7 +1600,7 @@ dependencies = [ [[package]] name = "cellule-store" version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule?rev=6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6#6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6" +source = "git+https://github.com/crabbuild/cellule?rev=25809dc7a3acc37411776d79f526512088e466c3#25809dc7a3acc37411776d79f526512088e466c3" dependencies = [ "async-trait", "blake3", @@ -1623,7 +1623,7 @@ dependencies = [ [[package]] name = "cellule-types" version = "0.1.0" -source = "git+https://github.com/crabbuild/cellule?rev=6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6#6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6" +source = "git+https://github.com/crabbuild/cellule?rev=25809dc7a3acc37411776d79f526512088e466c3#25809dc7a3acc37411776d79f526512088e466c3" dependencies = [ "schemars 0.8.22", "serde", @@ -5111,11 +5111,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", ] [[package]] @@ -6079,9 +6079,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.32.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb8270bb4060bd76c6e96f20c52d80620f1d82a3470885694e41e0f81ef6fe7" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -8157,11 +8157,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + [[package]] name = "rusqlite" -version = "0.34.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags 2.11.1", "fallible-iterator", @@ -8169,6 +8179,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -9040,6 +9051,18 @@ dependencies = [ "der", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index d6fd34b4c..dc14e7c7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,11 +34,11 @@ repository = "https://github.com/crabbuild/crab" rust-version = "1.97" [workspace.dependencies] -cellule-runtime = { git = "https://github.com/crabbuild/cellule", rev = "6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6", default-features = false } -cellule-app = { git = "https://github.com/crabbuild/cellule", rev = "6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6", default-features = false } -cellule-host = { git = "https://github.com/crabbuild/cellule", rev = "6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6", default-features = false } -cellule-ltx = { git = "https://github.com/crabbuild/cellule", rev = "6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6", default-features = false } -cellule-store = { git = "https://github.com/crabbuild/cellule", rev = "6ac8a03ff784e2cd6ac001d75c84e3890c0ba4c6", default-features = false } +cellule-runtime = { git = "https://github.com/crabbuild/cellule", rev = "25809dc7a3acc37411776d79f526512088e466c3", default-features = false } +cellule-app = { git = "https://github.com/crabbuild/cellule", rev = "25809dc7a3acc37411776d79f526512088e466c3", default-features = false } +cellule-host = { git = "https://github.com/crabbuild/cellule", rev = "25809dc7a3acc37411776d79f526512088e466c3", default-features = false } +cellule-ltx = { git = "https://github.com/crabbuild/cellule", rev = "25809dc7a3acc37411776d79f526512088e466c3", default-features = false } +cellule-store = { git = "https://github.com/crabbuild/cellule", rev = "25809dc7a3acc37411776d79f526512088e466c3", default-features = false } crab-auth = { version = "=0.1.0", path = "crates/crab-auth", default-features = false } crab-auth-server = { version = "=1.2.4", path = "crates/crab-auth-server", default-features = false } crab-auth-store = { version = "=0.1.0", path = "crates/crab-auth-store", default-features = false } @@ -136,7 +136,7 @@ globset = "0.4" object_store = { version = "0.14.1", default-features = false } prost = "0.13" reqwest = { version = "0.12", default-features = false } -rusqlite = { version = "0.34", features = ["bundled"] } +rusqlite = { version = "0.39", features = ["bundled", "fallible_uint"] } schemars = "0.8" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/crab-cache/REFERENCE.md b/crates/crab-cache/REFERENCE.md index fc0e4edcd..d2a35f641 100644 --- a/crates/crab-cache/REFERENCE.md +++ b/crates/crab-cache/REFERENCE.md @@ -332,8 +332,14 @@ supply its supposedly corrupted header; its original assertions remain. Strict combined-feature all-target Clippy also passes. Native-platform coverage and the failed 1,024-file push rerun remain required. -The pinned SQLite 3.49.1 also falls in the affected range of the upstream -[WAL-reset race](https://www.sqlite.org/wal.html#walreset). A matched Crab and -Cellule dependency upgrade to a fixed SQLite release is a prerequisite for -release qualification; these focused tests do not establish immunity to that -race. No dependency or lockfile change is included in this optimization. +The workspace selects SQLite 3.51.3 through rusqlite 0.39 and the matched +[Cellule backport](https://github.com/crabbuild/cellule/pull/78). The earlier +SQLite 3.49.1 was affected by the upstream +[WAL-reset race](https://www.sqlite.org/wal.html#walreset). This dependency +upgrade preserves unsigned conversion checks and rejects progress-hook +installation failures before shard-hint inspection. The checkpoint policy +and dependency change are separate reviewable commits. Private-VFS, cache, +affected-consumer and native workflow qualification must run against this +linked SQLite version before release. The combined cache suite has been +repeated on this dependency: 309 library tests and three credential tests +pass; native workflow and affected-consumer gates remain open. diff --git a/crates/crab-cache/src/shard_hints.rs b/crates/crab-cache/src/shard_hints.rs index 5f34852dd..8553211b0 100644 --- a/crates/crab-cache/src/shard_hints.rs +++ b/crates/crab-cache/src/shard_hints.rs @@ -237,10 +237,12 @@ fn inspect_database_at_with_limits( let interrupt_cancel = cancel.clone(); // SQLite calls this between VM operations, not inside blocking VFS I/O. // Keep lock admission bounded separately; this is not a hard syscall deadline. - database.progress_handler( - progress_ops, - Some(move || interrupt_cancel.is_cancelled() || Instant::now() >= deadline), - ); + database + .progress_handler( + progress_ops, + Some(move || interrupt_cancel.is_cancelled() || Instant::now() >= deadline), + ) + .map_err(|source| inspection_error(&path, source, cancel, deadline, timeout))?; let transaction = database .transaction_with_behavior(TransactionBehavior::Deferred) .map_err(|source| inspection_error(&path, source, cancel, deadline, timeout))?; From c92318d5c2b86145ffd2a81d357c27449d728b12 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 21:42:14 -0700 Subject: [PATCH 050/127] feat: combine indexed lookup and shared staging Apply canonical ingestion, exact staging retirement, recovery and force-publication fixes to the indexed-catalog and fixed-SQLite branch. Keep its captured catalog formats and lookup authority unchanged. Strengthen multi-edit qualification with one shared exact-retirement oracle. --- .../git-protocol-v2-partial-clone.yml | 14 + .../docs/architecture/indexed-remote-reads.md | 508 +++++ .../e2e/run_add_commit_push_rustfs_smoke.py | 93 +- .../e2e/run_large_file_edits_rustfs.py | 34 +- ..._protocol_v2_partial_clone_rustfs_smoke.py | 57 +- crab/src/cmd/add.rs | 351 ++-- crab/src/cmd/staging.rs | 13 + crab/src/git/capsule_push.rs | 393 +++- crab/src/git/clean.rs | 501 ++++- crab/src/git/filter_process.rs | 140 ++ crab/src/git/index.rs | 664 +++++++ crab/src/git/push.rs | 87 +- crab/src/git/push_staging.rs | 35 +- crab/src/git/xet_publication.rs | 220 ++- crates/crab-auth/README.md | 5 + crates/crab-auth/src/protected_push.rs | 44 + .../src/cosmosdb_coordinator.rs | 4 - .../src/dynamodb_coordinator.rs | 4 - .../src/spanner_coordinator.rs | 4 - .../src/write_coordinator.rs | 12 - crates/crab-staging/AGENTS.md | 21 + crates/crab-staging/README.md | 74 + crates/crab-staging/src/add_push_plan.rs | 8 - crates/crab-staging/src/index.rs | 1654 +++++++++++------ crates/crab-staging/src/lib.rs | 807 ++++++-- crates/crab-staging/src/push_snapshot.rs | 70 + crates/crab-staging/src/recovery.rs | 66 +- crates/crab-staging/src/stream.rs | 308 ++- .../tests/publication_history_recovery.rs | 128 ++ .../crab-staging/tests/push_snapshot_guard.rs | 174 ++ .../tests/retained_recipe_visibility.rs | 90 + .../unit/prop_compaction_preserves_reads.rs | 20 +- 32 files changed, 5581 insertions(+), 1022 deletions(-) create mode 100644 crates/crab-staging/src/push_snapshot.rs create mode 100644 crates/crab-staging/tests/publication_history_recovery.rs create mode 100644 crates/crab-staging/tests/push_snapshot_guard.rs create mode 100644 crates/crab-staging/tests/retained_recipe_visibility.rs diff --git a/.github/workflows/git-protocol-v2-partial-clone.yml b/.github/workflows/git-protocol-v2-partial-clone.yml index a829566aa..ad027b55a 100644 --- a/.github/workflows/git-protocol-v2-partial-clone.yml +++ b/.github/workflows/git-protocol-v2-partial-clone.yml @@ -45,6 +45,8 @@ on: - "crab/tests/**" - "crates/crab-git/**" - "crates/crab-auth-server/**" + - "crates/crab-auth/**" + - "crates/crab-staging/**" - "crates/crab-metadata/**" - "crates/crab-read/**" - "crates/crab-remote-git/**" @@ -99,6 +101,8 @@ on: - "crab/tests/**" - "crates/crab-git/**" - "crates/crab-auth-server/**" + - "crates/crab-auth/**" + - "crates/crab-staging/**" - "crates/crab-metadata/**" - "crates/crab-read/**" - "crates/crab-remote-git/**" @@ -176,6 +180,11 @@ jobs: cargo test -p crab --bin crab pull_machine_errors_keep_the_selected_mode_and_schema --locked --features gix-transport cargo test -p crab --lib git::push_native::tests --locked --features gix-transport cargo test -p crab --lib git::push_staging::tests --locked --features gix-transport + cargo test -p crab-staging --locked --test push_snapshot_guard --test publication_history_recovery + cargo test -p crab-auth --lib protected_push --locked --features crab-auth-client + cargo test -p crab --lib git::index::tests --locked --features gix-transport + cargo test -p crab --lib publication_recovery --locked --features gix-transport + cargo test -p crab --lib git::capsule_push::tests --locked --features gix-transport cargo test -p crab --lib --locked --features gix-transport -- \ git::push::tests::lookup_staging \ git::push::tests::under_lock_refresh \ @@ -268,6 +277,11 @@ jobs: cargo test -p crab --lib cmd::lfs::install::tests --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport cargo test -p crab --lib git::push_native::tests --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport cargo test -p crab --lib git::push_staging::tests --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport + cargo test -p crab-staging --locked --test push_snapshot_guard --test publication_history_recovery + cargo test -p crab-auth --lib protected_push --locked --features crab-auth-client + cargo test -p crab --lib git::index::tests --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport + cargo test -p crab --lib publication_recovery --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport + cargo test -p crab --lib capsule_push_retires_staging_only_after_confirmed_ref_publication --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport cargo test -p crab --lib git::remote_helper::tests::list_session_does_not_open_a_damaged_staging_index --locked --no-default-features --features simd-accel,tier,watch,nfs,gix-pathmatch,gix-transport - name: Run LFS pre-push contracts diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index d86c5c6a1..1b57dd83b 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -1688,3 +1688,511 @@ strict local-cache all-target Clippy. The full failed workload still needs a rerun. The pinned SQLite 3.49.1 is also affected by the upstream WAL-reset race; a matched Crab/Cellule dependency upgrade is required before release qualification. Remaining consumer, lifecycle and 10 GiB gates still apply. + + +## Prepared reuse retains payloads before recipe sealing + +The unified-staging inventory exposed a reproducible ownership gap in +`Index::claim_prepared_chunks`. A direct-stream preparation could select an +existing prepared Xorb without retaining it. Rolling back the original recipe +then removed the canonical placement and physical body before the incoming +recipe sealed. The negative control returned `ChunkNotFound` from a real staged +payload read; its receipt is `prepared-reuse-retention-negative.log`. + +Selection now inserts the existing `preparation_payloads` pin in the same +SQLite transaction, once per distinct Xorb in the bounded claim batch. Existing +recipe sealing, finalization, abort, replacement and retirement paths already +respect this ownership table. No schema, public API, chunk precedence, format, +dependency version or configuration changes. Production growth is thirteen +lines at the owner boundary; tests cover actual payload survival and release. + +The regression streams real bytes through CDC and the prepared-Xorb writer, +selects them for another preparation, and rolls back their original recipe. +Both abort and seal/publish/release branches verify byte-identical reads while +owned and absence after final retirement. A separate transaction regression +injects an insertion failure and a later size conflict; neither leaves pins, +and the insertion error retains its typed SQLite cause. The pinned rusqlite +0.34.0 transaction implementation defaults to rollback on drop. + +Proof: 35 focused prepared-payload cases and 29 streaming lifecycle cases pass; +the existing manual 128-MiB RSS case remains ignored. Selected formatting and +all-targets staging Clippy pass with the repository CI exception for unfulfilled +lint expectations. The unrestricted Clippy attempt fails only on two unchanged +`double_must_use` expectations in `add_push_plan.rs`; its failed receipt remains +preserved. No lint annotations or gate inventories are changed in this slice. + +This slice is a prerequisite, not completion of unified staging. Segment-backed +reuse needs separate retention, lookup and compaction proof: claims consult +`chunk_payloads`, while ordinary readers still consult committed/pending +occurrence rows. Git-clean retains its separate streaming implementation. +Unifying ingestion requires preserving every occurrence, native rollback and +clean flush-before-pointer ordering, followed by the alternating 10-GiB +multi-edit/push/read qualification. Representative S3 qualification and the +remaining indexed history/catalog/lifecycle/blame gates remain open. + +## Segment reuse has independent payload ownership + +The sibling audit reproduced `ChunkNotFound` after a direct-stream recording +selected segment-backed chunks and their original recipe rolled back. It also +found stale canonical locators after compaction: the previous compactor moved +file occurrence rows while leaving `chunk_payloads` at the original position. +The negative receipt is `segment-reuse-negative.log`. + +Segment point/batch reads, deduplication and registration now share canonical +payload inventory. Cleanup retains inventory while any raw occurrence, recording +term or sealed recipe owns the chunk. Recordings therefore survive original-file +retirement before sealing, without copying segment bytes or fabricating file +occurrences. Batch rollback releases its recording retention. Sealed recipes keep +all ordered occurrences, including repeats, independently of physical deduplication. + +Compaction counts and copies each retained physical payload once, moves inventory +and committed/pending locators in one transaction, and retires the old segment +registry entry before unlink. Output allocation is provisional within that +transaction; failures cannot publish a second current segment. Durable output +installation precedes locator transfer. Exclusive-open recovery removes orphan +compaction bodies only when neither registry nor canonical inventory references +them; a referenced unregistered body fails as corruption and remains intact. +Current-segment recovery includes retained recipe payload boundaries. Segment +handles are captured with locators before releasing the owner lock, so compaction +and reader-pool eviction cannot invalidate captured reads. No synchronous guard +crosses an await. Abandoned cleanup preserves writer-to-index lock ordering. + +Release inspection confirms staging layout v1 and the old compactor in v1.2.4. +Layout v2 repairs stale inventory from the committed/pending locators released +readers used, with committed precedence, then updates admission atomically. +Both writable and shared handles can perform this metadata-only upgrade under +their existing SQLite serialization contract. Old binaries reject v2 admission. +Tables, index inventories, payload encoding and public Rust APIs are unchanged. +Upgrade failures retain v1 admission and old locators; missing retained segment +dependencies fail explicitly. Normal v2 opening does not repeat the migration +scan. Migration cannot reconstruct bytes already deleted by an older binary. + +The released corrupt-prepared-plan repair contract still uses surviving segment +occurrences. Coalesced planning preserves that source preference with batched hash +lookups, while prepared-only source groups stay bounded to one Xorb. Canonical +inventory retention is not confused with the presence of those old occurrence +rows. The stricter recipe-page occurrence proof used by product plan repair is +unchanged; it is not a new permissive fallback or a claim of unified ingestion. + +Real file cases cover recording versus sealed reuse, current versus compacted +segments, original rollback, ordered repeated chunks, captured reads after pool +eviction/unlink, close/reopen and final release. SQLite cases cover two recording +owners, incomplete transfer rollback, writable/shared released-shape upgrade and +early/late upgrade failure with typed causes. A real failed-compaction case checks +one current segment, byte-identical recovery and orphan cleanup. The existing +compaction property fixture follows the changed private transaction API; its +read assertions are preserved. Failed receipts remain recorded. + +Proof: the final staging run passes 233 cases with the existing manual RSS case +ignored. Selected formatting and all-targets Clippy pass with the repository's +existing unfulfilled-expectation exception. Seven product push-plan cases pass, +including corrupt-body segment repair, stale remote placements, global dedup and +prepared reuse without rereading. The minimal CLI consumer profile emits its +existing disabled-feature warnings; this is not a clean CLI-wide lint claim. +Production code before test modules shrinks by 146 lines across the three changed +Rust modules. Public dependency manifests, lockfiles and gate inventories are +unchanged. Broader CI and real-provider performance qualification remain required. + +Unified byte ingestion, native/Git-clean crossover publication policy and the +10-GiB alternating multi-edit/push/read grid remain required. Representative S3 +qualification, pending pack-layout gate approval and the remaining indexed +history/catalog/lifecycle/blame gates also remain open. + +### Shared byte-ingestion foundation + +Native descriptor staging now delegates to an `AsyncRead + Unpin` ingestion +engine. `stage_reader_streaming` exposes that same engine for non-seeking byte +sources; it does not open or stat its logical lease path. Native wrappers retain +descriptor/path identity validation and changed-file detection. CDC, hashing, +ordered recipes, preparation ownership and failure cleanup stay in one engine. +Pending segment authority flushes before the returned version lease is recorded. +The result retains an open batch; callers still own publication or rollback. + +A bounded pipe test covers both segment and prepared-Xorb modes, closes no +staging handle before recovery, and verifies reconstructed bytes, file identity +and absent source metadata. Cancellation of a pending read previously timed out; +the retained negative receipt is `unified-stream-cancel-negative.log`. A token +branch now wakes that read and runs attempt cleanup. Tokio 1.53.1 documents +`read_buf` as cancellation-safe when another `select!` branch wins. The focused +stream run passes 31 cases with the existing manual RSS case ignored. + +This is a draft foundation, not completed native/Git-clean unification. Packet +transport wiring, exact prepared-recipe replay, deferred Git-index publication +and crossover failure/10-GiB qualification remain required. The separate +10-GiB run uses the preceding segment-lifecycle binary; it is not evidence for +this extraction. Per-file flush cost and short-read performance remain unmeasured. + +### Git-index publication recovery serialization + +The unified-staging audit reproduced recovery returning success while another +writer still held Git's index lock. Recovery previously inspected the prior +index without serialization and could roll back a still-active publication. +The negative receipt is `index-recovery-lock-negative.log`; its source patch is +retained separately. Git 2.50.1's [add implementation](https://github.com/git/git/blob/v2.50.1/builtin/add.c) +holds that lock during cleaning, then installs the index after conversion. +Filter success therefore cannot attest to index publication. + +Publication recovery now belongs to `crab/src/git/index.rs`, shared by native +add and push. It uses gix-lock 23.0.1's non-writing Marker to acquire the exact +index lock, reads the index and fresh intent list under that lock, and keeps the +marker until staging publication/rollback finishes. Lock acquisition fails +immediately to avoid waiting in the inverse Git-index/staging order used by +Git and its filter child. The typed lock error remains an I/O source. Drop +releases only the marker acquired by recovery; no index replacement is written. +Existing exact-new, exact-prior and ambiguous-state decisions are preserved. +No schema, path encoding, intent format or dependency changes are required. + +The regressions cover both exclusive and shared staging handles. A real Git +child pauses inside a controlled clean filter: recovery must retain the recipe +and payload until Git finishes installing its pointer, then publish on retry. +The test releases its filter and reaps Git on exit. Two publication-focused +runs each pass 26 tests, including both handle variants and the real Git child. +The production Clippy contract from `.github/workflows/rust.yml` passes with +the minimal consumer profile and existing warnings. The broader all-test +invocation fails on existing test lint debt; new test `expect` calls were +corrected without changing assertions or adding suppressions. Both successful +and failed receipts remain retained. This fixes the recovery boundary; routing +Crab's clean stream through shared ingestion and deferring its recipe +publication remain required. + +### Unified production byte ingestion and external publication handoffs + +Production Git-clean now routes ordinary bytes and changed prepared streams through +`crab-staging::stream::stage_reader_streaming`, the same engine native add uses. +Packet adaptation consumes exactly one filter request and preserves its terminal +flush. It restores typed framing errors after ingestion cleanup. The synchronous +filter runs on its existing blocking thread; this does not promise interruption +of a stalled underlying blocking stdin read. Short reads coalesce before CDC to +avoid retaining a full read allocation per small packet. Exact prepared inputs +are verified without CDC or rewriting payloads. Changed inputs replay bounded +verified chunks from a pinned recipe, then continue with the supplied stream. +Late edits still pay prefix reread/CDC cost; no speedup is claimed without measurement. + +Git's filter success precedes its index installation. A new durable handoff owner +therefore distinguishes externally delegated leases from active native batches +and published path heads. Git-clean and native `--defer` delegate only complete, +verified recipes with durable payloads. Native atomic index publication retains +its existing intent and rollback protocol. Pointer discovery precedes batched +published/delegated recipe selection in push. Snapshots capture exact eligible +leases and retire only captured owners after remote publication, preserving active +native owners and handoffs created after capture. Both direct and intent publication +supersede only published leases; another caller's open lease remains its property. +The old test requiring deletion of an open prepared owner was removed because its +behavior violated this invariant; the new regression covers both publication paths. + +Layout v3 adds the handoff table with a cascading composite lease foreign key. +Released v1 schema creation, locator repair and admission update commit together; +upgrade failure preserves the released shape. The unreleased v2 prototype is +rejected rather than receiving a compatibility reader. The released v1.2.4 public +custom `ChunkStager` API retains its adapter until a breaking library release; +production local ingestion uses the canonical engine. No dependency or lockfile +changes are required. + +This remains draft work. New-path 10-GiB multi-edit/push/reconstruction, representative +S3 performance, Git termination and abandoned-handoff reconciliation/reclamation, +full failure injection and broad CI remain required. The concurrent 10-GiB run is +pinned to the older segment-lifecycle binary and cannot qualify these changes. + +The final expanded local clean consumer run passes 57 tests. The six-version +crossover exercises both native/Git orders, an unchanged transition, middle/tail +edits, append and truncation. It compares physical prepared-body names and sizes +on the unchanged transition and bounds retained physical growth for small edits, +besides reconstructing every ordered recipe exactly. Staging passes 238 tests +with one existing manual RSS test ignored. Packet/push/publication and lint +receipts remain pending; these local cases do not establish the 10-GiB gate. + +CI on `ffcc26edd0d` fails the existing pack-layout architecture assertion in +`crab/src/read/mod.rs`, requiring the retired direct installer symbol. The failing +job is `116161095860` in run `38654549030`. The previously proposed gate update +remains pending approval; no baseline or gate inventory has been changed. + +The follow-up real-staging malformed-input case passes: a complete changed payload +without Git's terminal flush returns the original typed EOF error, retains the +prior prepared recipe and leaves exactly its original batch/lease ownership. +Both the temporary prepared-prefix pin and reader attempt are released. This +adds failure proof through the production adapter, rather than only a mocked +stager or packet reader. Two packet tests, one delegated push-admission test, +seven prepared-plan tests and 26 publication tests also pass. Staging all-targets +Clippy passes with the existing unfulfilled-expectation exception. The minimal +product consumer passes the CI correctness/suspicious lint profile with existing +warning debt; strict full-CLI lint cleanliness is not claimed. No production Rust +changes accompany this follow-up test. + +### Delegated index acceptance during explicit staging maintenance + +A follow-up slice freezes delegated batch membership, rejects unfinished recording +or preparation ownership, and requires every completed path to be handed off in +one transaction. It keeps released mutable-open-batch behavior before delegation. +The maintenance listing rejects partial or unverified ownership instead of +publishing a subset of an ambiguous batch. Direct publication consumes delegation +markers in the same transaction that installs canonical path heads. + +`crab staging clean` now checks delegated batches under Git's existing immediate +index lock. It reloads immutable lease identities under that lock and retains the +lock through promotions. Every batch path must have a regular-file index entry +whose verified pointer blob matches its file hash and size. Object headers reject +large raw blobs before loading their body; loaded small blobs are checked against +their Git object ID and parsed with the shared pointer contract. Gix 0.83.0's +`find_header` uses gix-odb 0.80.0's fixed loose-header buffer; locked gix-pack 0.70.0 +probes delta headers in 32 bytes. Header inspection does not prove full packed +base reconstruction cost, and this is explicit maintenance rather than a new +foreground query. Typed dependency causes are retained. Cancellation drops the +owned Git lock and preserves batches not yet promoted. + +Missing/different entries retain their leases, including native `--defer` work +and older unpushed history. Current index absence cannot establish that payloads +are disposable. Full abandoned-handoff reclamation and termination qualification +remain required; this slice does not add a silent age-based expiry or a ref-tip-only +history scan. Normal push keeps its selected published/delegated batch lookup. + +Proof: the staging library passes 242 tests with one existing manual RSS test +ignored. The Git-index module passes four tests, including three new real-Git +cases covering acceptance, held index locks, complete multi-path batches, shared +native ownership, invalid small blob identities and oversized raw-blob headers. +The public occurrence regression also passes after the final ownership-query +change. Consumer checks remain in progress at draft publication. The separate +qualification worktree and release artifact remain pinned to #566. + +The follow-up occurrence audit reproduced `chunks_for_file` returning an empty +list for a real prepared-only, verified open version: the public readers selected +only published recipes, then consulted segment rows. The negative receipt is +`retained-recipe-visibility-negative-r1.log`. Both writable and shared hash/size +readers now use any retained verified immutable recipe, including native/delegated +leases and push snapshot pins. Recipe conflicts still fail, and publication/push +eligibility remains a separate, unchanged query. Unsealed segment enumeration +keeps its released staging contract. The public regression compares against direct +CDC, retains repeated occurrences and covers before/after delegation on both +handle types. The corrected regression passes with snapshot-only retention after batch rollback +and empty enumeration after the final snapshot release. + +The separate #566 release-artifact 10-GiB qualification completed native add and +initial push, then Git-clean add and commit. Its next push failed with +`CRAB-E0080`: the pointer file had no staged chunk recipe in publication assembly +(`crab/src/git/xet_publication.rs`). The failed fixture retains both delegated +leases. This maintenance/occurrence slice does not resolve that push blocker; +full alternating-path and cold-reconstruction qualification remains incomplete. + +### Capsule publication uses selected delegated recipes + +The preserved 10-GiB failure identifies a verified delegated recipe with 166,469 +ordered occurrences and a 10,737,418,240-byte size. Public occurrence enumeration +was fixed in #569, but capsule assembly still loaded prepared plans and remote +chunk authority through published-only queries. That would reconstruct delegated +files and rebuild their xorbs instead of using their ingestion proof. + +A minimized real-staging regression also demonstrates capsule assembly accepting +an active native batch before handoff. The negative run completes with the +assertion `unaccepted or inconsistent recipe became publishable` in +`capsule-handoff-negative-r1.log`; its source patch is captured separately. + +The follow-up moves the existing published/delegated merge rule into +`crab/src/git/push_staging.rs`, shared by pipeline admission and capsule assembly. +Capsule assembly requires a selected verified recipe with the pointer's exact +size, reads its ordered pages, and batches prepared authority by recipe identity. +It reads remote authority from that same recipe. Physical xorbs still undergo +body, logical-hash and placement verification before they become dependencies. +There is no admission path for an unrelated active native lease. + +The new regression covers native rejection, delegated size mismatch, and exact +preservation of a non-default prepared encoding. All nine publication tests pass, including the new three-case regression and +existing protected-publication, catalog-integrity, deduplication and concurrency +cases. Real 10-GiB retry and cold historical reconstruction remain pending. +The historical catalog scan in `prepare_delta` remains stage 6 work. + +The caller audit confirms `crab/src/cmd/push.rs` and the remote helper both use +`capsule_push::run`. Neither those success paths nor capsule publication currently +captures or retires staging push snapshots. The legacy pipeline's snapshot tests +therefore do not qualify capsule lifecycle. A separate follow-up must bind exact +selected owners before publication and release only proven committed owners, +including cancellation, partial/ref-CAS failure and concurrent readers. Until that +is implemented and tested, local post-push reclamation is not qualified. + +### Capsule snapshot lifecycle follow-up + +Capsule push fences retirement before selecting local recipes, pins selected +published/delegated owners, and retains that snapshot through upload and ref +publication. Confirmed remote commit precedes the local committed journal and +exact retirement. Local cleanup errors leave maintenance work and preserve the +successful remote result. Ref rejection or failure retains unpublished leases. + +`StagingPushSnapshot` owns marker/snapshot cleanup on drop, including task abort. +Sibling readers defer retirement to exclusive maintenance. It shares the existing +index and physical-payload implementation with the released asynchronous API; +there is no storage-format or dependency change. Guard tests cover drop, task +abort, shared native ownership, sibling deferral and local journal failure. + +Native publication-intent recovery now runs in capsule push before snapshot +admission. Recovery uses Git's immediate index lock. If the complete prior index +state is restored, a bounded-memory local Git object walk checks refs and reflogs +before native rollback. Recipes still referenced there acquire canonical history +owners before native leases are released. Cancellation or an incomplete history +walk fails without authorizing rollback. Ordinary pushes without unresolved +intents do not perform this history scan; this does not replace the historical +remote-catalog indexing workstream. + +The minimum Git 2.30 dependency contract supports the selected walk flags and +includes other worktrees with `--all --reflog` by default +([revision options](https://github.com/git/git/blob/v2.30.0/Documentation/rev-list-options.txt)). +Its transport allowlist takes precedence over configured transport permissions +([transport implementation](https://github.com/git/git/blob/v2.30.0/transport.c)); +the empty allowlist prevents an implicit fetch during local recovery. Windows +startup sets stdout to binary mode, matching the OID parser's LF delimiter +([MinGW startup](https://github.com/git/git/blob/v2.30.0/compat/mingw.c)). + +Both exclusive native-add recovery and shared push recovery use this ownership +rule. The shared staging helper validates all requested recipes against the +intent before adding history owners, preserves the original intent across an +interrupted prefix, and permits an idempotent retry. Prepared-only recovery tests +cover subset retention, full reconstruction and physical xorb deletion only after +confirmed snapshot commit. + +Both protected finalize clients use the shared auth response validator. A response +with coordinator metadata must attest `Committed` or `Materialized` before it can +permit local retirement. `Pending`, `ObjectsUploaded` and `Aborted` are rejected; +the existing successful response without coordinator metadata remains supported. +The minimal protected-push suite passes 30 tests. + +The existing managed-file smoke checks post-push owners, snapshots, raw payloads +and prepared ownership bytes for native and Git helper push. The frozen #566 CLI +baseline completes native add/commit/push and verifies the remote tip, then fails +the retirement assertion with retained owners/prepared payload. Corrected CLI +proof and final-source 10-GiB qualification remain required. + +The large-file harness adds `--pattern entropy|repeated|zero`; entropy preserves +its original seed/default. Repeated input uses a deterministic one-MiB block; +zero input writes real blocks. All patterns retain four alternating native/Git +versions, beginning/middle/end edits, two-file identity checks, source removal and +cold historical reconstruction. Syntax checks pass; the new patterns are unrun. + +The frozen #570 release build succeeds with `simd-accel,gix-pathmatch` and no +default features. Its binary hash is recorded externally. Resumed workload +`unified-ingestion-10g-r3` passes the formerly failing v1 Git-clean push and the +subsequent v2 native push, with 2,216,109 and 2,133,870 incremental xorb bytes. +The full workload is still running; cold historical reconstruction is unproven. +This artifact precedes the lifecycle and finalize-contract changes above. + + +The 10-GiB harness also selects `--first-ingestion native|git`, covering both +native→clean→native and clean→native→clean. Receipts include the selected order +and each add's ingestion path. After each add, the harness rejects local +segment/prepared storage (including temporary stream bodies) beyond one initial +representation, cumulative edited +bytes and metadata allowance (10% of source size, at least 32 MiB). This detects +the observed extra full 10-GiB representation; it is not a latency target or a +claim that all allowed overhead is necessary. Existing runs retain their +original harness identity and cannot prove these newly added checks. + +The local history dependency probe additionally reproduces exit zero with a +missing reflog commit and an omitted expected pointer. Git reports that partial +walk only through diagnostics; its +[reflog traversal implementation](https://github.com/git/git/blob/v2.30.0/revision.c) +confirms that contract. Recovery therefore requires a diagnostic-free successful +walk before accepting absence. A warning retains the unresolved intent and its +payloads; there is no log-message matching or alternate deletion path. Both a +missing reflog tip and a missing referenced tree are regression cases. Earlier +consumer builds do not qualify this tightened scanner until rerun. + +An extracted-source probe compiles the exact helper and canonical subprocess +owner against the frozen release's actual error/dependency types. The old helper +accepts the incomplete walk (negative runtime exits 1); the tightened helper +preserves the healthy reflog result and rejects the missing tip (positive runtime +exits 0). Source hashes are recorded with both probes. This is function-level +runtime proof; the full updated consumer still needs its own Cargo/CLI run. + + +Reduced crossover qualification passes with the frozen #570 release: Git-first +repeated data and native-first zero data, each with two 128-MiB files, four +versions, source removal, eight cold file digests and remote fsck. Each run records +71 commands and 53 passing checks. An older debug binary fails the new storage +assertion after its first native-to-clean switch: 67,156,162 prepared bytes plus +68,224,532 segment bytes exceed the 102,760,448-byte bound for its 64-MiB input. +These runs qualify the order/storage oracle; they do not qualify the final +lifecycle source or replace the full 10-GiB matrix. The later inclusion of +stream-temporary bytes in the accounting remains part of that final matrix. + +The dedicated #572 provider run passes its provider contract and the initial +native cutover push's retirement check, then stops at the old missing-manifest +fixture. It deleted a retired key while `v2/root` remained authoritative. Missing +and corrupt fixtures now initialize a healthy repository, attest the live root, +and verify their deletion/overwrite took effect before requiring list/clone to +fail. Corruption still requires the integrity error; expectations were not +relaxed. Both updated fixtures pass against the unchanged #570 root consumer +(23 commands, 23 checks), and the 11 source-isolation tests pass. The explicit +v1 upgrade/reset fixtures remain separate. Full final-source canary proof is +still pending. + +## Forced publication without a local remote tip + +A clone can diverge after another writer advances the remote. Its explicit +force push must not require fetching the discarded tip. Capsule admission now +allows an unavailable ancestry result only for an explicitly forced update or +matched ref lease when receive policy permits rewrites. Stale leases and +`receive.denyNonFastForwards` still reject; publication retains its expected-old +conditional transition. Real ancestry-probe errors remain errors. + +Proven fast-forwards and rewrites with an available prior tip keep incremental +visibility. Rewrites lacking that tip publish the +existing authenticated replacement-closure representation, walking only the +new tip. This removes old objects from that ref's authorization without reading +discarded history. Pack generation and LFS publication already exclude only +remote tips present in the local object database. Both native CLI and Git +remote-helper publication use this capsule owner. + +The regression matrix covers ordinary, forced, matched/stale lease and denied +updates with the old remote commit absent locally. Successful rewrites verify +the remote tip, revoke old-tip visibility, install packs into an independent +repository and run Git connectivity checks. The existing RustFS ref matrix +also exercises native and remote-helper force pushes without fetching first. +These tests remain pending until the updated-source CI run completes. + +## Packed-object qualification fault fixture + +The Git-current protocol canary stopped before corruption injection because the +LFS pointer blob was packed, while the fixture opened a loose-object pathname. +The same failure reproduces locally after `git repack -ad`. + +The fixture now exports its small disposable source history and unpacks it into +an empty object database before injection. Healthy packed duplicates cannot +mask the corrupt loose blob. The original object database is moved aside and +restored in `finally`; the success check compares every original object-file +hash, the pointer, source/remote refs and working bytes. The caller deliberately +packs its source first so the previous failure remains a regression case. +The checksum mismatch and successful restored dry run remain required. + +This changes qualification setup only. Full Git-current CI remains required; +a source-layout repair does not establish a product-level compatibility result. + +Local proof with the unchanged #572 release: the repaired packed-source mirror +workflow passes 66 commands and 13 checks; deliberate assertion failure after +the verified checksum rejection restores every original object-file hash and +passes strict Git fsck (34 commands, three checks; cleanup scope only). Seventeen +existing log/source-isolation tests pass. These are selected fixture and mirror +receipts, not a full final-source protocol qualification. + +## Combined indexed-catalog and staging qualification + +The integration branch uses `4ebfc3dcece` as its base: captured catalog lookups, +run/checkpoint pages, persistent cache WAL policy and the fixed SQLite runtime. +It applies the canonical staging, recovery, exact snapshot retirement and force +publication work through `033bc5042cf`. Catalog encoding and visibility owners +remain those of the indexed branch; no hybrid format or alternate reader is +introduced. The two benchmark audit reports remain excluded. + +The large-file harness now checks exact staging retirement after every push. +Managed smokes, committed-history smokes and the multi-edit harness share the +same assertion for heads, leases, unpublished batches, reclaimable ownership, +snapshot journals and payload references. Existing check names remain intact. +The stronger oracle passes 83 commands and 65 checks with the frozen #572 +release. Its negative control rejects #570 after a successful publication: +two path heads, two leases, 65 payload records and 4,194,304 retained segment +bytes remain. These receipts establish the oracle and staging component; they +do not qualify the new integrated binary or catalog format. + +Combined-source CLI compilation and runtime qualification are still pending. +Remaining catalog consumers include shared/protected preparation, pinned +hydration/mount lookup and the current-state file lookup opener. Complete +inspection remains appropriate for fsck, compaction and GC; foreground lookups +must not inherit those scans. Carry forward applicable CLI range/cancellation +contracts before final qualification. The complete integrated 10-GiB matrix, +history scaling, lifecycle/migration, fuller blame and representative S3 gates +remain open. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index d8a27e5c8..b79cbe9b9 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -8,7 +8,7 @@ The large-file cases prove pointer staging, object-store publication, fresh clone, hydration, and byte identity. A separate ordinary-Git matrix proves -missing/corrupt manifest handling, update/fetch, deletion, force and +missing/corrupt metadata-root handling, update/fetch, deletion, force and non-fast-forward behavior, atomic rejection, tag following, shallow history, immutable-pack failures, and concurrent push serialization. @@ -745,6 +745,20 @@ def staging_payload_inventory(self, repo: Path) -> dict[str, int]: finally: connection.close() + def assert_staging_retired(self, repo: Path, check_name: str) -> None: + record = self.run_crab(repo, ["staging", "stats", "--json"], name=check_name) + lifecycle = json.loads(self.read_stdout(record))["data"]["lifecycle"] + self.check( + check_name, + all(lifecycle[field] == 0 for field in ( + "path_heads", "path_leases", "open_batches_without_publication", + "reclaimable_superseded_leases", "reclaimable_files", + "open_push_snapshots", "committed_push_snapshots", "payloads", + "current_head_prepared_bytes", "snapshot_pinned_prepared_bytes", + )), + lifecycle, + ) + def sqlite_cache_files(self) -> list[str]: return sorted(path.name for path in self.cache_dir.rglob("*.sqlite")) @@ -850,19 +864,25 @@ def clone_git( check=check, ) - def run_missing_manifest_case(self) -> None: - case_name = "missing-manifest" + def run_missing_root_case(self) -> None: + case_name = "missing-root" _repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) + root_key = f"{repo_prefix}/v2/root" + self.ls_remote(remote_url, name=f"{case_name} healthy list") + self.head_key(root_key) self.run_aws( - "delete canonical manifest fixture", + "delete canonical root fixture", [ "delete-object", "--bucket", self.args.bucket, "--key", - f"{repo_prefix}/manifest", + root_key, ], ) + status, _, _ = self.signed_s3_request("HEAD", root_key) + self.check(f"{case_name}-fixture-removed", status == 404, + {"key": root_key, "status": status}) listing = self.run_git( self.run_root, ["ls-remote", remote_url], @@ -954,6 +974,17 @@ def run_ref_update_delete_force_and_tag_case(self) -> None: {"expected": third}, ) + missing_tip = self.run_git( + contender, + ["cat-file", "-e", third], + name=f"{case_name} remote tip absent locally", + check=False, + ) + self.check( + f"{case_name}-force-push-does-not-require-fetch", + missing_tip.exit_code != 0, + {"remote_tip": third}, + ) self.run_git(contender, ["push", "--force", "origin", "main"]) self.check( f"{case_name}-force-push-updates-tip", @@ -964,6 +995,20 @@ def run_ref_update_delete_force_and_tag_case(self) -> None: {"expected": divergent}, ) + self.run_crab( + repo, + ["push", "--force", "origin", "main"], + name=f"{case_name} native force without fetching contender", + ) + self.check( + f"{case_name}-native-force-push-updates-tip", + self.ls_remote(remote_url, name=f"{case_name} list after native force").get( + "refs/heads/main" + ) + == third, + {"expected": third}, + ) + self.run_git(contender, ["push", "--force", "origin", "main"]) self.run_git(contender, ["tag", "-a", "v1", "-m", "version one"]) tag_oid = self.rev_parse(contender, "refs/tags/v1") self.run_git(contender, ["push", "--follow-tags", "origin", "main"]) @@ -1174,13 +1219,15 @@ def run_shallow_case(self) -> None: {"commit_count": full_depth}, ) - def run_corrupt_manifest_case(self) -> None: - case_name = "corrupt-manifest" - remote_url, repo_prefix = self.remote_for_case(case_name) - manifest_key = f"{repo_prefix}/manifest" + def run_corrupt_root_case(self) -> None: + case_name = "corrupt-root" + _repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) + root_key = f"{repo_prefix}/v2/root" + self.ls_remote(remote_url, name=f"{case_name} healthy list") + self.head_key(root_key) status, _, response_body = self.signed_s3_request( "PUT", - manifest_key, + root_key, body=b'{"version":', ) self.check( @@ -1193,6 +1240,11 @@ def run_corrupt_manifest_case(self) -> None: }, ) + status, _, body = self.signed_s3_request("GET", root_key) + self.check(f"{case_name}-fixture-overwritten", + status == 200 and body == b'{"version":', + {"key": root_key, "status": status}) + ls_remote = self.run_git( self.run_root, ["ls-remote", remote_url], @@ -1607,6 +1659,7 @@ def run_case(self, case_name: str, use_crab_add: bool) -> None: ) self.head_repository_state(repo_prefix) + self.assert_staging_retired(repo, f"{case_name}-retires-published-staging-owners") after_xorbs = self.list_keys(".crab/xorbs/") after_shards = self.list_keys(".crab/shards/") new_xorbs = len(after_xorbs - before_xorbs) @@ -2084,21 +2137,7 @@ def run_committed_restage_before_first_push_case(self) -> None: fsck_data, ) - after_push_record = self.run_crab( - repo, - ["staging", "stats", "--json"], - name=f"{case_name} staging ownership after push", - ) - after_push = json.loads(self.read_stdout(after_push_record))["data"]["lifecycle"] - self.check( - f"{case_name}-retires-both-pushed-version-owners", - after_push["path_heads"] == 0 - and after_push["path_leases"] == 0 - and after_push["open_batches_without_publication"] == 0 - and after_push["reclaimable_superseded_leases"] == 0 - and after_push["reclaimable_files"] == 0, - after_push, - ) + self.assert_staging_retired(repo, f"{case_name}-retires-both-pushed-version-owners") clone_dir = self.run_root / case_name / "clone" self.run_cmd( @@ -2233,8 +2272,8 @@ def run(self) -> None: self.write_report() return self.run_v1_hard_cutover_reset_case() - self.run_missing_manifest_case() - self.run_corrupt_manifest_case() + self.run_missing_root_case() + self.run_corrupt_root_case() self.run_ref_update_delete_force_and_tag_case() self.run_atomic_rejection_case() self.run_shallow_case() diff --git a/crab/scripts/e2e/run_large_file_edits_rustfs.py b/crab/scripts/e2e/run_large_file_edits_rustfs.py index 71323b5bd..ccc6c64e6 100644 --- a/crab/scripts/e2e/run_large_file_edits_rustfs.py +++ b/crab/scripts/e2e/run_large_file_edits_rustfs.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 """Qualify repeated 10-GiB edits through both Git clean and native Crab push. -Two copy-on-write files start with identical, non-sparse deterministic entropy. +Two copy-on-write files start with identical, non-sparse deterministic content. +Choose entropy, a repeated one-MiB block, or zero-filled bytes. +Choose native-first or Git-first alternating ingestion. Edits diverge at the beginning, middle and end. Each committed version must hydrate byte-identically from a fresh cache after the source repository is removed. Only this invocation's disposable data is released; logs and receipts remain. @@ -77,9 +79,15 @@ def qualify(runner: AddCommitPushSmoke, proxy: RequestCountingProxy) -> None: repo, remote, prefix = runner.prepare_repo("source") cap_cache(repo) paths = [repo / "model-a.bin", repo / "model-b.bin"] + repeated_block = None + if args.pattern == "zero": + repeated_block = bytes(MIB) + elif args.pattern == "repeated": + repeated_block = hashlib.shake_256(b"large-edits:repeated").digest(MIB) with paths[0].open("wb") as output: for block in range(args.file_mib): - output.write(hashlib.shake_256(f"large-edits:{block}".encode()).digest(MIB)) + output.write(repeated_block if repeated_block is not None else + hashlib.shake_256(f"large-edits:{block}".encode()).digest(MIB)) clone_flag = "-c" if sys.platform == "darwin" else "--reflink=always" runner.run_cmd("copy-on-write second large file", ["cp", clone_flag, str(paths[0]), str(paths[1])], repo) @@ -89,13 +97,15 @@ def qualify(runner: AddCommitPushSmoke, proxy: RequestCountingProxy) -> None: (code / f"module-{index:05}.rs").write_text(f"pub const VALUE: u64 = {index};\n") runner.check("workload", all(path.stat().st_size == size for path in paths), {"file_bytes": size, "files": 2, "initial_unique_bytes": size, - "versions": 4, "code_files": args.code_files}) + "versions": 4, "code_files": args.code_files, "pattern": args.pattern, + "first_ingestion": args.first_ingestion}) history = [] phases = [] def record() -> None: receipt = runner.artifacts / "large-file-edits.json" - receipt.write_text(json.dumps({"history": history, "phases": phases, + receipt.write_text(json.dumps({"history": history, "phases": phases, "pattern": args.pattern, + "first_ingestion": args.first_ingestion, "transport": proxy.snapshot()}, indent=2) + "\n") runner.report.artifacts["large_file_edits"] = str(receipt) runner.write_report() @@ -109,7 +119,8 @@ def record() -> None: stream.write(hashlib.shake_256(f"edit:{version}:{index}".encode()).digest(MIB)) (code / "module-00000.rs").write_text(f"pub const VERSION: u64 = {version};\n") before = proxy.snapshot() - if version % 2: + native_add = (version % 2 == 0) == (args.first_ingestion == "native") + if not native_add: add = runner.run_git(repo, ["add", "--", *[p.name for p in paths]], name=f"v{version} Git clean-filter add") else: @@ -117,9 +128,17 @@ def record() -> None: staging = repo / ".crab" / "staging" local_bytes = { name: sum(path.stat().st_size for path in (staging / name).rglob("*") if path.is_file()) - for name in ("segments", "push-plans") + for name in ("segments", "push-plans", "stream-prepared-xorbs") } + # One initial representation plus all edited bytes is sufficient. Allow + # encoded metadata overhead, while rejecting the observed second full copy. + max_local_bytes = size + 2 * version * MIB + max(size // 10, 32 * MIB) + runner.check(f"v{version}-no-second-full-local-payload", + sum(local_bytes.values()) <= max_local_bytes, + {"staging_directory_bytes": local_bytes, + "max_bytes": max_local_bytes}) phases.append({"version": version, "phase": "add", "duration_ms": add.duration_ms, + "ingestion": "native" if native_add else "git", "staging_directory_bytes": local_bytes, "transport": RequestCountingProxy.delta(before, proxy.snapshot())}) for path in paths: @@ -132,6 +151,7 @@ def record() -> None: push = runner.run_crab(repo, ["push", "--jsonl", "--upload-concurrency", str(concurrency), "origin", "HEAD:refs/heads/main"], name=f"v{version} Crab push") transport = RequestCountingProxy.delta(before, proxy.snapshot()) + runner.assert_staging_retired(repo, f"v{version}-retires-published-staging-owners") after_inventory = object_inventory(runner, ".crab/xorbs/") added_bytes = after_inventory["bytes"] - before_inventory["bytes"] phases.append({"version": version, "phase": "push", "duration_ms": push.duration_ms, @@ -194,6 +214,8 @@ def main() -> None: parser.add_argument("--endpoint-url", required=True) parser.add_argument("--file-mib", type=int, default=10240) parser.add_argument("--code-files", type=int, default=5000) + parser.add_argument("--pattern", choices=("entropy", "repeated", "zero"), default="entropy") + parser.add_argument("--first-ingestion", choices=("native", "git"), default="native") args = parser.parse_args() if args.file_mib < 16 or args.code_files < 1 or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", args.run_id): parser.error("require >=16 MiB/file, positive code files and a safe run id") diff --git a/crab/scripts/e2e/run_protocol_v2_partial_clone_rustfs_smoke.py b/crab/scripts/e2e/run_protocol_v2_partial_clone_rustfs_smoke.py index bc35dee2b..bce4d99ed 100644 --- a/crab/scripts/e2e/run_protocol_v2_partial_clone_rustfs_smoke.py +++ b/crab/scripts/e2e/run_protocol_v2_partial_clone_rustfs_smoke.py @@ -2680,19 +2680,41 @@ def mirror_lfs_discovery_check(self, source: Path, destination: str) -> None: oid = self.git_value(source, ["rev-parse", "HEAD:hook-lfs.bin"], name="capture LFS pointer Git OID") head = self.git_value(source, ["rev-parse", "HEAD"], name="capture LFS discovery source ref") before = self.git_value(source, ["ls-remote", "--refs", destination], name="capture refs before LFS discovery fault") - path = source / ".git/objects" / oid[:2] / oid[2:] - original = path.read_bytes() + pointer = self.git_value(source, ["cat-file", "blob", oid], name="capture healthy LFS pointer") working_bytes = (source / "hook-lfs.bin").read_bytes() - large_file = self.artifacts / "lfs-discovery-large-blob.bin" - large_file.write_bytes(b"x" * 65536) - large_oid = self.git_value(source, ["hash-object", "-w", str(large_file)], name="create LFS discovery large-header fault") - replacement = (source / ".git/objects" / large_oid[:2] / large_oid[2:]).read_bytes() - command = [str(self.crab_bin), "lfs", "push", "--dry-run", destination] - # Only this disposable fixture's directory entry changes. Restore it - # even if the assertion fails; no source worktree data is rewritten. - path.unlink() + objects = source / ".git/objects" + original_objects = { + str(path.relative_to(objects)): sha256_file(path) + for path in objects.rglob("*") if path.is_file() + } + saved_objects = source / ".git/lfs-discovery-saved-objects" + loose_objects = source / ".git/lfs-discovery-loose-objects" + loose_objects.mkdir() + pack = self.artifacts / "lfs-discovery-source.pack" + self.run_binary( + "export disposable LFS source objects", + [str(self.git_bin), "pack-objects", "--all", "--stdout"], source, pack, + ) + # Unpack into an empty database: Git skips objects already present, and + # a healthy packed duplicate could mask the corrupt loose entry in gix. + self.run_git( + source, ["unpack-objects", "--strict"], + name="prepare isolated loose LFS fault database", + extra_env={"GIT_OBJECT_DIRECTORY": str(loose_objects), "GIT_ALTERNATE_OBJECT_DIRECTORIES": ""}, + input_data=pack.read_bytes(), + ) + make_private_fixture_tree(loose_objects) + objects.rename(saved_objects) try: + loose_objects.rename(objects) + path = objects / oid[:2] / oid[2:] + large_file = self.artifacts / "lfs-discovery-large-blob.bin" + large_file.write_bytes(b"x" * 65536) + large_oid = self.git_value(source, ["hash-object", "-w", str(large_file)], name="create LFS discovery large-header fault") + replacement = (objects / large_oid[:2] / large_oid[2:]).read_bytes() + path.unlink() path.write_bytes(replacement) + command = [str(self.crab_bin), "lfs", "push", "--dry-run", destination] rejected = self.run_cmd("LFS discovery rejects oversized corrupt pointer", command, source, check=False) self.check( "mirror-lfs-discovery-rejects-corrupt-git-pointer", @@ -2702,13 +2724,19 @@ def mirror_lfs_discovery_check(self, source: Path, destination: str) -> None: and self.git_value(source, ["ls-remote", "--refs", destination], name="verify remote refs after LFS discovery refusal") == before, ) finally: - if path.exists(): - path.unlink() - path.write_bytes(original) + # Restore the original object database even when injection or its + # assertion fails; packed/loose layout and original bytes stay intact. + if objects.exists(): + shutil.rmtree(objects) + saved_objects.rename(objects) self.run_cmd("LFS discovery accepts restored Git pointer", command, source) self.check( "mirror-lfs-discovery-restoration-preserves-refs", - path.read_bytes() == original and (source / "hook-lfs.bin").read_bytes() == working_bytes + {str(path.relative_to(objects)): sha256_file(path) + for path in objects.rglob("*") if path.is_file()} == original_objects + and self.git_value(source, ["cat-file", "blob", oid], name="verify restored LFS pointer") == pointer + and self.git_value(source, ["rev-parse", "HEAD"], name="verify restored LFS source ref") == head + and (source / "hook-lfs.bin").read_bytes() == working_bytes and self.git_value(source, ["ls-remote", "--refs", destination], name="verify refs after LFS discovery restoration") == before, ) @@ -2765,6 +2793,7 @@ def mirror_pre_push_batch_checks(self) -> None: ) self.run_git(source, ["add", ".gitattributes", "hook-lfs.bin"]) self.run_git(source, ["commit", "-m", "hook LFS data"]) + self.run_git(source, ["repack", "-ad"], name="pack hook source before LFS discovery fault") self.mirror_lfs_discovery_check(source, destination) self.run_git(source, ["tag", "-a", "hook-v1", "-m", "hook annotated tag"]) selected = self.git_value(source, ["rev-parse", "topic"], name="hook selected commit") diff --git a/crab/src/cmd/add.rs b/crab/src/cmd/add.rs index 4dea2fddd..5dc8759fc 100644 --- a/crab/src/cmd/add.rs +++ b/crab/src/cmd/add.rs @@ -35,6 +35,9 @@ use crate::core::error::{self, CrabError, Result}; use crate::core::output::event_payloads::{FileDonePayload, ProgressPayload}; use crate::core::output::{JsonlStream, OutputMode, emit_json}; use crate::core::pattern::{PatternFilter, build_filter}; +#[cfg(test)] +use crate::git::index::reconcile_publication_intents; +use crate::git::index::{git_index_path_state, reconcile_publication_intents_writer}; use crate::git::progress::{format_bytes, format_rate, is_tty, render_bar}; use crate::git::push::PushConfig; use crab_staging::{PublicationIntentEntry, PublicationIntentId, StagingArea, StagingAreaReadOnly}; @@ -1047,7 +1050,7 @@ async fn execute_add( // pointers published by the process that just released the staging lock. let staging_root = worktree_ctx.shared_staging_dir(); let lock_wait_start = Instant::now(); - let staging = Arc::new(open_add_staging(&staging_root, &repo_root).await?); + let staging = Arc::new(open_add_staging(&staging_root, &repo_root, cancel).await?); let lock_wait_duration_ms = lock_wait_start.elapsed().as_millis() as u64; let (candidates, clean_skipped) = filter_clean_indexed_candidates(&repo_root, candidates, jobs, cancel).await?; @@ -1389,6 +1392,37 @@ async fn execute_add( } let publish_git_index = should_publish_git_index(args, &summary, staged_entries.len()); + if args.skip_git_add && summary.files_failed == 0 { + let handoffs = staged_entries + .iter() + .filter_map(|entry| { + entry.batch_id.as_ref().map(|batch| { + ( + batch, + entry + .abs_path + .strip_prefix(&repo_root) + .unwrap_or(&entry.abs_path), + ) + }) + }) + .collect::>(); + // --defer hands completed versions to a later Git writer. Active + // native index transactions stay caller-owned and cannot be retired + // merely because another push sees the same file hash. + if let Err(error) = staging.delegate_publication(&handoffs) { + let cleanup = rollback_unpublished_open_entries( + &staging, + &mut summary, + &mut staged_entries, + "external publication handoff failed", + ); + stop_progress_ticker(ticker.take(), &ticker_cancel).await; + cleanup?; + return Err(error.into()); + } + } + if publish_git_index && let Err(error) = pin_committed_recipes_before_path_replacement(&staging, &repo_root, &staged_entries) @@ -1622,12 +1656,16 @@ async fn execute_add( Ok(summary) } -async fn open_add_staging(staging_root: &Path, repo_root: &Path) -> Result { +async fn open_add_staging( + staging_root: &Path, + repo_root: &Path, + cancel: &CancellationToken, +) -> Result { let staging = StagingArea::open_blocking(staging_root.to_path_buf(), ADD_STAGING_LOCK_WAIT_BUDGET) .await .map_err(CrabError::from)?; - reconcile_publication_intents_writer(&staging, repo_root)?; + reconcile_publication_intents_writer(&staging, repo_root, cancel)?; Ok(staging) } @@ -3624,121 +3662,6 @@ fn write_pointer_blob_to_repo(repo: &gix::Repository, payload: &[u8]) -> Result< Ok(oid.to_string()) } -fn git_index_path_state(index: &gix_index::File, path: &bstr::BStr) -> String { - let mut hasher = blake3::Hasher::new(); - hasher.update(b"crab git index path state v1\0"); - if let Some(range) = index.entry_range(path) { - for entry in &index.entries()[range] { - hasher.update(&entry.flags.bits().to_le_bytes()); - hasher.update(&entry.mode.bits().to_le_bytes()); - hasher.update(entry.id.as_bytes()); - } - } - hasher.finalize().to_hex().to_string() -} - -pub(crate) async fn reconcile_publication_intents( - staging: &StagingAreaReadOnly, - repo_root: &Path, -) -> Result<()> { - for action in - publication_recovery_actions(staging.unresolved_publication_intents()?, repo_root)? - { - match action { - PublicationRecoveryAction::Publish(intent_id) => { - staging.publish_publication_intent(&intent_id)?; - } - PublicationRecoveryAction::Rollback(intent_id) => { - staging.rollback_publication_intent(&intent_id).await?; - } - } - } - Ok(()) -} - -fn reconcile_publication_intents_writer(staging: &StagingArea, repo_root: &Path) -> Result<()> { - for action in - publication_recovery_actions(staging.unresolved_publication_intents()?, repo_root)? - { - match action { - PublicationRecoveryAction::Publish(intent_id) => { - staging.publish_publication_intent(&intent_id)?; - } - PublicationRecoveryAction::Rollback(intent_id) => { - staging.rollback_publication_intent(&intent_id)?; - } - } - } - Ok(()) -} - -enum PublicationRecoveryAction { - Publish(PublicationIntentId), - Rollback(PublicationIntentId), -} - -fn publication_recovery_actions( - intents: Vec, - repo_root: &Path, -) -> Result> { - use bstr::ByteSlice; - use gix_index::{File, decode, entry}; - - if intents.is_empty() { - return Ok(Vec::new()); - } - let index_path = - crate::git::worktree::WorktreeContext::resolve_from_path(repo_root)?.index_path(); - let index = File::at_or_default( - &index_path, - gix_hash::Kind::Sha1, - false, - decode::Options::default(), - ) - .map_err(|error| { - CrabError::Internal(format!( - "failed to read Git index for add publication recovery: {error}" - )) - })?; - - let mut actions = Vec::with_capacity(intents.len()); - for intent in intents { - let new_matches = intent - .entries - .iter() - .filter(|expected| { - index - .entry_by_path_and_stage( - expected.path_bytes.as_bstr(), - entry::Stage::Unconflicted, - ) - .is_some_and(|actual| actual.id.to_string() == expected.expected_pointer_oid) - }) - .count(); - let old_matches = intent - .entries - .iter() - .filter(|expected| { - git_index_path_state(&index, expected.path_bytes.as_bstr()) - == expected.previous_index_state - }) - .count(); - if new_matches == intent.entries.len() { - actions.push(PublicationRecoveryAction::Publish(intent.intent_id)); - } else if old_matches == intent.entries.len() { - actions.push(PublicationRecoveryAction::Rollback(intent.intent_id)); - } else { - return Err(CrabError::StagingCorrupt(format!( - "add publication {} has an ambiguous Git index state ({new_matches}/{} new paths, {old_matches}/{} prior paths); restore one complete index state, then retry", - intent.intent_id.as_str(), - intent.entries.len(), - intent.entries.len(), - ))); - } - } - Ok(actions) -} - #[cfg(test)] fn publish_git_index_entries( entries: &[GitIndexEntry], @@ -5972,7 +5895,8 @@ mod tests { let repo_root = repo_root.clone(); let path = path.clone(); async move { - let staging = open_add_staging(&root, &repo_root).await?; + let staging = + open_add_staging(&root, &repo_root, &CancellationToken::new()).await?; let filtered = filter_clean_indexed_candidates( &repo_root, vec![(path, data.len() as u64)], @@ -6306,7 +6230,9 @@ mod tests { assert!(intent.is_some()); staging.close().await.unwrap(); - let recovered = open_add_staging(&staging_root, &repo).await.unwrap(); + let recovered = open_add_staging(&staging_root, &repo, &CancellationToken::new()) + .await + .unwrap(); assert!( recovered @@ -6352,7 +6278,7 @@ mod tests { staging.close().await.unwrap(); let recovered = StagingAreaReadOnly::open(staging_root).await.unwrap(); - reconcile_publication_intents(&recovered, &repo) + reconcile_publication_intents(&recovered, &repo, &CancellationToken::new()) .await .unwrap(); @@ -6371,6 +6297,187 @@ mod tests { ); } + #[tokio::test] + async fn publication_recovery_preserves_attempt_while_git_index_is_locked() { + for read_only in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let repo = dir.path().join("repo"); + std::fs::create_dir(&repo).unwrap(); + assert!(init_git_repo(&repo)); + let root = repo.join(".crab/staging"); + let staging = StagingArea::open(root.clone()).await.unwrap(); + let path = PathBuf::from("model.bin"); + let data = b"Git has not installed its pointer yet"; + let (batch_id, file_hash) = stage_publication_recipe(&staging, &path, data).await; + let index = read_test_index(&repo); + staging + .create_publication_intent(&[PublicationIntentEntry { + batch_id, + path: path.clone(), + expected_pointer_oid: "1111111111111111111111111111111111111111".to_owned(), + previous_index_state: git_index_path_state( + &index, + git_index_path_bstring(&path).as_bstr(), + ), + }]) + .unwrap(); + let lock = gix_lock::Marker::acquire_to_hold_resource( + repo.join(".git/index"), + gix_lock::acquire::Fail::Immediately, + None, + ) + .unwrap(); + let (error, intents, bytes) = if read_only { + staging.close().await.unwrap(); + let staging = StagingAreaReadOnly::open(root).await.unwrap(); + let result = + reconcile_publication_intents(&staging, &repo, &CancellationToken::new()).await; + ( + result.unwrap_err(), + staging.unresolved_publication_intents().unwrap().len(), + staging.get_chunk(&compute_data_hash(data)).await.unwrap(), + ) + } else { + let result = reconcile_publication_intents_writer( + &staging, + &repo, + &CancellationToken::new(), + ); + let state = ( + result.unwrap_err(), + staging.unresolved_publication_intents().unwrap().len(), + staging.get_chunk(&compute_data_hash(data)).await.unwrap(), + ); + staging.close().await.unwrap(); + state + }; + let owns_lock = repo.join(".git/index.lock").exists(); + drop(lock); + assert!( + matches!(error, CrabError::Io(_)) + && intents == 1 + && bytes.as_deref() == Some(data.as_slice()) + && owns_lock, + "{file_hash}: recovery must preserve the attempt and other owner's lock" + ); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn publication_recovery_waits_for_real_git_clean_index_installation() { + struct GitWriter(std::process::Child, PathBuf); + impl Drop for GitWriter { + fn drop(&mut self) { + // Release the child filter as well as the Git parent on panic. + let _ = std::fs::write(&self.1, b""); + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + let dir = tempfile::tempdir().unwrap(); + let repo = dir.path(); + assert!(init_git_repo(repo)); + let root = repo.join(".crab/staging"); + let staging = StagingArea::open(root.clone()).await.unwrap(); + let path = PathBuf::from("model.bin"); + let data = b"real Git clean has not returned its pointer"; + let (batch_id, file_hash) = stage_publication_recipe(&staging, &path, data).await; + let pointer = Pointer { + file_hash: file_hash.into(), + size: data.len() as u64, + shard_hint: None, + } + .serialize(); + let expected_pointer_oid = write_pointer_blob(repo, &pointer).unwrap(); + staging + .create_publication_intent(&[PublicationIntentEntry { + batch_id, + path: path.clone(), + expected_pointer_oid, + previous_index_state: git_index_path_state( + &read_test_index(repo), + git_index_path_bstring(&path).as_bstr(), + ), + }]) + .unwrap(); + staging.close().await.unwrap(); + let staging = StagingAreaReadOnly::open(root).await.unwrap(); + std::fs::write(repo.join(&path), data).unwrap(); + std::fs::write( + repo.join(".gitattributes"), + b"model.bin filter=paused -text\n", + ) + .unwrap(); + std::fs::write(repo.join("filter.pointer"), pointer).unwrap(); + std::fs::write(repo.join("filter.sh"), + b"cat > /dev/null\n: > filter.ready\nwhile test ! -e filter.release; do sleep 0.01; done\ncat filter.pointer\n" + ).unwrap(); + let mut git = GitWriter( + std::process::Command::new("git") + .args([ + "-c", + "filter.paused.clean=sh filter.sh", + "-c", + "filter.paused.required=true", + "add", + "--", + "model.bin", + ]) + .current_dir(repo) + .spawn() + .unwrap(), + repo.join("filter.release"), + ); + tokio::time::timeout(Duration::from_secs(10), async { + while !repo.join("filter.ready").exists() { + assert!( + git.0.try_wait().unwrap().is_none(), + "Git exited before cleaning" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let blocked = reconcile_publication_intents(&staging, repo, &CancellationToken::new()) + .await + .unwrap_err(); + let retained = staging.unresolved_publication_intents().unwrap().len() == 1 + && staging + .get_chunk(&compute_data_hash(data)) + .await + .unwrap() + .as_deref() + == Some(data.as_slice()); + std::fs::write(repo.join("filter.release"), b"").unwrap(); + let status = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(status) = git.0.try_wait().unwrap() { + break status; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(status.success(), "Git index installation failed"); + reconcile_publication_intents(&staging, repo, &CancellationToken::new()) + .await + .unwrap(); + assert!( + matches!(blocked, CrabError::Io(_)) + && retained + && staging.unresolved_publication_intents().unwrap().is_empty() + && staging + .published_recipe_for_file(&file_hash) + .unwrap() + .is_some() + && !repo.join(".git/index.lock").exists(), + "recovery must retain bytes during clean and publish after Git installs the pointer" + ); + } + #[tokio::test] async fn publication_recovery_rejects_mixed_index_state() { let dir = tempfile::tempdir().unwrap(); @@ -6417,7 +6524,7 @@ mod tests { staging.close().await.unwrap(); let recovered = StagingAreaReadOnly::open(staging_root).await.unwrap(); - let error = reconcile_publication_intents(&recovered, &repo) + let error = reconcile_publication_intents(&recovered, &repo, &CancellationToken::new()) .await .expect_err("mixed publication state must fail closed"); diff --git a/crab/src/cmd/staging.rs b/crab/src/cmd/staging.rs index 0a8637714..abb892d0d 100644 --- a/crab/src/cmd/staging.rs +++ b/crab/src/cmd/staging.rs @@ -214,6 +214,17 @@ pub async fn run_staging_clean( check_cancelled(cancel)?; + let handoffs_published = if staging.delegated_publications()?.is_empty() { + 0 + } else { + let worktree = crate::git::worktree::WorktreeContext::resolve()?; + crate::git::index::reconcile_delegated_publications( + &staging, + &worktree.current_worktree_root, + cancel, + )? + }; + check_cancelled(cancel)?; let clean_stats = staging.clean()?; let abandoned_stats = if prune_abandoned { @@ -231,10 +242,12 @@ pub async fn run_staging_clean( stale_markers_removed = clean_stats.stale_markers_removed, abandoned_segments_removed = abandoned_stats.map_or(0, |(s, _, _)| s), abandoned_bytes_reclaimed = abandoned_stats.map_or(0, |(_, b, _)| b), + handoffs_published, "staging clean complete" ); println!("Staging clean complete:"); + println!(" Handoffs published: {handoffs_published}"); println!(" Segments removed: {}", clean_stats.segments_removed); println!( " Segments compacted: {}", diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index 48d716deb..dbcad109e 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use bytes::Bytes; use crab_staging::StagingAreaReadOnly; +use crab_xet::hash::MerkleHash; use gix_object::{Exists, Find, FindHeader}; use tokio_util::sync::CancellationToken; @@ -256,12 +257,18 @@ async fn run_inner( } else { is_ancestor(&common_git_dir, old_oid, &new_oid)? }; - let Some(is_fast_forward) = is_fast_forward else { - outcomes.insert( - spec.dst.clone(), - RefPushOutcome::Rejected(PushRejectReason::StaleInfo), - ); - continue; + let is_fast_forward = match is_fast_forward { + Some(proven) => proven, + // A forced rewrite need not have fetched the remote tip. The ref + // lease was checked above; publication still compares the old tip. + None if update.force && !config.receive_deny_non_fast_forwards => false, + None => { + outcomes.insert( + spec.dst.clone(), + RefPushOutcome::Rejected(PushRejectReason::StaleInfo), + ); + continue; + } }; let decision = if !is_fast_forward && config.receive_deny_non_fast_forwards { Err(PushRejectReason::DenyNonFastForward) @@ -322,6 +329,25 @@ async fn run_inner( config.force_full_graph, ) .await?; + if !prepared.pointers.is_empty() + && let Some(staging) = staging + && !staging.unresolved_publication_intents()?.is_empty() + { + // A native add can install Git's pointer before its local publication + // acknowledgement. Recover under the canonical index lock before admission. + let worktree = gix::open(&git_dir) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))? + .workdir() + .map(Path::to_path_buf) + .ok_or_else(|| { + CrabError::StagingCorrupt( + "native publication recovery requires a Git working tree".to_owned(), + ) + })?; + super::index::reconcile_publication_intents(staging, &worktree, cancel).await?; + } + + let staging_snapshot = capture_staging_snapshot(staging, &prepared.pointers)?; let visibility_delta = prepare_visibility_delta( &common_git_dir, &edits, @@ -519,6 +545,11 @@ async fn run_inner( } }; if let Some(active_active_commit) = committed { + if let Some(snapshot) = staging_snapshot + && let Err(error) = snapshot.commit() + { + tracing::warn!(%error, "capsule committed; local staging cleanup needs maintenance"); + } for ref_name in ref_names { outcomes.insert(ref_name, RefPushOutcome::Ok); } @@ -544,6 +575,41 @@ async fn run_inner( Ok((PushResult::new(outcomes), None)) } +fn capture_staging_snapshot( + staging: Option<&Arc>, + pointers: &[crab_types::pointer::Pointer], +) -> Result> { + let Some(staging) = staging.filter(|_| !pointers.is_empty()) else { + return Ok(None); + }; + // Fence sibling retirement before selecting local roots; snapshot pins + // then protect exactly those leases through upload and the ref transition. + let snapshot = crab_staging::StagingPushSnapshot::begin( + Arc::clone(staging), + uuid::Uuid::now_v7().to_string(), + )?; + let hashes = pointers + .iter() + .map(|pointer| MerkleHash::from(pointer.file_hash)) + .collect::>(); + let selected = super::push_staging::recipes_for_selected_pointers(staging, &hashes)?; + for pointer in pointers { + if let Some(Some(recipe)) = selected.get(&MerkleHash::from(pointer.file_hash)) + && recipe.file_size() != pointer.size + { + return Err(CrabError::StagingCorrupt( + "selected push recipe disagrees with pointer size".to_owned(), + )); + } + } + let recipes = selected.into_values().flatten().collect::>(); + if recipes.is_empty() { + return Ok(None); + } + snapshot.capture(&recipes)?; + Ok(Some(snapshot)) +} + enum CapsulePublishAttempt { Committed(Option), RefChanged, @@ -676,6 +742,15 @@ fn prepare_visibility_delta( ) -> Result> { let maximum = usize::try_from(crab_metadata::git_visibility::MAX_GIT_VISIBILITY_OBJECTS) .map_err(|_| CrabError::Internal("Git visibility limit does not fit usize".to_owned()))?; + let rewritten_old_refs = edits + .iter() + .filter(|edit| edit.new_oid().is_some() && !fast_forward_refs.contains(edit.ref_name())) + .filter_map(|edit| { + edit.expected_old() + .map(|oid| (edit.ref_name().to_owned(), oid.to_owned())) + }) + .collect(); + let available_old_tips = locally_available_remote_tips(git_dir, &rewritten_old_refs)?; let mut visibility = BTreeMap::new(); for edit in edits { let Some(new_oid) = edit.new_oid() else { @@ -696,7 +771,10 @@ fn prepare_visibility_delta( pack_objects.iter().map(ToString::to_string).collect(), Vec::new(), ) - } else if let Some(old_oid) = edit.expected_old() { + } else if let Some(old_oid) = edit.expected_old() + && (fast_forward_refs.contains(edit.ref_name()) + || available_old_tips.iter().any(|tip| tip == old_oid)) + { let added = super::push::enumerate_visibility_difference( git_dir, new_oid, @@ -724,11 +802,13 @@ fn prepare_visibility_delta( removed, ) } else { + // Rewrites may lack the old tip locally. A complete replacement + // closure revokes its visibility without fetching discarded history. let objects = super::push::enumerate_visibility_difference(git_dir, new_oid, None, maximum)? .ok_or_else(|| visibility_limit_error(edit.ref_name()))?; crab_metadata::git_visibility::GitVisibilityEdit::from_replacement_objects( - None, + edit.expected_old().map(str::to_owned), new_oid.to_owned(), objects, ) @@ -1351,6 +1431,156 @@ mod tests { ); } + #[tokio::test] + async fn force_rewrite_without_local_remote_tip_preserves_policy_and_visibility() { + let _git_env = crate::test::git_repo::CleanGitEnvGuard::new(); + for (name, force, lease, deny, succeeds) in [ + ("ordinary", false, None, false, false), + ("force", true, None, false, true), + ("lease", false, Some(true), false, true), + ("stale-lease", true, Some(false), false, false), + ("deny-force", true, None, true, false), + ("deny-lease", false, Some(true), true, false), + ] { + let source = tempfile::tempdir().expect("source repository"); + git(source.path(), &["init", "-b", "main"]); + git(source.path(), &["config", "user.name", "Crab Test"]); + git( + source.path(), + &["config", "user.email", "crab@example.invalid"], + ); + let first = commit(source.path(), "first"); + let contender_root = tempfile::tempdir().expect("contender directory"); + let contender = contender_root.path().join("repo"); + git( + contender_root.path(), + &[ + "clone", + "--no-local", + source.path().to_str().unwrap(), + "repo", + ], + ); + git(&contender, &["config", "user.name", "Crab Test"]); + git( + &contender, + &["config", "user.email", "crab@example.invalid"], + ); + let second = commit(source.path(), "second"); + let divergent = commit(&contender, "divergent"); + let git_dir = contender.join(".git"); + assert_eq!( + is_ancestor(&git_dir, &second, &divergent).unwrap(), + None, + "{name}" + ); + + let store = crate::storage::store::Store::from_storage(crab_storage::Store::new( + Arc::new(InMemory::new()), + )); + let router = crate::storage::StoreLayout::new(store.clone(), "repos/test".to_owned()); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/test".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let limits = crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 16 * 1024 * 1024, + max_frontier_bytes: 16 * 1024 * 1024, + }; + let spec = PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }; + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..PushConfig::default() + }; + let (initial, _) = run( + &config, + std::slice::from_ref(&spec), + &store, + &router, + None, + &[], + None, + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(initial.all_ok(), "{name}: initial publication"); + let mut config = PushConfig { + git_dir: Some(git_dir), + receive_deny_non_fast_forwards: deny, + ..PushConfig::default() + }; + if let Some(matches) = lease { + config.expected_refs.insert( + "refs/heads/main".to_owned(), + Some(if matches { second.clone() } else { first }), + ); + } + let spec = PushSpec { force, ..spec }; + let (result, _) = run( + &config, + &[spec], + &store, + &router, + None, + &[], + None, + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(result.all_ok(), succeeds, "{name}"); + let view = crab_read::capsule_protocol::open_view(&layout, limits) + .await + .unwrap(); + assert_eq!( + view.refs()["refs/heads/main"], + if succeeds { + divergent.clone() + } else { + second.clone() + }, + "{name}: remote tip" + ); + if succeeds { + let visibility = view.git_visibility_index().unwrap(); + let refs = visibility.ref_closures(); + let closure = &refs["refs/heads/main"]; + assert!( + closure.binary_search(&divergent).is_ok() + && closure.binary_search(&second).is_err(), + "{name}: rewrite must retire old visibility" + ); + let destination = tempfile::tempdir().unwrap(); + git(destination.path(), &["init", "--bare"]); + crab_read::capsule_protocol::install_git_packs_from_store( + &view, + &layout, + destination.path(), + 16 * 1024 * 1024, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + git( + destination.path(), + &["cat-file", "-e", &format!("{divergent}^{{commit}}")], + ); + git(destination.path(), &["fsck", "--connectivity-only"]); + } + } + } + #[test] fn missing_remote_tip_requests_refresh_instead_of_internal_failure() { let source = tempfile::tempdir().expect("source repository"); @@ -2001,6 +2231,153 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn capsule_push_retires_staging_only_after_confirmed_ref_publication() { + use bstr::ByteSlice; + use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; + let _git_env = crate::test::git_repo::CleanGitEnvGuard::new(); + for (stale, native_intent, restored_index) in [ + (false, false, false), + (true, false, false), + (false, true, false), + (true, true, false), + (false, true, true), + ] { + let source = tempfile::tempdir().unwrap(); + git(source.path(), &["init", "-b", "main"]); + git(source.path(), &["config", "user.name", "Crab Test"]); + git( + source.path(), + &["config", "user.email", "crab@example.invalid"], + ); + let first = commit(source.path(), "initial"); + let staging_root = source.path().join(".crab/staging"); + let staging = crab_staging::StagingArea::open(staging_root.clone()) + .await + .unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(b"capsule payload"), + Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + if !native_intent { + staging + .delegate_publication(&[(&staged.batch_id, Path::new("model.bin"))]) + .unwrap(); + } + let pointer = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); + if native_intent { + let index = gix_index::File::at_or_default( + source.path().join(".git/index"), + gix_hash::Kind::Sha1, + false, + gix_index::decode::Options::default(), + ) + .unwrap(); + let previous_index_state = + super::super::index::git_index_path_state(&index, b"model.bin".as_bstr()); + staging + .create_publication_intent(&[crab_staging::PublicationIntentEntry { + batch_id: staged.batch_id.clone(), + path: "model.bin".into(), + expected_pointer_oid: git( + source.path(), + &["hash-object", "-w", "model.bin"], + ), + previous_index_state, + }]) + .unwrap(); + } + git(source.path(), &["add", "model.bin"]); + git(source.path(), &["commit", "-m", "managed payload"]); + let tip = git(source.path(), &["rev-parse", "HEAD"]); + if restored_index { + git(source.path(), &["read-tree", "HEAD^"]); + } + staging.close().await.unwrap(); + let staging = Arc::new(StagingAreaReadOnly::open(staging_root).await.unwrap()); + let storage = crab_storage::Store::new(Arc::new(InMemory::new())); + let store = crate::storage::store::Store::from_storage(storage); + let router = crate::storage::StoreLayout::new(store.clone(), "repos/test".to_owned()); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/test".to_owned()); + let root = crab_write::capsule_protocol::initialize( + &layout, + &"1".repeat(64), + "refs/heads/main", + ) + .await + .unwrap(); + let limits = crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 16 * 1024 * 1024, + max_frontier_bytes: 16 * 1024 * 1024, + }; + let initial = crab_read::capsule_protocol::open_view_from_root(&layout, root, limits) + .await + .unwrap(); + if stale { + publish_ref_edits_with_visibility( + &layout, + &source.path().join(".git"), + vec![crab_metadata::capsule_protocol::CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(first.clone()), + None, + )], + ) + .await; + } + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..Default::default() + }; + let (result, _) = run( + &config, + &[PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &router, + Some(initial.into()), + &[], + Some(&staging), + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + let health = staging.lifecycle_health().unwrap(); + let view = crab_read::capsule_protocol::open_view(&layout, limits) + .await + .unwrap(); + assert!( + result.all_ok() != stale + && view.refs().get("refs/heads/main") + == Some(if stale { &first } else { &tip }) + && health.path_leases == u64::from(stale) + && health.payloads == u64::from(stale) + && health.open_push_snapshots == 0 + && health.committed_push_snapshots == 0 + && health.unresolved_publications == 0 + && staging.list_inflight().unwrap().is_empty(), + "stale={stale}, native_intent={native_intent}, restored_index={restored_index}, health={health:?}" + ); + } + } + #[tokio::test] async fn single_ref_successor_extends_committed_multi_ref_history() { let source = tempfile::tempdir().expect("source repository"); diff --git a/crab/src/git/clean.rs b/crab/src/git/clean.rs index 1b6dde055..6c82b7dc5 100644 --- a/crab/src/git/clean.rs +++ b/crab/src/git/clean.rs @@ -291,6 +291,13 @@ impl FileIndexChecker for StoreFileIndexChecker { /// The real implementation writes chunks to the segment-based staging area /// and records them in index.db within a transaction. pub trait ChunkStager: Send { + /// Return the local reader-ingestion owner when this backend provides it. + /// + /// The default preserves custom stagers in the released v1.2.4 API. + fn reader_stager(&self) -> Option<&StagingChunkStager> { + None + } + /// Stage a batch of chunks transactionally. /// /// On success, all chunks are recorded in index.db. Callers must @@ -412,6 +419,7 @@ impl ChunkStager for NoopChunkStager { /// `StagingArea` API by using `Handle::block_on`. This is safe because /// the filter-process loop runs inside `spawn_blocking`, so blocking on /// async calls won't starve the tokio runtime. +#[derive(Clone)] pub struct StagingChunkStager { staging: Arc, handle: tokio::runtime::Handle, @@ -425,12 +433,43 @@ impl StagingChunkStager { pub fn new(staging: Arc, handle: tokio::runtime::Handle) -> Self { Self { staging, handle } } + + fn stage_reader( + &self, + reader: &mut R, + path: &Path, + ctx: &AppContext, + ) -> Result { + use crab_staging::stream::{StreamStageProgress, StreamStageXorbBuilder}; + use crab_xet::xorb::builder::XorbBuilder; + + let config = Arc::new(crate::git::push::PushConfig::from_config(ctx.config())); + let builder = StreamStageXorbBuilder::new(1, move || { + config.configure_builder(XorbBuilder::with_policy(config.compression_policy())) + }); + self.handle + .block_on(crab_staging::stream::stage_reader_streaming( + reader, + path, + &self.staging, + StreamStageProgress { + xorb_builder: Some(builder), + ..StreamStageProgress::default() + }, + &ctx.cancel_token(), + )) + .map_err(CrabError::from) + } } // SAFETY: StagingArea is Send (Arc-wrapped internals), and Handle is Send+Sync. unsafe impl Send for StagingChunkStager {} impl ChunkStager for StagingChunkStager { + fn reader_stager(&self) -> Option<&StagingChunkStager> { + Some(self) + } + fn stage_chunks( &self, chunks: &[([u8; 32], Bytes)], @@ -484,7 +523,7 @@ impl ChunkStager for StagingChunkStager { .record_verified_recipe_lease(&batch_id, path, recipe)?; self.staging .record_file_path(&file_hash, &path.to_string_lossy())?; - self.staging.mark_batch_published(&batch_id)?; + self.staging.delegate_publication(&[(&batch_id, path)])?; Ok(()) })(); if let Err(error) = publish { @@ -1102,6 +1141,75 @@ impl PreparedRecipeVerifier { } } +fn prepared_prefix_reader( + stager: StagingChunkStager, + recipe: FileRecipe, + occurrences: u64, + error: std::rc::Rc>>, +) -> impl tokio::io::AsyncRead + Unpin { + let state = ( + stager, + PreparedRecipeCursor::new(recipe), + occurrences, + std::collections::VecDeque::::new(), + ); + let chunks = futures_util::stream::try_unfold(state, move |state| { + let error = error.clone(); + async move { + let (stager, mut cursor, mut remaining, mut payloads) = state; + let read = async { + if payloads.is_empty() && remaining > 0 { + let mut terms = Vec::new(); + while remaining > 0 && terms.len() < PREPARED_REPLAY_BATCH_CHUNKS { + let chunk = cursor.next(&stager)?.ok_or_else(|| { + CrabError::StagingCorrupt( + "prepared prefix ended before verified occurrences".to_owned(), + ) + })?; + terms.push(chunk); + remaining -= 1; + } + let hashes = terms + .iter() + .map(|chunk| chunk.chunk_hash) + .collect::>(); + // This source is polled inside the engine's block_on. + // Await staging reads directly; a nested stager bridge + // would enter the runtime twice and panic. + let read = stager.staging.get_chunks_batch(&hashes).await?; + if read.len() != terms.len() { + return Err(CrabError::StagingCorrupt( + "prepared prefix lost an authoritative chunk".to_owned(), + )); + } + for ((hash, bytes), term) in read.into_iter().zip(terms) { + if hash != term.chunk_hash + || bytes.len() as u64 != term.len + || crab_xet::hash::compute_data_hash(&bytes) != term.chunk_hash + { + return Err(CrabError::StagingCorrupt( + "prepared prefix payload differs from verified input".to_owned(), + )); + } + payloads.push_back(bytes); + } + } + Ok(payloads.pop_front()) + } + .await; + match read { + Ok(Some(bytes)) => Ok(Some((bytes, (stager, cursor, remaining, payloads)))), + Ok(None) => Ok(None), + Err(cause) => { + *error.borrow_mut() = Some(cause); + Err(std::io::Error::other("prepared prefix read failed")) + } + } + } + }); + tokio_util::io::StreamReader::new(Box::pin(chunks)) +} + struct PreparedChangedReplay { chunker: Option, buffered: BatchedChunkStager, @@ -1854,6 +1962,14 @@ impl CleanSession { /// hydrated files succeed even while another crab process holds /// `.crab/staging`. fn crab_clean_path(&mut self, pathname: &str, content: Vec) -> Result> { + if self.staging_unavailable.is_none() + && let Some(stager) = self.chunk_stager.reader_stager().cloned() + { + return self.crab_clean_reader(pathname, &mut std::io::Cursor::new(content), &stager); + } + // Custom stagers remain a released v1.2.4 library contract. Their + // byte-to-chunk adapter is retained until a breaking library release; + // the production local backend delegates ingestion to crab-staging. use crab_xet::chunker::GearChunker; let file_size = content.len() as u64; @@ -2097,6 +2213,19 @@ impl CleanSession { return self.crab_clean_prepared_stream(pathname, first, reader, reached_flush, recipe); } + if self.staging_unavailable.is_none() + && let Some(stager) = self.chunk_stager.reader_stager().cloned() + { + let cancel = self.ctx.cancel_token(); + let mut stream = reader.byte_stream(first, reached_flush, &cancel); + let result = self.crab_clean_reader(pathname, &mut stream, &stager); + if result.is_err() + && let Some(error) = stream.take_error() + { + return Err(error); + } + return result; + } let staging_available = self.staging_unavailable.is_none(); let mut file_hasher = blake3::Hasher::new(); let mut chunker = staging_available.then(GearChunker::new); @@ -2236,6 +2365,160 @@ impl CleanSession { result } + fn crab_clean_reader( + &mut self, + pathname: &str, + reader: &mut R, + stager: &StagingChunkStager, + ) -> Result> { + let staged = stager.stage_reader(reader, Path::new(pathname), &self.ctx)?; + let publication = (|| -> Result<(Vec, bool)> { + self.ctx.check_cancelled()?; + if let Some(pointer) = self.try_index_pointer_match(pathname, &staged.file_hash) { + return Ok((pointer, false)); + } + if let Some(pointer) = self.try_fast_path(staged.size, &staged.file_hash)? { + return Ok((pointer, false)); + } + self.preserve_committed_recipe_before_publication(pathname, &staged.file_hash)?; + let pointer = self + .build_pointer(staged.file_hash, staged.size, None) + .serialize(); + Ok((pointer, true)) + })(); + match publication { + Ok((pointer, false)) => { + stager.staging.rollback_batch(&staged.batch_id)?; + Ok(pointer) + } + Ok((pointer, true)) => { + if let Err(error) = stager + .staging + .delegate_publication(&[(&staged.batch_id, Path::new(pathname))]) + { + let _ = stager.staging.rollback_batch(&staged.batch_id); + return Err(error.into()); + } + Ok(pointer) + } + Err(error) => { + if let Err(cleanup) = stager.staging.rollback_batch(&staged.batch_id) { + tracing::warn!(error = %cleanup, path = %pathname, "clean reader rollback failed"); + } + Err(error) + } + } + } + + fn crab_clean_prepared_reader( + &mut self, + pathname: &str, + first: Vec, + reader: &mut crate::git::filter_process::PktLineReader, + reached_flush: bool, + prepared: FileRecipe, + stager: &StagingChunkStager, + ) -> Result> { + use tokio::io::AsyncReadExt; + + // Retain the source recipe before verification/replay. Original native + // rollback cannot invalidate a prefix the filter already accepted. + let batch = stager.staging.create_batch()?; + if let Err(error) = + stager + .staging + .record_verified_recipe_lease(&batch, Path::new(pathname), &prepared) + { + let _ = stager.staging.rollback_batch(&batch); + return Err(error.into()); + } + let mut delegated = false; + let result = (|| { + let mut verifier = PreparedRecipeVerifier::new(prepared.clone()); + let mut hasher = blake3::Hasher::new(); + let mut size = 0u64; + let mut bytes = Bytes::from(first); + let mut at_flush = reached_flush; + let (occurrences, changed) = loop { + self.ctx.check_cancelled()?; + hasher.update(&bytes); + size = size.checked_add(bytes.len() as u64).ok_or_else(|| { + CrabError::StagingCorrupt("prepared verification size overflow".to_owned()) + })?; + match verifier.feed(&bytes, stager)? { + PreparedVerification::Changed { + verified_occurrences, + bytes, + } => break (verified_occurrences, bytes), + PreparedVerification::Matching => {} + PreparedVerification::Exact => { + return Err(CrabError::Internal( + "prepared verifier ended before EOF".to_owned(), + )); + } + } + if !at_flush { + if let Some(packet) = reader.read_packet()? { + bytes = Bytes::copy_from_slice(packet); + continue; + } + at_flush = true; + } + match verifier.finish(stager)? { + PreparedVerification::Changed { + verified_occurrences, + bytes, + } => break (verified_occurrences, bytes), + PreparedVerification::Exact => { + let hash = *hasher.finalize().as_bytes(); + if prepared.file_hash() != MerkleHash::from(hash) + || prepared.file_size() != size + { + return Err(CrabError::StagingCorrupt( + "prepared chunks matched but file identity differed".to_owned(), + )); + } + self.preserve_committed_recipe_before_publication(pathname, &hash)?; + stager + .staging + .delegate_publication(&[(&batch, Path::new(pathname))])?; + delegated = true; + return Ok(self.build_pointer(hash, size, None).serialize()); + } + PreparedVerification::Matching => { + return Err(CrabError::Internal( + "prepared verifier incomplete at EOF".to_owned(), + )); + } + } + }; + let original = std::rc::Rc::new(std::cell::RefCell::new(None)); + let prefix = prepared_prefix_reader( + stager.clone(), + prepared.clone(), + occurrences, + original.clone(), + ); + let cancel = self.ctx.cancel_token(); + let tail = reader.byte_stream(changed.to_vec(), at_flush, &cancel); + let mut source = prefix.chain(tail); + let result = self.crab_clean_reader(pathname, &mut source, stager); + if result.is_err() { + if let Some(error) = original.borrow_mut().take() { + return Err(error); + } + if let Some(error) = source.get_mut().1.take_error() { + return Err(error); + } + } + result + })(); + if !delegated && let Err(error) = stager.staging.rollback_batch(&batch) { + tracing::warn!(error = %error, path = %pathname, "prepared verification lease cleanup failed"); + } + result + } + /// Verify a deferred recipe against its ordered local chunks and promote /// it without CDC, staging writes, or a full-file spool. A changed stream /// reconstructs only its verified prefix before continuing through the @@ -2248,6 +2531,18 @@ impl CleanSession { reached_flush: bool, prepared: FileRecipe, ) -> Result> { + if self.staging_unavailable.is_none() + && let Some(stager) = self.chunk_stager.reader_stager().cloned() + { + return self.crab_clean_prepared_reader( + pathname, + first, + reader, + reached_flush, + prepared, + &stager, + ); + } let verify_start = std::time::Instant::now(); let mut hasher = blake3::Hasher::new(); let mut file_size = 0u64; @@ -4730,6 +5025,210 @@ mod tests { ); } + #[test] + fn native_and_git_clean_crossover_reuses_payloads_and_reconstructs_edits() { + for native_first in [false, true] { + let rt = tokio::runtime::Runtime::new().unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("model.bin"); + let staging = Arc::new( + rt.block_on(StagingArea::open(tmp.path().join("staging"))) + .unwrap(), + ); + let mut random = 0x4352_4142_2d53_4841u64; + let mut content = (0..4 * 1024 * 1024) + .map(|_| { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random as u8 + }) + .collect::>(); + let mut initial_recipe = None; + let mut initial_payload_inventory = None; + for version in 0..6 { + if version == 2 { + let offset = content.len() / 2; + content[offset] ^= 127; + } + if version == 3 { + let offset = content.len() - 17; + content[offset] ^= 63; + } + if version == 4 { + content.extend_from_slice(b"appended tail"); + } + if version == 5 { + content.truncate(content.len() - 123); + } + let native = native_first == (version % 2 == 0); + let expected_hash = *blake3::hash(&content).as_bytes(); + let recipe = if native { + std::fs::write(&path, &content).unwrap(); + let staged = rt + .block_on(crab_staging::stream::stage_file_streaming( + &path, + tmp.path(), + &staging, + crab_staging::stream::StreamStageProgress { + xorb_builder: Some( + crab_staging::stream::StreamStageXorbBuilder::new( + 1, + crab_xet::xorb::builder::XorbBuilder::new, + ), + ), + ..crab_staging::stream::StreamStageProgress::default() + }, + &tokio_util::sync::CancellationToken::new(), + )) + .unwrap(); + // A deferred native first version exercises changed-prefix replay. + if version > 0 { + staging.mark_batch_published(&staged.batch_id).unwrap(); + } + staged.recipe + } else { + let mut session = CleanSession::with_deps( + AppContext::default(), + Box::new(NoopFileIndexChecker), + Box::new(StagingChunkStager::new( + staging.clone(), + rt.handle().clone(), + )), + &[], + u64::MAX, + DEFAULT_CHUNK_BUFFER_CAP, + ); + let framed = frame_as_pktlines(&content); + let mut reader = crate::git::filter_process::PktLineReader::from_slice(&framed); + let pointer = session.clean_stream("model.bin", &mut reader).unwrap(); + assert_eq!(Pointer::parse(&pointer).unwrap().file_hash, expected_hash); + staging + .unpublished_local_recipe_for_path(Path::new("model.bin")) + .unwrap() + .unwrap() + }; + let payloads = std::fs::read_dir(staging.root().join("push-plans/payloads")) + .unwrap() + .flat_map(|directory| std::fs::read_dir(directory.unwrap().path()).unwrap()) + .map(|file| { + let file = file.unwrap(); + (file.file_name(), file.metadata().unwrap().len()) + }) + .collect::>(); + if version == 0 { + initial_recipe = Some(recipe.clone()); + initial_payload_inventory = Some(payloads.clone()); + } else if version == 1 { + assert!( + Some(&recipe) == initial_recipe.as_ref() + && Some(&payloads) == initial_payload_inventory.as_ref(), + "switching add paths without editing must reuse identical physical payloads" + ); + } else { + let original_bytes = initial_payload_inventory + .as_ref() + .unwrap() + .values() + .sum::(); + assert!( + payloads.values().sum::() < original_bytes + 2 * 1024 * 1024, + "small edits must retain reused payloads without another full-file copy" + ); + } + let mut hashes = Vec::new(); + let mut occurrence = 0; + while occurrence < recipe.chunk_count() { + let page = staging.recipe_page(&recipe, occurrence).unwrap(); + occurrence = page.next_occurrence(); + hashes.extend(page.chunks.into_iter().map(|chunk| chunk.chunk_hash)); + } + let reconstructed = rt + .block_on(staging.get_chunks_batch(&hashes)) + .unwrap() + .into_iter() + .flat_map(|(_, bytes)| bytes.to_vec()) + .collect::>(); + let health = staging.lifecycle_health().unwrap(); + let stats = staging.stats().unwrap(); + assert!( + reconstructed == content + && recipe.file_hash() == MerkleHash::from(expected_hash) + && stats.total_staged_bytes == 0 + && health.current_head_prepared_bytes < 2 * content.len() as u64, + "crossover version {version} must retain one shared payload representation" + ); + } + rt.block_on(Arc::try_unwrap(staging).ok().unwrap().close()) + .unwrap(); + } + } + + #[test] + fn failed_packet_input_preserves_prepared_version_and_releases_attempt_leases() { + let rt = tokio::runtime::Runtime::new().unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("model.bin"); + let original = vec![0x71; 3 * 1024 * 1024]; + std::fs::write(&path, &original).unwrap(); + let staging = Arc::new( + rt.block_on(StagingArea::open(tmp.path().join("staging"))) + .unwrap(), + ); + let prepared = rt + .block_on(crab_staging::stream::stage_file_streaming( + &path, + tmp.path(), + &staging, + crab_staging::stream::StreamStageProgress { + xorb_builder: Some(crab_staging::stream::StreamStageXorbBuilder::new( + 1, + crab_xet::xorb::builder::XorbBuilder::new, + )), + ..crab_staging::stream::StreamStageProgress::default() + }, + &tokio_util::sync::CancellationToken::new(), + )) + .unwrap(); + let before = staging.lifecycle_health().unwrap(); + let mut session = CleanSession::with_deps( + AppContext::default(), + Box::new(NoopFileIndexChecker), + Box::new(StagingChunkStager::new( + staging.clone(), + rt.handle().clone(), + )), + &[], + u64::MAX, + DEFAULT_CHUNK_BUFFER_CAP, + ); + let mut changed = original; + changed[0] ^= 0x7f; + let mut framed = frame_as_pktlines(&changed); + // A complete payload without Git's flush is not a complete request. + // The prepared prefix pin and reader attempt must both roll back. + framed.truncate(framed.len() - 4); + let mut reader = crate::git::filter_process::PktLineReader::from_slice(&framed); + let result = session.clean_stream("model.bin", &mut reader); + drop(session); + let after = staging.lifecycle_health().unwrap(); + assert!( + matches!(result, Err(CrabError::Io(ref error)) + if error.kind() == std::io::ErrorKind::UnexpectedEof) + && staging + .unpublished_local_recipe_for_path(Path::new("model.bin")) + .unwrap() + == Some(prepared.recipe) + && before.path_leases == after.path_leases + && before.open_batches_without_publication + == after.open_batches_without_publication + && staging.stats().unwrap().total_staged_bytes == 0, + "failed framing must retain the original owner and release only attempt ownership" + ); + rt.block_on(Arc::try_unwrap(staging).ok().unwrap().close()) + .unwrap(); + } + #[test] fn clean_stream_promotes_unpublished_prepared_recipe_without_segment_rewrite() { use crate::git::filter_process::PktLineReader; diff --git a/crab/src/git/filter_process.rs b/crab/src/git/filter_process.rs index bc9a67e74..1a4126720 100644 --- a/crab/src/git/filter_process.rs +++ b/crab/src/git/filter_process.rs @@ -2107,6 +2107,89 @@ impl PktLineReader { } } +/// Bounded byte view of one filter request, preserving the terminal flush. +pub(crate) struct PktLineByteStream<'a, R: Read> { + reader: &'a mut PktLineReader, + pending: Bytes, + finished: bool, + error: Option, + cancel: &'a tokio_util::sync::CancellationToken, +} + +impl PktLineReader { + /// Join a consumed pointer probe with the remaining request's packet bytes. + pub(crate) fn byte_stream<'a>( + &'a mut self, + prefix: Vec, + reached_flush: bool, + cancel: &'a tokio_util::sync::CancellationToken, + ) -> PktLineByteStream<'a, R> { + PktLineByteStream { + reader: self, + pending: prefix.into(), + finished: reached_flush, + error: None, + cancel, + } + } +} + +impl PktLineByteStream<'_, R> { + /// Recover the original framing error after the ingestion engine cleans up. + pub(crate) fn take_error(&mut self) -> Option { + self.error.take() + } +} + +impl tokio::io::AsyncRead for PktLineByteStream<'_, R> { + fn poll_read( + self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + output: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + let before = output.filled().len(); + while output.remaining() > 0 { + if this.error.is_some() { + // AsyncRead cannot report an error after appending bytes in + // the same poll. Return that prefix first, then fail the next + // read; the product boundary restores the original error. + if output.filled().len() == before { + return std::task::Poll::Ready(Err(io::Error::other( + "filter packet input failed", + ))); + } + break; + } + if this.cancel.is_cancelled() { + this.error = Some(CrabError::Cancelled); + continue; + } + if !this.pending.is_empty() { + let take = this.pending.len().min(output.remaining()); + output.put_slice(&this.pending[..take]); + this.pending = this.pending.slice(take..); + continue; + } + if this.finished { + break; + } + match this.reader.read_packet() { + Ok(Some(packet)) => { + let take = packet.len().min(output.remaining()); + output.put_slice(&packet[..take]); + if take < packet.len() { + this.pending = Bytes::copy_from_slice(&packet[take..]); + } + } + Ok(None) => this.finished = true, + Err(error) => this.error = Some(error), + } + } + std::task::Poll::Ready(Ok(())) + } +} + impl<'a> PktLineReader<&'a [u8]> { /// Construct a reader from an in-memory byte slice. /// @@ -3935,6 +4018,63 @@ size 1048576\n"; } } + #[tokio::test] + async fn packet_byte_stream_preserves_bytes_and_next_command_boundary() { + use tokio::io::AsyncReadExt; + + let payload = (0..2 * 1024 * 1024 + 17) + .map(|index| ((index * 41) ^ (index >> 8)) as u8) + .collect::>(); + for output_size in [1, PKT_LINE_MAX_BODY, crab_staging::stream::READ_BUF_SIZE] { + let prefix_len = 137; + let mut framed = Vec::new(); + for packet in payload[prefix_len..].chunks(4093) { + framed.extend(pkt_data(packet)); + } + framed.extend(pkt_flush()); + let boundary = framed.len() as u64; + framed.extend(pkt_text("command=clean")); + let mut cursor = std::io::Cursor::new(framed); + let cancel = tokio_util::sync::CancellationToken::new(); + let mut actual = Vec::new(); + { + let mut reader = PktLineReader::from_read(&mut cursor); + let mut stream = reader.byte_stream(payload[..prefix_len].to_vec(), false, &cancel); + let mut buffer = vec![0; output_size]; + loop { + let count = stream.read(&mut buffer).await.unwrap(); + if count == 0 { + break; + } + actual.extend_from_slice(&buffer[..count]); + } + assert!(stream.take_error().is_none()); + } + assert!(actual == payload && cursor.position() == boundary); + } + } + + #[tokio::test] + async fn packet_byte_stream_rejects_framing_failure_after_valid_prefix() { + use tokio::io::AsyncReadExt; + + for suffix in [b"zzzz".as_slice(), b"0008abc", b"0001"] { + let mut input = pkt_data(b"valid prefix"); + input.extend_from_slice(suffix); + let mut reader = PktLineReader::from_slice(&input); + let cancel = tokio_util::sync::CancellationToken::new(); + let mut stream = reader.byte_stream(Vec::new(), false, &cancel); + let mut actual = Vec::new(); + let result = stream.read_to_end(&mut actual).await; + let original = stream.take_error(); + assert!( + result.is_err() + && actual == b"valid prefix" + && matches!(original, Some(CrabError::Io(_))) + ); + } + } + #[test] fn pkt_line_reader_terminal_states_do_not_consume_another_packet() { for terminal in [b"0000", b"zzzz"] { diff --git a/crab/src/git/index.rs b/crab/src/git/index.rs index 5ad5828a0..4af4fed25 100644 --- a/crab/src/git/index.rs +++ b/crab/src/git/index.rs @@ -1,7 +1,11 @@ +use crab_xet::hash::MerkleHash; +use std::collections::HashSet; use std::path::Path; use std::process::Command; +use tokio_util::sync::CancellationToken; use crate::core::error::{CrabError, Result}; +use crab_staging::{PublicationIntentId, StagingArea, StagingAreaReadOnly}; /// Stage repository metadata paths and propagate Git failures. pub(crate) fn stage_paths(repo_root: &Path, paths: &[&str]) -> Result<()> { @@ -33,11 +37,409 @@ pub(crate) fn stage_paths(repo_root: &Path, paths: &[&str]) -> Result<()> { ))) } +/// Fingerprint the complete semantic state of one Git index path. +pub(crate) fn git_index_path_state(index: &gix_index::File, path: &bstr::BStr) -> String { + let mut hasher = blake3::Hasher::new(); + hasher.update(b"crab git index path state v1\0"); + if let Some(range) = index.entry_range(path) { + for entry in &index.entries()[range] { + hasher.update(&entry.flags.bits().to_le_bytes()); + hasher.update(&entry.mode.bits().to_le_bytes()); + hasher.update(entry.id.as_bytes()); + } + } + hasher.finalize().to_hex().to_string() +} + +/// Reconcile shared staging publications while holding the Git index lock. +pub(crate) async fn reconcile_publication_intents( + staging: &StagingAreaReadOnly, + repo_root: &Path, + cancel: &CancellationToken, +) -> Result<()> { + crate::core::error::check_cancelled(cancel)?; + if staging.unresolved_publication_intents()?.is_empty() { + return Ok(()); + } + let (_index_lock, index) = lock_publication_index(repo_root)?; + // Another shared recovery may have completed before lock acquisition. + // Read intents again under the lock rather than acting on a stale list. + let actions = publication_recovery_actions( + staging.unresolved_publication_intents()?, + &index, + repo_root, + cancel, + )?; + for action in actions { + crate::core::error::check_cancelled(cancel)?; + match action { + PublicationRecoveryAction::Publish(intent_id) => { + staging.publish_publication_intent(&intent_id)?; + } + PublicationRecoveryAction::Rollback(intent_id, history) => { + if !history.is_empty() { + staging.preserve_publication_history(&intent_id, &history, cancel)?; + } + staging.rollback_publication_intent(&intent_id).await?; + } + } + } + Ok(()) +} + +/// Reconcile exclusive staging publications while holding the Git index lock. +pub(crate) fn reconcile_publication_intents_writer( + staging: &StagingArea, + repo_root: &Path, + cancel: &CancellationToken, +) -> Result<()> { + crate::core::error::check_cancelled(cancel)?; + if staging.unresolved_publication_intents()?.is_empty() { + return Ok(()); + } + let (_index_lock, index) = lock_publication_index(repo_root)?; + // Another shared recovery may have completed before lock acquisition. + // Read intents again under the lock rather than acting on a stale list. + let actions = publication_recovery_actions( + staging.unresolved_publication_intents()?, + &index, + repo_root, + cancel, + )?; + for action in actions { + crate::core::error::check_cancelled(cancel)?; + match action { + PublicationRecoveryAction::Publish(intent_id) => { + staging.publish_publication_intent(&intent_id)?; + } + PublicationRecoveryAction::Rollback(intent_id, history) => { + if !history.is_empty() { + staging.preserve_publication_history(&intent_id, &history, cancel)?; + } + staging.rollback_publication_intent(&intent_id)?; + } + } + } + Ok(()) +} + +/// Reconcile complete external batches against a stable Git index during maintenance. +pub(crate) fn reconcile_delegated_publications( + staging: &StagingArea, + repo_root: &Path, + cancel: &tokio_util::sync::CancellationToken, +) -> Result { + crate::core::error::check_cancelled(cancel)?; + if staging.delegated_publications()?.is_empty() { + return Ok(0); + } + let (_index_lock, index) = lock_publication_index(repo_root)?; + let repo = gix::open(repo_root).map_err(|error| CrabError::Io(std::io::Error::other(error)))?; + let mut accepted = std::collections::HashMap::new(); + // Delegation freezes membership. Re-read under Git's lock and retain it + // through all promotions; a filter response alone cannot attest acceptance. + for lease in staging.delegated_publications()? { + crate::core::error::check_cancelled(cancel)?; + let matches = delegated_pointer_matches(&repo, &index, &lease)?; + accepted + .entry(lease.batch_id) + .and_modify(|all: &mut bool| *all &= matches) + .or_insert(matches); + } + let mut published = 0; + for (batch_id, matches) in accepted { + crate::core::error::check_cancelled(cancel)?; + if matches { + staging.mark_batch_published(&batch_id)?; + published += 1; + } + } + // Missing/different entries may be --defer work or older unpushed history. + // Releasing them needs additional evidence; current index absence is insufficient. + Ok(published) +} + +fn delegated_pointer_matches( + repo: &gix::Repository, + index: &gix_index::File, + lease: &crab_staging::DelegatedPublication, +) -> Result { + use bstr::ByteSlice; + use crab_types::pointer::{MAX_POINTER_SIZE, Pointer, is_supported_version_line}; + use gix_index::entry::{Mode, Stage}; + + let Some(entry) = + index.entry_by_path_and_stage(lease.path_bytes.as_bstr(), Stage::Unconflicted) + else { + return Ok(false); + }; + if !matches!(entry.mode, Mode::FILE | Mode::FILE_EXECUTABLE) { + return Ok(false); + } + // A user can replace a staged pointer with a raw blob. Check its header + // before materializing it so maintenance cannot allocate a 10-GiB file. + let header = repo + .find_header(entry.id) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))?; + if header.kind() != gix_object::Kind::Blob { + return Err(CrabError::CorruptObject { + path: entry.id.to_string(), + reason: "Git index file does not reference a blob".to_owned(), + }); + } + if header.size() > MAX_POINTER_SIZE as u64 { + return Ok(false); + } + let blob = repo + .find_blob(entry.id) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))?; + let actual = gix_object::compute_hash(repo.object_hash(), gix_object::Kind::Blob, &blob.data) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))?; + if actual != entry.id { + return Err(CrabError::CorruptObject { + path: entry.id.to_string(), + reason: "Git index blob hash mismatch".to_owned(), + }); + } + let first = blob + .data + .split(|byte| *byte == b'\n') + .next() + .unwrap_or_default(); + if !std::str::from_utf8(first).is_ok_and(is_supported_version_line) { + return Ok(false); + } + let pointer = + Pointer::parse(&blob.data).map_err(|error| CrabError::Io(std::io::Error::other(error)))?; + Ok(pointer.file_hash == <[u8; 32]>::from(lease.file_hash) && pointer.size == lease.size) +} + +enum PublicationRecoveryAction { + Publish(PublicationIntentId), + Rollback(PublicationIntentId, Vec), +} + +fn lock_publication_index(repo_root: &Path) -> Result<(gix_lock::Marker, gix_index::File)> { + use gix_index::{File, decode}; + + let index_path = + crate::git::worktree::WorktreeContext::resolve_from_path(repo_root)?.index_path(); + // Holding Git's lock through staging mutation prevents retirement while + // another writer is still installing the pointer. Fail immediately to + // avoid Git-index/staging lock inversion with an active filter child. + let index_lock = gix_lock::Marker::acquire_to_hold_resource( + &index_path, + gix_lock::acquire::Fail::Immediately, + None, + ) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))?; + let index = File::at_or_default( + &index_path, + gix_hash::Kind::Sha1, + false, + decode::Options::default(), + ) + .map_err(|error| { + CrabError::Internal(format!( + "failed to read Git index for add publication recovery: {error}" + )) + })?; + + Ok((index_lock, index)) +} + +fn publication_recovery_actions( + intents: Vec, + index: &gix_index::File, + repo_root: &Path, + cancel: &CancellationToken, +) -> Result> { + use bstr::ByteSlice; + use gix_index::entry; + + let mut actions = Vec::with_capacity(intents.len()); + let mut rollbacks = Vec::new(); + for intent in intents { + let new_matches = intent + .entries + .iter() + .filter(|expected| { + index + .entry_by_path_and_stage( + expected.path_bytes.as_bstr(), + entry::Stage::Unconflicted, + ) + .is_some_and(|actual| actual.id.to_string() == expected.expected_pointer_oid) + }) + .count(); + let old_matches = intent + .entries + .iter() + .filter(|expected| { + git_index_path_state(index, expected.path_bytes.as_bstr()) + == expected.previous_index_state + }) + .count(); + if new_matches == intent.entries.len() { + actions.push(PublicationRecoveryAction::Publish(intent.intent_id)); + } else if old_matches == intent.entries.len() { + rollbacks.push(intent); + } else { + return Err(CrabError::StagingCorrupt(format!( + "add publication {} has an ambiguous Git index state ({new_matches}/{} new paths, {old_matches}/{} prior paths); restore one complete index state, then retry", + intent.intent_id.as_str(), + intent.entries.len(), + intent.entries.len(), + ))); + } + } + let wanted = rollbacks + .iter() + .flat_map(|intent| intent.entries.iter()) + .map(|entry| entry.expected_pointer_oid.clone()) + .collect::>(); + let committed = committed_pointer_oids(repo_root, &wanted, cancel)?; + for intent in rollbacks { + let history = intent + .entries + .iter() + .filter(|entry| committed.contains(&entry.expected_pointer_oid)) + .map(|entry| entry.recipe_hash) + .collect(); + actions.push(PublicationRecoveryAction::Rollback( + intent.intent_id, + history, + )); + } + Ok(actions) +} + +fn committed_pointer_oids( + repo_root: &Path, + wanted: &HashSet, + cancel: &CancellationToken, +) -> Result> { + use std::io::{BufRead, Read}; + if wanted.is_empty() { + return Ok(HashSet::new()); + } + // Only crash recovery needs this local history walk. Stream OIDs from refs + // and reflogs: an old index alone cannot authorize deleting committed bytes. + let mut command = Command::new("git"); + command + .current_dir(repo_root) + .env("GIT_NO_LAZY_FETCH", "1") + .env("GIT_ALLOW_PROTOCOL", "") + .args([ + "--no-replace-objects", + "rev-list", + "--all", + "--reflog", + "--objects", + "--no-object-names", + "--use-bitmap-index", + ]); + for name in super::process::GIT_ENV_REMOVALS { + command.env_remove(name); + } + let output = super::process::run( + command, + cancel, + None:: Result<()>>, + |stdout| { + let mut found = HashSet::new(); + let mut reader = std::io::BufReader::new(stdout); + let mut line = String::with_capacity(41); + loop { + crate::core::error::check_cancelled(cancel)?; + line.clear(); + let count = (&mut reader).take(42).read_line(&mut line)?; + if count == 0 { + break; + } + let oid = line + .strip_suffix('\n') + .filter(|oid| oid.len() == 40 && oid.bytes().all(|b| b.is_ascii_hexdigit())) + .ok_or_else(|| { + CrabError::Internal( + "Git recovery history scan returned an invalid object ID".to_owned(), + ) + })?; + if wanted.contains(oid) { + found.insert(oid.to_owned()); + } + } + Ok(found) + }, + )?; + // Git can exit zero while skipping unavailable reflog tips. A diagnostic + // means this walk cannot attest absence and authorize payload deletion. + if !output.status.success() || !output.stderr.is_empty() { + return Err(CrabError::Internal(format!( + "Git recovery history scan did not complete cleanly ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + Ok(output.stdout) +} + #[cfg(test)] #[allow(clippy::unwrap_used, reason = "test assertions")] mod tests { use super::*; + #[test] + fn recovery_history_includes_reflogs_and_rejects_incomplete_walks() { + let _env = crate::test::git_repo::CleanGitEnvGuard::new(); + for revision in ["HEAD^{tree}", "HEAD"] { + let temp = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .args(args) + .current_dir(temp.path()) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output.stderr); + String::from_utf8(output.stdout).unwrap().trim().to_owned() + }; + git(&["init", "-q", "-b", "main"]); + git(&["config", "user.name", "Crab Test"]); + git(&["config", "user.email", "crab@example.invalid"]); + std::fs::write(temp.path().join("file"), b"initial").unwrap(); + git(&["add", "file"]); + git(&["commit", "-qm", "initial"]); + let initial = git(&["rev-parse", "HEAD"]); + std::fs::write(temp.path().join("file"), b"committed pointer").unwrap(); + git(&["add", "file"]); + git(&["commit", "-qm", "pointer"]); + let pointer = git(&["rev-parse", "HEAD:file"]); + let missing = git(&["rev-parse", revision]); + git(&["reset", "--hard", &initial]); + let wanted = HashSet::from([pointer]); + let cancel = CancellationToken::new(); + assert_eq!( + committed_pointer_oids(temp.path(), &wanted, &cancel).unwrap(), + wanted + ); + cancel.cancel(); + assert!(matches!( + committed_pointer_oids(temp.path(), &wanted, &cancel), + Err(CrabError::Cancelled) + )); + std::fs::remove_file( + temp.path() + .join(".git/objects") + .join(&missing[..2]) + .join(&missing[2..]), + ) + .unwrap(); + assert!( + committed_pointer_oids(temp.path(), &wanted, &CancellationToken::new()).is_err() + ); + } + } + #[test] fn metadata_staging_propagates_git_index_failure() { let dir = tempfile::tempdir().unwrap(); @@ -63,4 +465,266 @@ mod tests { .contains("git add repository metadata failed") ); } + #[tokio::test(flavor = "multi_thread")] + async fn delegated_recovery_waits_for_git_acceptance_and_preserves_other_versions() { + use crab_types::pointer::Pointer; + let tmp = tempfile::tempdir().unwrap(); + assert!( + Command::new("git") + .args(["init", "-q"]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let staging = StagingArea::open(tmp.path().join(".crab/staging")) + .await + .unwrap(); + let mut versions = Vec::new(); + for (path, data) in [ + ("accepted", b"first".as_slice()), + ("different", b"second"), + ("deferred", b"third"), + ] { + let staged = crab_staging::stream::stage_reader_streaming( + &mut std::io::Cursor::new(data), + Path::new(path), + &staging, + crab_staging::stream::StreamStageProgress::default(), + &tokio_util::sync::CancellationToken::new(), + ) + .await + .unwrap(); + staging + .delegate_publication(&[(&staged.batch_id, Path::new(path))]) + .unwrap(); + let mut pointer = Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + if path == "different" { + pointer.file_hash[0] ^= 1; + } + if path != "deferred" { + std::fs::write(tmp.path().join(path), pointer.serialize()).unwrap(); + } + versions.push(staged.recipe); + } + assert!( + Command::new("git") + .args(["add", "--", "accepted", "different"]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let lock = tmp.path().join(".git/index.lock"); + std::fs::write(&lock, b"another Git writer").unwrap(); + let blocked = reconcile_delegated_publications( + &staging, + tmp.path(), + &tokio_util::sync::CancellationToken::new(), + ); + assert!(blocked.is_err() && staging.delegated_publications().unwrap().len() == 3); + std::fs::remove_file(lock).unwrap(); + let promoted = reconcile_delegated_publications( + &staging, + tmp.path(), + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(); + assert!( + promoted == 1 + && staging.delegated_publications().unwrap().len() == 2 + && staging + .published_recipe_for_file(&versions[0].file_hash()) + .unwrap() + == Some(versions[0].clone()) + && staging + .published_recipe_for_file(&versions[1].file_hash()) + .unwrap() + .is_none() + && staging + .published_recipe_for_file(&versions[2].file_hash()) + .unwrap() + .is_none() + ); + staging.close().await.unwrap(); + } + #[tokio::test(flavor = "multi_thread")] + async fn delegated_recovery_requires_small_blob_integrity_and_skips_oversized_body() { + use std::io::Write; + for oversized in [false, true] { + let tmp = tempfile::tempdir().unwrap(); + assert!( + Command::new("git") + .args(["init", "-q"]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let staging = StagingArea::open(tmp.path().join(".crab/staging")) + .await + .unwrap(); + let staged = crab_staging::stream::stage_reader_streaming( + &mut std::io::Cursor::new(b"prepared bytes"), + Path::new("oversized"), + &staging, + crab_staging::stream::StreamStageProgress::default(), + &tokio_util::sync::CancellationToken::new(), + ) + .await + .unwrap(); + staging + .delegate_publication(&[(&staged.batch_id, Path::new("oversized"))]) + .unwrap(); + // Header inspection is enough to reject this as a pointer. Loading + // the deliberately incomplete 10-GiB body would fail or allocate it. + let oid = "d".repeat(40); + let object_dir = tmp.path().join(".git/objects").join(&oid[..2]); + std::fs::create_dir_all(&object_dir).unwrap(); + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + if oversized { + encoder + .write_all(b"blob 10737418240\0header probe padding") + .unwrap(); + } else { + let bytes = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + } + .serialize(); + encoder + .write_all(format!("blob {}\0", bytes.len()).as_bytes()) + .unwrap(); + encoder.write_all(&bytes).unwrap(); + } + std::fs::write(object_dir.join(&oid[2..]), encoder.finish().unwrap()).unwrap(); + assert!( + Command::new("git") + .args([ + "update-index", + "--add", + "--cacheinfo", + &format!("100644,{oid},oversized") + ]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let promoted = reconcile_delegated_publications( + &staging, + tmp.path(), + &tokio_util::sync::CancellationToken::new(), + ); + assert!( + (if oversized { + matches!(promoted, Ok(0)) + } else { + matches!(promoted, Err(CrabError::CorruptObject { .. })) + }) && staging.delegated_publications().unwrap().len() == 1 + && !tmp.path().join(".git/index.lock").exists() + ); + staging.close().await.unwrap(); + } + } + #[tokio::test(flavor = "multi_thread")] + async fn delegated_recovery_requires_every_batch_path_and_keeps_shared_native_owner() { + use crab_types::pointer::Pointer; + let tmp = tempfile::tempdir().unwrap(); + assert!( + Command::new("git") + .args(["init", "-q"]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let staging = StagingArea::open(tmp.path().join(".crab/staging")) + .await + .unwrap(); + let mut versions = Vec::new(); + for path in ["first", "second"] { + versions.push( + crab_staging::stream::stage_reader_streaming( + &mut std::io::Cursor::new(b"shared payload"), + Path::new(path), + &staging, + crab_staging::stream::StreamStageProgress::default(), + &tokio_util::sync::CancellationToken::new(), + ) + .await + .unwrap(), + ); + } + let first = &versions[0]; + let second = &versions[1]; + staging + .record_verified_recipe_lease(&first.batch_id, Path::new("second"), &second.recipe) + .unwrap(); + staging + .delegate_publication(&[ + (&first.batch_id, Path::new("first")), + (&first.batch_id, Path::new("second")), + ]) + .unwrap(); + let pointer = Pointer { + file_hash: first.file_hash, + size: first.size, + shard_hint: None, + } + .serialize(); + std::fs::write(tmp.path().join("first"), &pointer).unwrap(); + assert!( + Command::new("git") + .args(["add", "--", "first"]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let partial = reconcile_delegated_publications( + &staging, + tmp.path(), + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(); + assert!(partial == 0 && staging.delegated_publications().unwrap().len() == 2); + std::fs::write(tmp.path().join("second"), pointer).unwrap(); + assert!( + Command::new("git") + .args(["add", "--", "second"]) + .current_dir(tmp.path()) + .status() + .unwrap() + .success() + ); + let complete = reconcile_delegated_publications( + &staging, + tmp.path(), + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(); + let health = staging.lifecycle_health().unwrap(); + assert!( + complete == 1 + && health.path_heads == 2 + && health.path_leases == 3 + && health.open_batches_without_publication == 1 + && health.payloads == 1 + ); + staging.rollback_batch(&second.batch_id).unwrap(); + assert_eq!( + staging + .published_recipe_for_file(&first.recipe.file_hash()) + .unwrap(), + Some(first.recipe.clone()) + ); + staging.close().await.unwrap(); + } } diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index 7ad33ce68..d9e7b97fe 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -9713,9 +9713,10 @@ impl PushPipeline { if !staging.unresolved_publication_intents()?.is_empty() { let worktree = crate::git::worktree::WorktreeContext::resolve()?; - crate::cmd::add::reconcile_publication_intents( + crate::git::index::reconcile_publication_intents( staging, &worktree.current_worktree_root, + &self.cancel, ) .await?; } @@ -9754,8 +9755,10 @@ impl PushPipeline { .iter() .map(|(file_hash, _)| *file_hash) .collect::>(); - let published_recipes = Arc::new(staging.published_recipes_for_files(&file_hashes)?); - let recipe_hashes = published_recipes + let selected_recipes = + super::push_staging::recipes_for_selected_pointers(staging, &file_hashes)?; + let selected_recipes = Arc::new(selected_recipes); + let recipe_hashes = selected_recipes .values() .filter_map(|recipe| { recipe @@ -9772,10 +9775,10 @@ impl PushPipeline { }; let verify_results = futures_util::stream::iter(unique_specs.into_iter().map( |(file_hash, pointer_size)| { - let published_recipes = Arc::clone(&published_recipes); + let selected_recipes = Arc::clone(&selected_recipes); let prepared_xorbs = prepared_xorbs.clone(); async move { - let recipe = published_recipes + let recipe = selected_recipes .get(&file_hash) .cloned() .ok_or_else(|| { @@ -9787,7 +9790,7 @@ impl PushPipeline { let Some(recipe) = recipe else { debug!( file_hash = %file_hash.hex(), - "step 2: no published staging recipe for pointer" + "step 2: no eligible staging recipe for pointer" ); return Ok(( file_hash, @@ -31139,6 +31142,78 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn lookup_staging_requires_delegation_before_adopting_open_prepared_recipe() { + use crate::git::walk::PointerBlob; + use crab_staging::{StagingArea, StagingAreaReadOnly}; + + for delegated in [false, true] { + let tmp = tempfile::tempdir().unwrap(); + let source = tmp.path().join("model.bin"); + std::fs::write(&source, vec![0x71; 1024 * 1024]).unwrap(); + let root = tmp.path().join("staging"); + let staging = StagingArea::open(root.clone()).await.unwrap(); + let staged = crab_staging::stream::stage_file_streaming( + &source, + tmp.path(), + &staging, + crab_staging::stream::StreamStageProgress { + xorb_builder: Some(crab_staging::stream::StreamStageXorbBuilder::new( + 1, + crab_xet::xorb::builder::XorbBuilder::new, + )), + ..crab_staging::stream::StreamStageProgress::default() + }, + &CancellationToken::new(), + ) + .await + .unwrap(); + if delegated { + staging + .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + } + staging.close().await.unwrap(); + let staging = Arc::new(StagingAreaReadOnly::open(root).await.unwrap()); + let pipeline = PushPipeline::new( + PushConfig::default(), + vec![], + None, + None, + Some(Arc::clone(&staging)), + "delegated-recipe".to_owned(), + test_router("delegated-recipe"), + None, + CancellationToken::new(), + None, + ); + pipeline.pointers.lock().await.push(PointerBlob { + oid: [0; 20], + file_hash: staged.file_hash, + size: staged.size, + }); + let lookup = pipeline.lookup_staging().await; + if !delegated { + assert!(matches!( + lookup, + Err(CrabError::PointerMissingStaging { .. }) + )); + continue; + } + lookup.unwrap(); + pipeline.classify_chunks().await.unwrap(); + pipeline.pack_xorbs().await.unwrap(); + assert!( + !pipeline.xorbs.lock().await.is_empty() + && staging + .published_recipe_for_file(&staged.recipe.file_hash()) + .unwrap() + .is_none(), + "selected handoff supplies prepared payloads without claiming index publication" + ); + } + } + #[tokio::test(flavor = "multi_thread")] async fn add_time_push_plan_adopts_prepared_xorbs_without_staging_reread() { use crate::git::walk::PointerBlob; diff --git a/crab/src/git/push_staging.rs b/crab/src/git/push_staging.rs index 80ddcd434..1aed2d940 100644 --- a/crab/src/git/push_staging.rs +++ b/crab/src/git/push_staging.rs @@ -1,11 +1,13 @@ //! Staging availability for native push admission. +use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use crab_staging::{StagingAreaReadOnly, StagingError}; +use crab_staging::{StagingAreaReadOnly, StagingError, recipe::FileRecipe}; +use crab_xet::hash::MerkleHash; -use crate::core::error::Result; +use crate::core::error::{CrabError, Result}; /// A push reader or the precise reason no reader is available. #[derive(Clone)] @@ -44,6 +46,35 @@ impl PushStaging { } } +/// Select published or delegated recipes for verified pointers discovered in Git. +pub(crate) fn recipes_for_selected_pointers( + staging: &StagingAreaReadOnly, + file_hashes: &[MerkleHash], +) -> Result>> { + let mut selected_recipes = staging.published_recipes_for_files(file_hashes)?; + let delegated_recipes = staging.external_recipes_for_files(file_hashes)?; + // Pointer discovery supplies Git reachability evidence. Delegated + // filter/defer leases are eligible only for these selected identities; + // unrelated or active native leases never become push-visible here. + for (file_hash, delegated) in delegated_recipes { + let slot = selected_recipes.get_mut(&file_hash).ok_or_else(|| { + CrabError::StagingCorrupt( + "delegated recipe lookup returned an unrequested identity".to_owned(), + ) + })?; + match (slot.as_ref(), delegated) { + (Some(published), Some(delegated)) if *published != delegated => { + return Err(CrabError::StagingCorrupt( + "published and delegated recipes disagree for one file identity".to_owned(), + )); + } + (None, Some(delegated)) => *slot = Some(delegated), + _ => {} + } + } + Ok(selected_recipes) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 251ded49e..3e13c2392 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -117,6 +117,15 @@ pub(crate) async fn prepare_delta( } })?; + let file_hashes = unresolved.iter().map(|(hash, _)| *hash).collect::>(); + let recipes = super::push_staging::recipes_for_selected_pointers(staging, &file_hashes)?; + let recipe_hashes = recipes + .values() + .flatten() + .map(|recipe| MerkleHash::from(recipe.hash())) + .collect::>(); + let mut prepared_xorbs = staging.prepared_xorbs_for_recipes(&recipe_hashes)?; + let mut xorb_entries = HashMap::::new(); let mut placements = ChunkPlacementMap::new(); @@ -132,25 +141,41 @@ pub(crate) async fn prepare_delta( for (file_hash, file_size) in unresolved { check_cancelled(cancel)?; - let chunks = staging.chunks_for_file_with_sizes(&file_hash)?; - if chunks.is_empty() && file_size != 0 { + let recipe = recipes + .get(&file_hash) + .and_then(Option::as_ref) + .ok_or_else(|| CrabError::PointerMissingStaging { + total: file_hashes.len(), + missing: 1, + example_file_hash: file_hash.hex(), + example_size: file_size, + })?; + if recipe.file_size() != file_size { return Err(CrabError::StagingCorrupt(format!( - "pointer file {} has no staged chunk recipe", - file_hash.hex() + "staged recipe for file {} has {} bytes, pointer size {file_size}", + file_hash.hex(), + recipe.file_size(), ))); } - let plan = staging.load_file_push_plan(&file_hash).await?; - if chunks.is_empty() || plan.is_none() { - staging - .verify_file_reconstruction_from_chunks(&file_hash, file_size, &chunks) - .await?; + let mut chunks = Vec::new(); + let mut next = 0; + while next < recipe.chunk_count() { + let page = staging.recipe_page(recipe, next)?; + next = page.next_occurrence(); + chunks.extend( + page.chunks + .iter() + .map(|chunk| (chunk.chunk_hash, chunk.len)), + ); } + let mut plan = crab_staging::push_plan::FilePushPlan::new_verified_recipe(recipe); + plan.prepared_xorbs = prepared_xorbs + .remove(&MerkleHash::from(recipe.hash())) + .unwrap_or_default(); - // An indexed plan is rebuilt from the caller-verified canonical recipe. - // Reconstructing the whole file here would duplicate add's stable-stream - // proof; payloads newly consumed below still receive hash verification. - if plan.is_some() - && let Some(recipe) = staging.published_recipe_for_file(&file_hash)? + // The selected immutable recipe carries the ingestion verification. + // Inspect its remote authority without reconstructing the entire file; + // each local or remote xorb consumed below is independently verified. { let expected_sizes = chunks.iter().copied().collect::>(); let mut next_occurrence = 0_u64; @@ -164,7 +189,7 @@ pub(crate) async fn prepare_delta( })? .min(recipe.chunk_count()); for (chunk_hash, candidate) in - staging.recipe_remote_chunk_range(&recipe, next_occurrence, page_end)? + staging.recipe_remote_chunk_range(recipe, next_occurrence, page_end)? { let expected_size = expected_sizes.get(&chunk_hash).ok_or_else(|| { CrabError::StagingCorrupt(format!( @@ -289,43 +314,41 @@ pub(crate) async fn prepare_delta( let mut candidate_chunks = placements.keys().copied().collect::>(); for (_, _, _, plan) in &prepared_files { - if let Some(plan) = plan { - for planned in &plan.prepared_xorbs { - let planned_hash = planned.hash()?; - if let Some(index) = candidate_indices.get(&planned_hash).copied() { - let existing = prepared_candidates.get(index).ok_or_else(|| { - CrabError::Internal("prepared xorb candidate index is invalid".to_owned()) - })?; - if !planned_xorbs_match(existing, planned) { - return Err(CrabError::StagingCorrupt(format!( - "prepared xorb {} has conflicting plans", - planned_hash.hex() - ))); - } - continue; - } - if xorb_entries.contains_key(&planned_hash) { - continue; - } - let planned_placements = planned - .placements - .iter() - .map(crab_staging::push_plan::PlannedPlacement::to_placement) - .collect::>>()?; - if planned_placements - .iter() - .all(|placement| candidate_chunks.contains(&placement.chunk_hash)) - { - continue; + for planned in &plan.prepared_xorbs { + let planned_hash = planned.hash()?; + if let Some(index) = candidate_indices.get(&planned_hash).copied() { + let existing = prepared_candidates.get(index).ok_or_else(|| { + CrabError::Internal("prepared xorb candidate index is invalid".to_owned()) + })?; + if !planned_xorbs_match(existing, planned) { + return Err(CrabError::StagingCorrupt(format!( + "prepared xorb {} has conflicting plans", + planned_hash.hex() + ))); } - candidate_chunks.extend( - planned_placements - .iter() - .map(|placement| placement.chunk_hash), - ); - candidate_indices.insert(planned_hash, prepared_candidates.len()); - prepared_candidates.push(planned.clone()); + continue; } + if xorb_entries.contains_key(&planned_hash) { + continue; + } + let planned_placements = planned + .placements + .iter() + .map(crab_staging::push_plan::PlannedPlacement::to_placement) + .collect::>>()?; + if planned_placements + .iter() + .all(|placement| candidate_chunks.contains(&placement.chunk_hash)) + { + continue; + } + candidate_chunks.extend( + planned_placements + .iter() + .map(|placement| placement.chunk_hash), + ); + candidate_indices.insert(planned_hash, prepared_candidates.len()); + prepared_candidates.push(planned.clone()); } } @@ -1212,6 +1235,15 @@ mod tests { ) .await .unwrap(); + staging.flush_pending().await.unwrap(); + let recipe = crab_staging::recipe::FileRecipe::from_staged_chunks( + crab_staging::recipe::ChunkingPolicyId::XetGearV1_64KiB, + MerkleHash::from(file_hash), + 18, + &[(chunk_hash, 9), (chunk_hash, 9)], + ) + .unwrap(); + staging.publish_verified_history_recipe(&recipe).unwrap(); staging.close().await.unwrap(); let staging = Arc::new( StagingAreaReadOnly::open(root.path().to_owned()) @@ -1248,6 +1280,94 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn capsule_publication_requires_handoff_and_preserves_prepared_encoding() { + use crab_staging::stream::{ + StreamStageProgress, StreamStageXorbBuilder, stage_reader_streaming, + }; + for (delegated, wrong_size) in [(false, false), (true, true), (true, false)] { + let root = tempfile::tempdir().unwrap(); + let staging = crab_staging::StagingArea::open(root.path().to_owned()) + .await + .unwrap(); + let bytes = vec![0x71; 2 * 1024 * 1024]; + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(&bytes), + Path::new("model.bin"), + &staging, + StreamStageProgress { + xorb_builder: Some(StreamStageXorbBuilder::new(1, || { + let policy = Arc::new(FixedCompression::new(CompressionScheme::None)) + as Arc; + XorbBuilder::with_policy(policy) + })), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await + .unwrap(); + if delegated { + staging + .delegate_publication(&[(&staged.batch_id, Path::new("model.bin"))]) + .unwrap(); + } + staging.close().await.unwrap(); + let staging = Arc::new( + StagingAreaReadOnly::open(root.path().to_owned()) + .await + .unwrap(), + ); + let expected = staging + .prepared_xorbs_for_recipes(&[MerkleHash::from(staged.recipe.hash())]) + .unwrap(); + let expected = &expected[&MerkleHash::from(staged.recipe.hash())]; + assert_eq!(expected.len(), 1); + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = crab_storage::StoreLayout::new(store, "repo".to_owned()); + let base = crab_write::capsule_protocol::initialize( + &layout, + &"1".repeat(64), + "refs/heads/main", + ) + .await + .unwrap(); + let result = prepare_delta( + &layout, + &base, + &[Pointer { + file_hash: staged.file_hash, + size: staged.size + u64::from(wrong_size), + shard_hint: None, + }], + Some(&staging), + None, + None, + 1, + false, + &CancellationToken::new(), + ) + .await; + if !delegated || wrong_size { + assert!( + result.is_err(), + "unaccepted or inconsistent recipe became publishable" + ); + continue; + } + let (delta, stats) = result.unwrap(); + let actual = &delta.xorbs()[&expected[0].hash]; + assert_eq!( + ( + stats.xorbs_uploaded, + actual.body_digest(), + actual.encoded_size() + ), + (1, expected[0].payload_hash.as_str(), expected[0].bytes), + ); + } + } + #[derive(Default)] struct UploadObserver { active: std::sync::atomic::AtomicUsize, diff --git a/crates/crab-auth/README.md b/crates/crab-auth/README.md index 56681e7e0..034f1643d 100644 --- a/crates/crab-auth/README.md +++ b/crates/crab-auth/README.md @@ -18,6 +18,11 @@ The crate also owns the wire contracts used by protected pushes and managed transfer grants. A caller can therefore validate an auth response before any credential is handed to an object-store builder. +A successful protected finalize response requires `status: updated`. If it +includes coordinator metadata, the complete metadata must attest `Committed` +or `Materialized`; pending, uploaded-only and aborted states are rejected before +a caller can retire its local staging payloads. + ## Architecture The main flow is: diff --git a/crates/crab-auth/src/protected_push.rs b/crates/crab-auth/src/protected_push.rs index b796f6419..96e855346 100644 --- a/crates/crab-auth/src/protected_push.rs +++ b/crates/crab-auth/src/protected_push.rs @@ -162,6 +162,17 @@ pub fn validate_push_finalize_response(response: &PushFinalizeResponse) -> Resul return invalid_finalize_response("partial active-active commit metadata"); } + // A successful finalize authorizes releasing the client's local payloads. + // Pending or aborted coordinator state cannot attest remote publication. + if let Some(state) = response.commit_state + && !matches!( + state, + PushTransactionState::Committed | PushTransactionState::Materialized + ) + { + return invalid_finalize_response("active-active push has not committed"); + } + if let Some(operation_id) = response.operation_id.as_deref() && operation_id.trim().is_empty() { @@ -560,6 +571,39 @@ mod tests { assert!(err.to_string().contains("partial active-active")); } + #[test] + fn finalize_success_requires_committed_coordinator_state() { + for state in [ + PushTransactionState::Pending, + PushTransactionState::ObjectsUploaded, + PushTransactionState::Committed, + PushTransactionState::Materialized, + PushTransactionState::Aborted, + ] { + let outcome = CommitOutcome { + operation_id: "operation".to_owned(), + coordinator_epoch: 1, + writer: "writer".to_owned(), + region: "region".to_owned(), + manifest_generation: 1, + commit_sequence: 1, + state, + }; + let response = PushFinalizeResponse::updated_with_commit_outcome( + vec![valid_ref_update()], + Some(&outcome), + ); + assert_eq!( + validate_push_finalize_response(&response).is_ok(), + matches!( + state, + PushTransactionState::Committed | PushTransactionState::Materialized + ), + "state={state:?}", + ); + } + } + #[test] fn validate_push_finalize_response_rejects_unexpected_status() { let response = PushFinalizeResponse { diff --git a/crates/crab-coordination/src/cosmosdb_coordinator.rs b/crates/crab-coordination/src/cosmosdb_coordinator.rs index 77ef3bb5f..d044c0005 100644 --- a/crates/crab-coordination/src/cosmosdb_coordinator.rs +++ b/crates/crab-coordination/src/cosmosdb_coordinator.rs @@ -69,10 +69,6 @@ pub struct CosmosDbCreateCoordinatorAccount { } /// Minimal Cosmos DB control-plane client needed by Crab-owned coordinator setup. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait CosmosDbCoordinatorControlPlaneClient { async fn describe_account( diff --git a/crates/crab-coordination/src/dynamodb_coordinator.rs b/crates/crab-coordination/src/dynamodb_coordinator.rs index 0d0db9309..d67564498 100644 --- a/crates/crab-coordination/src/dynamodb_coordinator.rs +++ b/crates/crab-coordination/src/dynamodb_coordinator.rs @@ -85,10 +85,6 @@ pub type DynamoDbRepoState = CoordinatorRepoState; pub type DynamoDbTransactionRecord = CoordinatorTransactionRecord; /// DynamoDB data-plane client for one serialized repo authority item. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait DynamoDbWriteCoordinatorClient { async fn read_repo_state( diff --git a/crates/crab-coordination/src/spanner_coordinator.rs b/crates/crab-coordination/src/spanner_coordinator.rs index f88720d12..cb2e7173a 100644 --- a/crates/crab-coordination/src/spanner_coordinator.rs +++ b/crates/crab-coordination/src/spanner_coordinator.rs @@ -64,10 +64,6 @@ pub struct SpannerCreateCoordinator { } /// Minimal Spanner control-plane client needed by Crab-owned coordinator setup. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait SpannerCoordinatorControlPlaneClient { async fn describe_instance( diff --git a/crates/crab-coordination/src/write_coordinator.rs b/crates/crab-coordination/src/write_coordinator.rs index a7b9172c7..12c9bcb11 100644 --- a/crates/crab-coordination/src/write_coordinator.rs +++ b/crates/crab-coordination/src/write_coordinator.rs @@ -373,10 +373,6 @@ pub struct CoordinatorControlPlaneStatus { } /// Management backend for a linearizable active-active coordinator. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait CoordinatorControlPlaneBackend: Send + Sync { fn provider(&self) -> ManagedCoordinatorProvider; @@ -605,10 +601,6 @@ fn coordinator_base_action(action: &str) -> &str { } /// Versioned CAS storage contract for managed coordinator data planes. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait VersionedCoordinatorStateStore: Send + Sync { async fn read_repo_state( @@ -653,10 +645,6 @@ where } /// Linearizable authority for active-active repository writes. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait WriteCoordinator: Send + Sync { async fn health(&self) -> Result; diff --git a/crates/crab-staging/AGENTS.md b/crates/crab-staging/AGENTS.md index f36b7e48d..3818b2299 100644 --- a/crates/crab-staging/AGENTS.md +++ b/crates/crab-staging/AGENTS.md @@ -33,6 +33,15 @@ Trace one path: `crab/src/cmd/add.rs` (`close_staging_before_indexing`) → Source: `crates/crab-staging/src/lib.rs`. - File-version reads must include all ordered chunk occurrences, even when chunk bytes deduplicate; inspect recipe consumers before changing index membership. Source: `crates/crab-staging/src/index.rs`. + Prepared-only versions remain enumerable through open/delegated leases and + push-snapshot pins. Chunk enumeration must not imply Git publication; push + admission uses its separate published/delegated selection. + +- Segment payload lifetime follows canonical inventory, including recording and + sealed recipe owners. Compaction transfers inventory and occurrence locators + together; reads capture handles before releasing locator ownership. Layout v3 + upgrades released v1 locators transactionally before admission. + Sources: `crates/crab-staging/src/index.rs`, `crates/crab-staging/src/lib.rs`. - Recovery distinguishes confirmed absence from filesystem lookup errors; only absence permits dropping pending locators. It rejects missing/undersized sealed segments; current-segment truncation and index cleanup must agree on the surviving durable data. Source: `crates/crab-staging/src/recovery.rs`. @@ -46,6 +55,18 @@ Trace one path: `crab/src/cmd/add.rs` (`close_staging_before_indexing`) → - Multipart abandonment scans reject unrepresentable clock values; never convert a failed time conversion into an expired-lease cutoff. Provider cleanup still requires claiming the observed row revision. Source: `crates/crab-staging/src/multipart_resume.rs`. +- Completed Git-clean and native `--defer` batches delegate publication only after + flush and verification. Delegation seals all batch paths and rejects further membership/recording edits. + Delegation is not Git-index acceptance. Push may capture + only selected pointer identities; retirement preserves uncaptured handoffs and + active native owners. Layout v3 upgrades released v1 atomically and rejects the + unreleased v2 prototype. Sources: `index.rs`, `lib.rs`, and crate README. + +- `StagingPushSnapshot` owns reader fencing and exact snapshot cleanup. Product + callers establish remote commit; the guard retains deferred committed journals + and preserves unpublished leases on cancellation. Read `src/push_snapshot.rs` + and the capsule consumer before changing this boundary. + ## Features and platform No declared Cargo features. Requires writable local filesystem/SQLite and platform lock semantics. Scale tests under `crates/crab-staging/tests` need dedicated disk/time budgets; do not run them as an incidental unit check. diff --git a/crates/crab-staging/README.md b/crates/crab-staging/README.md index a62f65295..7b954a0dc 100644 --- a/crates/crab-staging/README.md +++ b/crates/crab-staging/README.md @@ -49,6 +49,64 @@ failure rolls back the batch; an unlink failure leaves an orphan for recovery, not a missing live payload. This shares one cleanup path across publication, rollback and push, avoiding repeated per-file commits and inventory scans. +A direct-stream preparation retains each reused prepared Xorb in the same +transaction that selects its locator. The original recipe can then retire +without deleting bytes needed by the unsealed recording. Sealing transfers +retention to recipe leases; abort releases preparation retention and reclaims +only bodies with no other owner. Selection failures roll back these pins. + +Segment payload inventory is also independent of file occurrence rows. An +in-flight recipe recording or sealed recipe retains reused segment bytes when +the original file retires. Reads and deduplication use that inventory; recipes +preserve every ordered occurrence separately. Batch rollback releases recording +retention, and cleanup removes inventory only after the last owner retires. + +Compaction copies each retained payload once, then atomically moves inventory +and all committed/pending occurrence locators. Output reservation remains in +that transaction until the new body is durable. Readers capture file handles +with their locators, so unlink and reader-pool eviction cannot invalidate an +already captured segment read. Recovery reclaims unindexed compaction bodies +only after checking both segment registry and canonical payload ownership. + +Staging layout v3 upgrades the released v1 layout transactionally on writable +or shared open. It repairs inventory using the locators v1 readers used and +creates explicit external-publication ownership in the same transaction. Old +writers reject v3. Payload encoding stays unchanged. Missing dependencies or +failed schema installation leave v1 admission intact; deleted bytes cannot be +recovered by migration. The unreleased v2 prototype is rejected and needs a +fresh staging root and restaging, rather than an implicit compatibility reader. + +Native add owns its batches until its Git-index transaction succeeds or rolls +back. Git-clean cannot acknowledge Git's subsequent index replacement. It +instead delegates a complete verified lease after payload flush, before returning +a pointer. Native `--defer` delegates the same way. Delegation freezes the +completed batch and requires all of its paths to be handed off, with no active +preparation. It does not create a published path head. +Explicit staging maintenance holds Git's index lock, checks bounded pointer +blobs and publishes only batches whose every path matches the index. A missing +or different index entry retains its lease, including native `--defer` work and +older unpushed history. Publication consumes its delegation markers atomically. + +Push discovers pointers in selected Git history, loads only +their published or delegated recipes, then captures exact eligible leases. +Retirement releases only captured owners after remote publication; newer handoffs +and active native transactions survive. Publishing a newer head never cancels +another caller's open lease. Failed Git conversion can leave a retained handoff; +reconciliation and abandoned-handoff reclamation still need qualification. + +`StagingPushSnapshot::begin` fences retirement before recipe selection; +`capture` pins exact selected leases. Call `commit` only after confirmed remote +publication. It records a committed journal before retiring captured owners. +Drop clears the reader marker and discards an uncommitted snapshot when no +sibling reader prevents cleanup. Deferred journals remain for exclusive +maintenance; native and uncaptured leases remain owned throughout. + +Native intent recovery may preserve caller-attested committed recipes under +canonical history owners before rolling back an old Git index state. +`preserve_publication_history` accepts only verified recipe roots belonging to +that unresolved intent. It leaves the original intent intact on cancellation or +error, so interrupted preservation can be retried before rollback. + ## Recovery contract Opening writable staging runs recovery before creating or reusing its writer. @@ -111,6 +169,22 @@ multiple batches with consecutive chunk offsets. Flush all staged bytes before publishing a push bundle; otherwise a remote upload can reference data that is not yet durable locally. +`stream::stage_reader_streaming` accepts an `AsyncRead + Unpin` byte source +without seeking, reopening or reading source metadata. Native file staging uses +the same ingestion engine behind descriptor/path validation. Both paths compute +ordered recipes and use the existing segment/prepared-Xorb ownership machinery. +Successful ingestion flushes pending segment authority before recording the +version lease. The returned batch stays open until its caller publishes or +rolls back; the supplied logical path is not proof of a Git index update. +Cancel through the token while awaiting the operation so failed attempts clean +up. Production Git-clean adapts packet bytes to this engine without reopening or +seeking the source. Exact prepared inputs reuse their recipe after byte verification; +changed inputs replay a pinned, bounded prefix and continue through the same engine. +Short reads coalesce into one bounded block before CDC. The released custom +`ChunkStager` injection API retains its byte-to-chunk adapter until a breaking +library release. Real 10-GiB crossover, termination and failure qualification remain +required before declaring the unification complete. + ## Boundaries - [`crab-xet`](../crab-xet/README.md) computes chunk hashes and builds Xorb diff --git a/crates/crab-staging/src/add_push_plan.rs b/crates/crab-staging/src/add_push_plan.rs index eb7201ac1..d6e22ff5a 100644 --- a/crates/crab-staging/src/add_push_plan.rs +++ b/crates/crab-staging/src/add_push_plan.rs @@ -42,10 +42,6 @@ pub struct AddPushPlanSummary { } /// Looks up already-uploaded chunk placements for staged chunks. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait ExistingChunkLookup: Send + Sync { async fn lookup_existing_candidates( @@ -55,10 +51,6 @@ pub trait ExistingChunkLookup: Send + Sync { } /// Adds local prepared-xorb candidates to the staging cache. -#[expect( - clippy::double_must_use, - reason = "async_trait marks boxed futures must-use; Result is also must-use" -)] #[async_trait] pub trait LocalXorbCandidateLookup: Send + Sync { async fn load_candidates( diff --git a/crates/crab-staging/src/index.rs b/crates/crab-staging/src/index.rs index 153f241ab..80ce572fe 100644 --- a/crates/crab-staging/src/index.rs +++ b/crates/crab-staging/src/index.rs @@ -18,15 +18,23 @@ type StoredRecipeMetadata = (i64, i64, Vec, i64, Vec, String); type StoredRecipeWithHashRow = (Vec, i64, i64, Vec, i64, Vec, String, Vec); type ResidualAuthorityRow = (i64, Option>, Option>, Option); -/// Canonical pre-release on-disk layout contract. -const LAYOUT_VERSION: &str = "1"; -const PUBLISHED_RECIPE_LOOKUP_BATCH_SIZE: usize = 512; +/// Version 3 records leases handed to an external Git publisher. +const LAYOUT_VERSION: &str = "3"; +const RECIPE_LOOKUP_BATCH_SIZE: usize = 512; + +#[derive(Clone, Copy)] +enum RecipeOwner { + Published, + External, + Retained, +} const CANONICAL_TABLES: &[&str] = &[ "add_preparation_batches", "add_preparations", "chunk_payloads", "chunks", + "external_publications", "file_paths", "file_recipes", "files", @@ -281,9 +289,9 @@ fn nonnegative_count(field: &str, value: i64) -> Result { } fn retired_staging_schema() -> StagingError { - StagingError::StagingCorrupt( - "staging schema is not canonical v1; remove .crab/staging and restage".to_owned(), - ) + StagingError::StagingCorrupt(format!( + "staging schema is not canonical v{LAYOUT_VERSION}; remove .crab/staging and restage" + )) } fn sqlite_i64(field: &str, value: u64) -> Result { @@ -303,11 +311,12 @@ fn validate_chunk_index(expected: usize, actual: i64) -> Result<()> { Ok(()) } -fn ensure_recording_batch_open( +fn ensure_batch_mutable( tx: &rusqlite::Transaction<'_>, batch_id: &str, operation: &str, ) -> Result<()> { + ensure_batch_not_delegated(tx, batch_id, operation)?; let batch_is_open: bool = tx .query_row( "SELECT EXISTS( @@ -330,6 +339,24 @@ fn ensure_recording_batch_open( }) } +fn ensure_batch_not_delegated( + tx: &rusqlite::Transaction<'_>, + batch_id: &str, + operation: &str, +) -> Result<()> { + let delegated: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM external_publications WHERE batch_id = ?1)", + params![batch_id], + |row| row.get(0), + )?; + if delegated { + return Err(StagingError::StagingCorrupt(format!( + "{operation} cannot mutate delegated batch {batch_id}" + ))); + } + Ok(()) +} + fn publication_intent_batch_ids( tx: &rusqlite::Transaction<'_>, intent_id: &str, @@ -497,7 +524,7 @@ impl Index { /// Open (or create) the staging index at `path`. /// /// Enables WAL mode, `PRAGMA synchronous = NORMAL`, and foreign key - /// enforcement. Initializes a fresh canonical v1 schema or rejects any + /// enforcement. Initializes or upgrades the canonical schema, rejecting any /// other shape with an explicit restage instruction. /// /// # Errors @@ -580,11 +607,32 @@ impl Index { fn open_canonical_schema(&mut self) -> Result<()> { let existing_tables = self.application_objects("table")?; let fresh = existing_tables.is_empty(); + let mut needs_payload_upgrade = false; if !fresh { + if !existing_tables.iter().any(|name| name == "staging_meta") { + return Err(retired_staging_schema()); + } + let version: Option = self + .conn + .query_row( + "SELECT value FROM staging_meta WHERE key = 'layout_version'", + [], + |row| row.get(0), + ) + .optional()?; + needs_payload_upgrade = match version.as_deref() { + Some("1") => true, + Some(LAYOUT_VERSION) => false, + _ => return Err(retired_staging_schema()), + }; + let expected_tables = CANONICAL_TABLES + .iter() + .copied() + .filter(|name| !needs_payload_upgrade || *name != "external_publications"); if existing_tables .iter() .map(String::as_str) - .ne(CANONICAL_TABLES.iter().copied()) + .ne(expected_tables) { return Err(retired_staging_schema()); } @@ -633,23 +681,58 @@ impl Index { { return Err(retired_staging_schema()); } - let version: Option = self - .conn - .query_row( - "SELECT value FROM staging_meta WHERE key = 'layout_version'", - [], - |row| row.get(0), - ) - .optional() - .map_err(|e| { - StagingError::Internal(format!("failed to read staging layout version: {e}")) - })?; - if version.as_deref() != Some(LAYOUT_VERSION) { + if !needs_payload_upgrade + && self + .table_columns("external_publications")? + .iter() + .map(String::as_str) + .ne(["batch_id", "path_bytes"]) + { return Err(retired_staging_schema()); } + if !needs_payload_upgrade { + let keys = self.conn.prepare( + "SELECT name, pk FROM pragma_table_info('external_publications') ORDER BY cid" + )?.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)))? + .collect::, _>>()?; + let foreign = self.conn.prepare( + "SELECT [table], [from], [to], on_delete FROM pragma_foreign_key_list('external_publications') ORDER BY id, seq" + )?.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, + row.get::<_, String>(2)?, row.get::<_, String>(3)?)))? + .collect::, _>>()?; + if keys != [("batch_id".to_owned(), 1), ("path_bytes".to_owned(), 2)] + || foreign + != [ + ( + "path_leases".to_owned(), + "batch_id".to_owned(), + "batch_id".to_owned(), + "CASCADE".to_owned(), + ), + ( + "path_leases".to_owned(), + "path_bytes".to_owned(), + "path_bytes".to_owned(), + "CASCADE".to_owned(), + ), + ] + { + return Err(retired_staging_schema()); + } + } } + // Admission, new ownership schema and released locator repair commit + // together. A failed upgrade must leave the v1 shape and bytes intact. + let tx = self.conn.unchecked_transaction()?; self.initialize_schema()?; + if needs_payload_upgrade { + Self::repair_segment_payload_authority(&tx)?; + tx.execute( + "UPDATE staging_meta SET value = ?1 WHERE key = 'layout_version'", + params![LAYOUT_VERSION], + )?; + } if fresh { self.conn .execute( @@ -664,6 +747,47 @@ impl Index { "initialized canonical staging schema" ); } + tx.commit()?; + Ok(()) + } + + fn repair_segment_payload_authority(tx: &rusqlite::Transaction<'_>) -> Result<()> { + // Released v1 compaction moved occurrence rows without updating payload + // inventory. Repair from the locators v1 readers actually used, then + // change admission atomically so an old writer cannot recreate the gap. + tx.execute( + "UPDATE chunk_payloads AS payload + SET (segment_id, segment_offset) = ( + SELECT candidate.segment_id, candidate.segment_offset FROM ( + SELECT segment_id, segment_offset, size, 0 AS priority, rowid + FROM chunks WHERE chunk_hash = payload.chunk_hash + UNION ALL + SELECT segment_id, segment_offset, size, 1 AS priority, rowid + FROM pending_chunks WHERE chunk_hash = payload.chunk_hash + ) AS candidate + JOIN segments USING (segment_id) + WHERE candidate.size = payload.size + ORDER BY candidate.priority, candidate.rowid LIMIT 1 + ) + WHERE EXISTS (SELECT 1 FROM chunks JOIN segments USING (segment_id) + WHERE chunk_hash = payload.chunk_hash AND size = payload.size) + OR EXISTS (SELECT 1 FROM pending_chunks JOIN segments USING (segment_id) + WHERE chunk_hash = payload.chunk_hash AND size = payload.size)", + [], + )?; + Self::prune_unowned_segment_payloads(tx)?; + let missing_segment: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM chunk_payloads AS payload + WHERE NOT EXISTS (SELECT 1 FROM segments + WHERE segments.segment_id = payload.segment_id))", + [], + |row| row.get(0), + )?; + if missing_segment { + return Err(StagingError::StagingCorrupt( + "released payload inventory references a missing segment".to_owned(), + )); + } Ok(()) } @@ -703,7 +827,7 @@ impl Index { }) } - fn initialize_schema(&mut self) -> Result<()> { + fn initialize_schema(&self) -> Result<()> { self.conn .execute_batch( "CREATE TABLE IF NOT EXISTS files ( @@ -876,6 +1000,14 @@ impl Index { CREATE INDEX IF NOT EXISTS leases_by_file ON path_leases(file_hash); + CREATE TABLE IF NOT EXISTS external_publications ( + batch_id TEXT NOT NULL, + path_bytes BLOB NOT NULL, + PRIMARY KEY (batch_id, path_bytes), + FOREIGN KEY (batch_id, path_bytes) REFERENCES path_leases(batch_id, path_bytes) + ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS path_heads ( path_bytes BLOB PRIMARY KEY, batch_id TEXT NOT NULL, @@ -1369,33 +1501,12 @@ impl Index { Ok(count as u64) } - /// Look up a chunk by hash, returning its locator if staged. - /// - /// Single index lookup on `chunks_by_hash`. Returns `None` if the - /// chunk is not in the committed `chunks` table. - /// - /// # Errors - /// - /// Look up a chunk by hash, returning its locator if staged. - /// - /// Checks the committed `chunks` table first, falling back to - /// `pending_chunks`. Post-`flush_pending` rows still live in - /// `pending_chunks`, so `locate` must check both tables to stay - /// consistent with `locate_batch` and `get_chunk`. - /// - /// # Errors - /// - /// Returns [`StagingError::Internal`] on SQLite failure. + /// Resolve a segment payload independently of file occurrence ownership. pub fn locate(&self, chunk_hash: &[u8; 32]) -> Result> { - let hash_slice: &[u8] = chunk_hash; - let committed = self - .conn + self.conn .query_row( - "SELECT segment_id, segment_offset, size - FROM chunks - WHERE chunk_hash = ?1 - LIMIT 1", - params![hash_slice], + "SELECT segment_id, segment_offset, size FROM chunk_payloads WHERE chunk_hash = ?1", + params![chunk_hash.as_slice()], |row| { Ok(ChunkLocator { segment_id: row.get(0)?, @@ -1405,13 +1516,7 @@ impl Index { }, ) .optional() - .map_err(|e| StagingError::Internal(format!("failed to locate chunk: {e}")))?; - - if committed.is_some() { - return Ok(committed); - } - - self.locate_pending(chunk_hash) + .map_err(StagingError::from) } /// Return the byte offset one past the last committed record for a @@ -1469,9 +1574,14 @@ impl Index { let offset: Option = self .conn .query_row( - "SELECT MAX(segment_offset + size + 8) - FROM chunks - WHERE segment_id = ?1", + "SELECT MAX(boundary) FROM ( + SELECT segment_offset + size + 8 AS boundary FROM chunks WHERE segment_id = ?1 + UNION ALL + SELECT payload.segment_offset + payload.size + 8 + FROM chunk_payloads AS payload WHERE payload.segment_id = ?1 + AND EXISTS (SELECT 1 FROM recipe_payload_leases AS lease + WHERE lease.chunk_hash = payload.chunk_hash) + )", params![segment_id], |row| row.get(0), ) @@ -1535,6 +1645,7 @@ impl Index { LEFT JOIN pending_chunks p ON p.segment_id = s.segment_id WHERE s.live_chunk_count = 0 AND s.sealed_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM chunk_payloads AS payload WHERE payload.segment_id = s.segment_id) GROUP BY s.segment_id HAVING COUNT(p.rowid) = 0", ) @@ -1574,7 +1685,7 @@ impl Index { COALESCE(p.n, 0) AS pending_chunks FROM segments s LEFT JOIN ( - SELECT segment_id, COUNT(*) AS n FROM chunks GROUP BY segment_id + SELECT segment_id, COUNT(*) AS n FROM chunk_payloads GROUP BY segment_id ) c ON c.segment_id = s.segment_id LEFT JOIN ( SELECT segment_id, COUNT(*) AS n FROM pending_chunks GROUP BY segment_id @@ -1654,7 +1765,8 @@ impl Index { ) p ON p.segment_id = s.segment_id WHERE s.segment_id != ?1 AND COALESCE(c.n, 0) = 0 - AND COALESCE(p.n, 0) = 0", + AND COALESCE(p.n, 0) = 0 + AND NOT EXISTS (SELECT 1 FROM chunk_payloads AS payload WHERE payload.segment_id = s.segment_id)", ) .map_err(|e| { StagingError::Internal(format!("failed to prepare abandoned query: {e}")) @@ -1746,19 +1858,24 @@ impl Index { /// /// Returns [`StagingError::Internal`] on SQLite failure. pub fn delete_orphan_pending_rows(&self) -> Result { - let n = self - .conn - .execute( - "DELETE FROM pending_chunks - WHERE segment_id NOT IN (SELECT segment_id FROM segments)", - [], - ) - .map_err(|e| { - StagingError::Internal(format!("failed to delete orphan pending rows: {e}")) - })?; + let tx = self.conn.unchecked_transaction()?; + let n = tx.execute( + "DELETE FROM pending_chunks WHERE segment_id NOT IN (SELECT segment_id FROM segments)", + [], + )?; + Self::prune_unowned_segment_payloads(&tx)?; + tx.commit()?; Ok(n as u64) } + pub fn segment_has_payloads(&self, segment_id: u64) -> Result { + Ok(self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM chunk_payloads WHERE segment_id = ?1)", + params![segment_id], + |row| row.get(0), + )?) + } + /// Mark a segment as sealed with the current timestamp. /// /// # Errors @@ -1862,10 +1979,7 @@ impl Index { .conn .query_row( "SELECT EXISTS( - SELECT 1 FROM chunks WHERE chunk_hash = ?1 - UNION ALL - SELECT 1 FROM pending_chunks WHERE chunk_hash = ?1 - LIMIT 1 + SELECT 1 FROM chunk_payloads WHERE chunk_hash = ?1 )", params![hash_slice], |row| row.get(0), @@ -2261,6 +2375,7 @@ impl Index { AND EXISTS ( SELECT 1 FROM staging_batches WHERE batch_id = ?2 AND state = 'open' + AND NOT EXISTS (SELECT 1 FROM external_publications WHERE batch_id = ?2) )", params![preparation_id, batch_id], ) @@ -2296,6 +2411,7 @@ impl Index { let tx = self.conn.unchecked_transaction().map_err(|error| { StagingError::Internal(format!("failed to begin prepared claim batch: {error}")) })?; + ensure_batch_not_delegated(&tx, batch_id, "prepared authority")?; let valid_owner: bool = tx .query_row( "SELECT EXISTS( @@ -2479,6 +2595,11 @@ impl Index { } } + let mut retained_xorbs = HashSet::new(); + let mut retain_payload = tx.prepare_cached( + "INSERT OR IGNORE INTO preparation_payloads (preparation_id, xorb_hash) + VALUES (?1, ?2)", + )?; let mut out = Vec::with_capacity(chunks.len()); for (chunk_hash, size) in chunks { if let Some(locator) = prepared.get(chunk_hash) { @@ -2489,6 +2610,13 @@ impl Index { locator.size ))); } + // The incoming recipe is not sealed yet. Pin reused bytes in + // this transaction so retiring their old recipe cannot remove + // them before this preparation transfers or releases ownership. + if retained_xorbs.insert(locator.xorb_hash) { + retain_payload + .execute(params![preparation_id, locator.xorb_hash.as_slice()])?; + } out.push(PreparedChunkClaim::Prepared(*locator)); continue; } @@ -2530,6 +2658,7 @@ impl Index { claims.insert(*chunk_hash, (preparation_id.to_owned(), *size)); out.push(PreparedChunkClaim::Claimed); } + drop(retain_payload); tx.commit().map_err(|error| { StagingError::Internal(format!("failed to commit prepared claim batch: {error}")) })?; @@ -2545,6 +2674,7 @@ impl Index { let tx = self.conn.unchecked_transaction().map_err(|error| { StagingError::Internal(format!("failed to begin prepared payload seal: {error}")) })?; + ensure_batch_not_delegated(&tx, owner_batch_id, "prepared authority")?; let recording: bool = tx .query_row( "SELECT EXISTS( @@ -3083,7 +3213,7 @@ impl Index { let tx = self.conn.unchecked_transaction().map_err(|e| { StagingError::Internal(format!("failed to begin recipe recording append: {e}")) })?; - ensure_recording_batch_open(&tx, batch_id, "recipe")?; + ensure_batch_mutable(&tx, batch_id, "recipe")?; Self::append_recipe_recording_terms_in_tx( &tx, batch_id, @@ -3377,7 +3507,7 @@ impl Index { let tx = self.conn.unchecked_transaction().map_err(|error| { StagingError::Internal(format!("failed to begin recording batch append: {error}")) })?; - ensure_recording_batch_open(&tx, batch_id, "combined")?; + ensure_batch_mutable(&tx, batch_id, "combined")?; Self::append_recording_remote_chunks_in_tx(&tx, batch_id, remote_chunks)?; Self::append_recipe_recording_terms_in_tx( &tx, @@ -3483,6 +3613,12 @@ impl Index { params![batch_id], ) .map_err(|e| StagingError::Internal(format!("failed to replace path heads: {e}")))?; + tx.execute( + "DELETE FROM external_publications WHERE batch_id = ?1", + params![batch_id], + )?; + // A new head cannot cancel another active caller or external writer. + // Supersede only published leases; open owners release themselves. tx.execute( "DELETE FROM path_leases AS lease WHERE EXISTS ( @@ -3490,6 +3626,10 @@ impl Index { WHERE incoming.batch_id = ?1 AND incoming.path_bytes = lease.path_bytes ) + AND EXISTS ( + SELECT 1 FROM staging_batches AS owner + WHERE owner.batch_id = lease.batch_id AND owner.state = 'published' + ) AND NOT EXISTS ( SELECT 1 FROM path_heads AS head WHERE head.batch_id = lease.batch_id @@ -3600,6 +3740,118 @@ impl Index { Ok(Some(unowned)) } + /// Delegate complete verified leases without asserting a Git index write. + pub fn delegate_publication(&self, entries: &[(String, Vec)]) -> Result<()> { + let tx = self.conn.unchecked_transaction()?; + for (batch_id, path_bytes) in entries { + let ready: bool = tx.query_row( + "SELECT EXISTS( + SELECT 1 FROM path_leases AS lease + JOIN staging_batches AS batch USING (batch_id) + JOIN verified_recipes USING (recipe_hash) + WHERE lease.batch_id = ?1 AND lease.path_bytes = ?2 AND batch.state = 'open' + AND NOT EXISTS (SELECT 1 FROM recipe_recording_terms AS term WHERE term.batch_id = lease.batch_id) + AND NOT EXISTS (SELECT 1 FROM recording_remote_chunks AS remote WHERE remote.batch_id = lease.batch_id) + AND NOT EXISTS (SELECT 1 FROM add_preparation_batches AS preparation + WHERE preparation.batch_id = lease.batch_id) + AND NOT EXISTS (SELECT 1 FROM publication_intent_entries AS intent + WHERE intent.batch_id = lease.batch_id) + )", + params![batch_id, path_bytes], + |row| row.get(0), + )?; + if !ready { + return Err(StagingError::StagingCorrupt( + "external publication requires a complete verified batch without an active native publisher" + .to_owned(), + )); + } + tx.execute("INSERT OR IGNORE INTO external_publications (batch_id, path_bytes) VALUES (?1, ?2)", + params![batch_id, path_bytes])?; + } + for batch_id in entries + .iter() + .map(|(batch, _)| batch) + .collect::>() + { + let incomplete: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM path_leases AS lease + WHERE lease.batch_id = ?1 AND NOT EXISTS ( + SELECT 1 FROM external_publications AS external + WHERE external.batch_id = lease.batch_id AND external.path_bytes = lease.path_bytes + ))", + params![batch_id], |row| row.get(0), + )?; + if incomplete { + return Err(StagingError::StagingCorrupt( + "delegation must hand off every completed path in a batch".to_owned(), + )); + } + } + tx.commit()?; + Ok(()) + } + + pub fn delegated_publications(&self) -> Result> { + let invalid: bool = self.conn.query_row( + "SELECT EXISTS(SELECT 1 FROM external_publications AS external + JOIN staging_batches AS batch USING (batch_id) + WHERE batch.state = 'open' AND ( + EXISTS (SELECT 1 FROM recipe_recording_terms AS term WHERE term.batch_id = external.batch_id) OR + EXISTS (SELECT 1 FROM recording_remote_chunks AS remote WHERE remote.batch_id = external.batch_id) OR + EXISTS (SELECT 1 FROM add_preparation_batches AS preparation + WHERE preparation.batch_id = external.batch_id) OR + NOT EXISTS ( + SELECT 1 FROM path_leases AS lease JOIN verified_recipes USING (recipe_hash) + JOIN file_recipes AS recipe USING (recipe_hash) + WHERE lease.batch_id = external.batch_id AND lease.path_bytes = external.path_bytes + AND lease.file_hash = recipe.file_hash + ) OR EXISTS ( + SELECT 1 FROM path_leases AS lease WHERE lease.batch_id = external.batch_id + AND NOT EXISTS (SELECT 1 FROM external_publications AS sibling + WHERE sibling.batch_id = lease.batch_id AND sibling.path_bytes = lease.path_bytes) + ) + ))", [], |row| row.get(0), + )?; + if invalid { + return Err(StagingError::StagingCorrupt( + "incomplete delegated batch ownership".to_owned(), + )); + } + let mut statement = self.conn.prepare_cached( + "SELECT lease.batch_id, lease.path_bytes, lease.file_hash, recipe.file_size + FROM external_publications AS external + JOIN path_leases AS lease + ON lease.batch_id = external.batch_id AND lease.path_bytes = external.path_bytes + JOIN staging_batches AS batch ON batch.batch_id = lease.batch_id + JOIN file_recipes AS recipe USING (recipe_hash) + JOIN verified_recipes USING (recipe_hash) + WHERE batch.state = 'open' + ORDER BY lease.batch_id, lease.path_bytes", + )?; + let rows = statement.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Vec>(1)?, + row.get::<_, Vec>(2)?, + row.get::<_, i64>(3)?, + )) + })?; + rows.map(|row| { + let (batch_id, path_bytes, file_hash, size) = row?; + Ok(super::DelegatedPublication { + batch_id: super::StagingBatchId(batch_id), + path_bytes, + file_hash: crab_xet::hash::MerkleHash::from(decode_hash_blob( + "delegated file hash", + file_hash, + )?), + size: nonnegative_count("delegated file size", size)?, + }) + }) + .collect() + } + pub fn create_publication_intent( &self, intent_id: &str, @@ -3631,7 +3883,9 @@ impl Index { JOIN staging_batches AS batch USING (batch_id) JOIN verified_recipes AS verified USING (recipe_hash) WHERE lease.batch_id = ?1 AND lease.path_bytes = ?2 - AND batch.state = 'open'", + AND batch.state = 'open' + AND NOT EXISTS (SELECT 1 FROM external_publications AS external + WHERE external.batch_id = lease.batch_id)", params![batch_id, path_bytes], |row| row.get(0), ) @@ -3824,6 +4078,8 @@ impl Index { "failed to replace publication path heads {intent_id}: {e}" )) })?; + // A new head cannot cancel another active caller or external writer. + // Supersede only published leases; open owners release themselves. tx.execute( "DELETE FROM path_leases AS lease WHERE EXISTS ( @@ -3831,6 +4087,10 @@ impl Index { WHERE incoming.intent_id = ?1 AND incoming.path_bytes = lease.path_bytes ) + AND EXISTS ( + SELECT 1 FROM staging_batches AS owner + WHERE owner.batch_id = lease.batch_id AND owner.state = 'published' + ) AND NOT EXISTS ( SELECT 1 FROM path_heads AS head WHERE head.batch_id = lease.batch_id @@ -3943,7 +4203,7 @@ impl Index { ) -> Result> { let mut selected = None; for (returned_hash, recipe) in - self.load_published_recipe_rows(std::slice::from_ref(file_hash))? + self.load_recipe_rows(std::slice::from_ref(file_hash), RecipeOwner::Published)? { if returned_hash != *file_hash { return Err(StagingError::StagingCorrupt( @@ -3968,24 +4228,49 @@ impl Index { pub fn published_recipes_for_files( &self, file_hashes: &[[u8; 32]], + ) -> Result>> { + self.recipes_for_files(file_hashes, RecipeOwner::Published) + } + + /// Load only caller-verified leases explicitly delegated to an external writer. + pub fn external_recipes_for_files( + &self, + file_hashes: &[[u8; 32]], + ) -> Result>> { + self.recipes_for_files(file_hashes, RecipeOwner::External) + } + + pub fn retained_recipe_for_file( + &self, + file_hash: &[u8; 32], + ) -> Result> { + let mut recipes = + self.recipes_for_files(std::slice::from_ref(file_hash), RecipeOwner::Retained)?; + Ok(recipes.remove(file_hash).flatten()) + } + + fn recipes_for_files( + &self, + file_hashes: &[[u8; 32]], + owner: RecipeOwner, ) -> Result>> { let mut recipes = HashMap::with_capacity(file_hashes.len()); - for batch in file_hashes.chunks(PUBLISHED_RECIPE_LOOKUP_BATCH_SIZE) { + for batch in file_hashes.chunks(RECIPE_LOOKUP_BATCH_SIZE) { for file_hash in batch { recipes.entry(*file_hash).or_insert(None); } - for (file_hash, recipe) in self.load_published_recipe_rows(batch)? { + for (file_hash, recipe) in self.load_recipe_rows(batch, owner)? { match recipes.get_mut(&file_hash) { Some(slot @ None) => *slot = Some(recipe), Some(Some(prior)) if prior == &recipe => {} Some(Some(_)) => { return Err(StagingError::StagingCorrupt( - "one file hash has conflicting published recipes".to_owned(), + "one file hash has conflicting stored recipes".to_owned(), )); } None => { return Err(StagingError::StagingCorrupt( - "published recipe query returned an unexpected file hash".to_owned(), + "stored recipe query returned an unexpected file hash".to_owned(), )); } } @@ -3994,9 +4279,10 @@ impl Index { Ok(recipes) } - fn load_published_recipe_rows( + fn load_recipe_rows( &self, file_hashes: &[[u8; 32]], + owner: RecipeOwner, ) -> Result> { if file_hashes.is_empty() { return Ok(Vec::new()); @@ -4004,6 +4290,30 @@ impl Index { let placeholders = std::iter::repeat_n("?", file_hashes.len()) .collect::>() .join(","); + let ownership = match owner { + RecipeOwner::Published => { + "JOIN path_heads AS head USING (recipe_hash) + JOIN staging_batches AS batch USING (batch_id) + WHERE batch.state = 'published'" + } + RecipeOwner::Retained => { + "WHERE (EXISTS (SELECT 1 FROM path_leases AS lease + WHERE lease.file_hash = recipe.file_hash + AND lease.recipe_hash = recipe.recipe_hash) + OR EXISTS (SELECT 1 FROM push_snapshots AS snapshot + WHERE snapshot.state IN ('open', 'committed') + AND EXISTS (SELECT 1 FROM push_snapshot_recipes AS pin + WHERE pin.snapshot_id = snapshot.snapshot_id + AND pin.recipe_hash = recipe.recipe_hash)))" + } + RecipeOwner::External => { + "JOIN path_leases AS lease USING (recipe_hash) + JOIN external_publications AS external + ON external.batch_id = lease.batch_id AND external.path_bytes = lease.path_bytes + JOIN staging_batches AS batch ON batch.batch_id = lease.batch_id + WHERE batch.state = 'open'" + } + }; let query = format!( "SELECT DISTINCT recipe.file_hash, recipe.recipe_hash, recipe.file_size, recipe.chunk_count, recipe.sequence_hash, @@ -4011,13 +4321,11 @@ impl Index { FROM file_recipes AS recipe JOIN files AS file USING (file_hash) JOIN verified_recipes AS verified USING (recipe_hash) - JOIN path_heads AS head USING (recipe_hash) - JOIN staging_batches AS batch USING (batch_id) - WHERE recipe.file_hash IN ({placeholders}) AND batch.state = 'published' + {ownership} AND recipe.file_hash IN ({placeholders}) ORDER BY recipe.file_hash, recipe.recipe_hash" ); let mut statement = self.conn.prepare_cached(&query).map_err(|e| { - StagingError::Internal(format!("failed to prepare published recipe query: {e}")) + StagingError::Internal(format!("failed to prepare stored recipe query: {e}")) })?; let rows = statement .query_map( @@ -4035,10 +4343,10 @@ impl Index { )) }, ) - .map_err(|e| StagingError::Internal(format!("failed to query published recipes: {e}")))? + .map_err(|e| StagingError::Internal(format!("failed to query stored recipes: {e}")))? .collect::, _>>() .map_err(|e| { - StagingError::Internal(format!("failed to collect published recipes: {e}")) + StagingError::Internal(format!("failed to collect stored recipes: {e}")) })?; let mut recipes = Vec::with_capacity(rows.len()); @@ -4053,8 +4361,8 @@ impl Index { policy_id, ) in rows { - let file_hash = decode_hash_blob("published recipe file hash", raw_file_hash)?; - let stored_recipe_hash = decode_hash_blob("published recipe hash", raw_recipe_hash)?; + let file_hash = decode_hash_blob("stored recipe file hash", raw_file_hash)?; + let stored_recipe_hash = decode_hash_blob("stored recipe hash", raw_recipe_hash)?; let recipe = self.load_stored_recipe( &stored_recipe_hash, &file_hash, @@ -4552,9 +4860,9 @@ impl Index { /// Atomically pin the exact immutable recipes consumed by one push. /// - /// Every recipe must already be owned by a published path lease. The - /// snapshot is therefore a reference to committed local staging state, - /// never a way to manufacture ownership from a pointer alone. + /// Every recipe must have a published head or an explicitly delegated + /// verified lease. Callers must first observe its Git pointer; ordinary + /// open native batches cannot manufacture push visibility. pub fn create_push_snapshot( &self, snapshot_id: &str, @@ -4634,6 +4942,13 @@ impl Index { FROM path_heads AS head JOIN staging_batches AS batch USING (batch_id) WHERE head.recipe_hash = ?1 AND batch.state = 'published' + ) OR EXISTS ( + SELECT 1 FROM external_publications AS external + JOIN path_leases AS lease + ON lease.batch_id = external.batch_id AND lease.path_bytes = external.path_bytes + JOIN staging_batches AS batch ON batch.batch_id = lease.batch_id + JOIN verified_recipes USING (recipe_hash) + WHERE lease.recipe_hash = ?1 AND batch.state = 'open' )", params![recipe_hash.as_slice()], |row| row.get(0), @@ -4645,7 +4960,7 @@ impl Index { })?; if !published_head { return Err(StagingError::StagingCorrupt(format!( - "push snapshot {snapshot_id} recipe {} has no published path head", + "push snapshot {snapshot_id} recipe {} has no published or delegated lease", blake3::Hash::from(recipe_hash).to_hex() ))); } @@ -4667,7 +4982,14 @@ impl Index { head.file_hash, head.recipe_hash FROM path_heads AS head JOIN staging_batches AS batch ON batch.batch_id = head.batch_id - WHERE head.recipe_hash = ?2 AND batch.state = 'published'", + WHERE head.recipe_hash = ?2 AND batch.state = 'published' + UNION + SELECT ?1, lease.batch_id, lease.path_bytes, lease.file_hash, lease.recipe_hash + FROM external_publications AS external + JOIN path_leases AS lease + ON lease.batch_id = external.batch_id AND lease.path_bytes = external.path_bytes + JOIN staging_batches AS batch ON batch.batch_id = lease.batch_id + WHERE lease.recipe_hash = ?2 AND batch.state = 'open'", params![snapshot_id, recipe_hash.as_slice()], ) .map_err(|e| { @@ -4677,7 +4999,7 @@ impl Index { })?; if captured == 0 { return Err(StagingError::StagingCorrupt(format!( - "push snapshot {snapshot_id} recipe {} lost its published lease", + "push snapshot {snapshot_id} recipe {} lost its eligible lease", blake3::Hash::from(recipe_hash).to_hex() ))); } @@ -4726,7 +5048,7 @@ impl Index { } } - /// Retire only the exact published path leases captured by one committed + /// Retire only the exact eligible path leases captured by one committed /// push snapshot. Returns file hashes whose final lease disappeared and /// whose physical chunk rows can therefore be reclaimed. pub fn retire_push_snapshot(&self, snapshot_id: &str) -> Result> { @@ -4856,12 +5178,18 @@ impl Index { })?; tx.execute( "DELETE FROM staging_batches - WHERE state = 'published' + WHERE (state = 'published' OR batch_id IN ( + SELECT batch_id FROM push_snapshot_leases WHERE snapshot_id = ?1 + )) AND NOT EXISTS ( SELECT 1 FROM path_leases WHERE path_leases.batch_id = staging_batches.batch_id + ) + AND NOT EXISTS ( + SELECT 1 FROM publication_intent_entries AS intent + WHERE intent.batch_id = staging_batches.batch_id )", - [], + params![snapshot_id], ) .map_err(|e| { StagingError::Internal(format!( @@ -5081,6 +5409,9 @@ impl Index { let tx = self.conn.unchecked_transaction().map_err(|e| { StagingError::Internal(format!("failed to begin recipe lease transaction: {e}")) })?; + // Delegation seals batch membership. Git-aware recovery can inspect + // identities outside this mutex only while those identities stay fixed. + ensure_batch_not_delegated(&tx, batch_id, "recipe lease")?; tx.execute("DELETE FROM temp.incoming_recipe_occurrences", []) .map_err(|e| { StagingError::Internal(format!("failed to clear incoming recipe occurrences: {e}")) @@ -5713,6 +6044,7 @@ impl Index { ) .map_err(|e| StagingError::Internal(format!("failed to delete staging batch: {e}")))?; + Self::prune_unowned_segment_payloads(&tx)?; let unleased = unowned_file_hashes(&tx, file_hashes)?; tx.commit() .map_err(|e| StagingError::Internal(format!("failed to commit batch rollback: {e}")))?; @@ -5924,6 +6256,22 @@ impl Index { }) } + fn prune_unowned_segment_payloads(conn: &Connection) -> Result<()> { + // Occurrences and recording terms retain bytes independently of their + // original file rows. Materialize recording hashes once so cleanup does + // not rescan a growing in-flight recipe for each inventory entry. + conn.execute( + "DELETE FROM chunk_payloads + WHERE NOT EXISTS (SELECT 1 FROM recipe_payload_leases AS lease + WHERE lease.chunk_hash = chunk_payloads.chunk_hash) + AND NOT EXISTS (SELECT 1 FROM chunks WHERE chunks.chunk_hash = chunk_payloads.chunk_hash) + AND NOT EXISTS (SELECT 1 FROM pending_chunks WHERE pending_chunks.chunk_hash = chunk_payloads.chunk_hash) + AND chunk_hash NOT IN (SELECT chunk_hash FROM recipe_recording_terms)", + [], + )?; + Ok(()) + } + /// Atomically remove files and their chunk rows, then reclaim unowned payload inventory. /// /// Returns per-file retirement stats and prepared bodies eligible for unlinking. @@ -5975,25 +6323,7 @@ impl Index { .map_err(|e| StagingError::Internal(format!("failed to delete file_path: {e}")))?; } - tx.execute( - "DELETE FROM chunk_payloads - WHERE NOT EXISTS ( - SELECT 1 FROM recipe_payload_leases - WHERE recipe_payload_leases.chunk_hash = chunk_payloads.chunk_hash - ) - AND NOT EXISTS ( - SELECT 1 FROM chunks - WHERE chunks.chunk_hash = chunk_payloads.chunk_hash - ) - AND NOT EXISTS ( - SELECT 1 FROM pending_chunks - WHERE pending_chunks.chunk_hash = chunk_payloads.chunk_hash - )", - [], - ) - .map_err(|e| { - StagingError::Internal(format!("failed to delete unleased chunk payloads: {e}")) - })?; + Self::prune_unowned_segment_payloads(&tx)?; let reclaimable_payloads = { let mut statement = tx @@ -6077,6 +6407,7 @@ impl Index { StagingError::Internal(format!("failed to begin chunk retirement: {e}")) })?; let stats = Self::delete_chunks_for_file_in(&tx, file_hash)?; + Self::prune_unowned_segment_payloads(&tx)?; tx.commit().map_err(|e| { StagingError::Internal(format!("failed to commit chunk retirement: {e}")) })?; @@ -7402,8 +7733,8 @@ impl Index { /// Insert chunk rows for a file, linking them to their segment locators. /// - /// For each `(chunk_hash, size)` pair, looks up the locator from - /// `chunks` (preferred) or `pending_chunks` and inserts into `chunks`. + /// For each `(chunk_hash, size)` pair, resolves canonical payload authority + /// and inserts one committed occurrence into `chunks`. /// Removes old chunk rows for this file first and keeps /// `live_chunk_count` in sync with the replacement. /// @@ -7482,16 +7813,10 @@ impl Index { { let mut locate_stmt = tx .prepare_cached( - "SELECT segment_id, segment_offset, size FROM chunks WHERE chunk_hash = ?1 LIMIT 1", + "SELECT segment_id, segment_offset, size FROM chunk_payloads WHERE chunk_hash = ?1", ) .map_err(|e| StagingError::Internal(format!("prepare locate: {e}")))?; - let mut locate_pending_stmt = tx - .prepare_cached( - "SELECT segment_id, segment_offset, size FROM pending_chunks WHERE chunk_hash = ?1 LIMIT 1", - ) - .map_err(|e| StagingError::Internal(format!("prepare locate pending: {e}")))?; - let mut insert_stmt = tx .prepare_cached( "INSERT OR REPLACE INTO chunks (chunk_hash, file_hash, chunk_index, size, segment_id, segment_offset) @@ -7501,7 +7826,6 @@ impl Index { for (idx, (chunk_hash, size)) in chunks.iter().enumerate() { let ch: &[u8] = chunk_hash; - // Try committed chunks first, then pending. let locator: Option<(u64, u64, i64)> = locate_stmt .query_row(params![ch], |row| { Ok((row.get(0)?, row.get(1)?, row.get(2)?)) @@ -7509,21 +7833,15 @@ impl Index { .optional() .map_err(|e| StagingError::Internal(format!("locate chunk: {e}")))?; - let (seg_id, seg_offset, _stored_size) = if let Some(loc) = locator { - loc - } else { - locate_pending_stmt - .query_row(params![ch], |row| { - Ok((row.get(0)?, row.get(1)?, row.get(2)?)) - }) - .optional() - .map_err(|e| StagingError::Internal(format!("locate pending chunk: {e}")))? - .ok_or_else(|| { - StagingError::Internal( - "chunk not found in chunks or pending_chunks".into(), - ) - })? - }; + let (seg_id, seg_offset, stored_size) = + locator.ok_or_else(|| StagingError::ChunkNotFound { + hash: crab_xet::hash::MerkleHash::from(*chunk_hash).hex(), + })?; + if nonnegative_count("registered chunk size", stored_size)? != *size { + return Err(StagingError::StagingCorrupt( + "registered chunk size does not match its payload".to_owned(), + )); + } insert_stmt .execute(params![ @@ -7577,6 +7895,7 @@ impl Index { } } + Self::prune_unowned_segment_payloads(&tx)?; tx.commit().map_err(|e| { StagingError::Internal(format!("failed to commit insert_chunks_for_file tx: {e}")) })?; @@ -7639,41 +7958,29 @@ impl Index { /// /// Returns [`StagingError::Internal`] on SQLite failure. pub fn delete_pending_for_segment(&self, segment_id: u64) -> Result { - let n = self - .conn - .execute( - "DELETE FROM pending_chunks WHERE segment_id = ?1", - params![segment_id], - ) - .map_err(|e| { - StagingError::Internal(format!( - "failed to delete pending chunks for segment {segment_id}: {e}" - )) - })?; + let tx = self.conn.unchecked_transaction()?; + let n = tx.execute( + "DELETE FROM pending_chunks WHERE segment_id = ?1", + params![segment_id], + )?; + Self::prune_unowned_segment_payloads(&tx)?; + tx.commit()?; Ok(n as u64) } - /// Delete pending rows whose framed record crosses a recovered boundary. - /// - /// Used by recovery to discard rows whose payload or framing was truncated, - /// while preserving rows fully contained within the valid range. + /// Discard pending locators and payload inventory beyond the recovered boundary. pub fn delete_pending_beyond_offset(&self, segment_id: u64, max_offset: u64) -> Result<()> { - #[expect(clippy::cast_possible_wrap, reason = "offset fits in i64")] - let max_offset_i64 = max_offset as i64; - self.conn - .execute( - "DELETE FROM pending_chunks - WHERE segment_id = ?1 - AND (segment_offset < 0 - OR size < 0 - OR segment_offset + size + 8 > ?2)", - params![segment_id, max_offset_i64], - ) - .map_err(|e| { - StagingError::Internal(format!( - "failed to delete pending chunks crossing offset {max_offset} for segment {segment_id}: {e}" - )) - })?; + let tx = self.conn.unchecked_transaction()?; + tx.execute( + "DELETE FROM pending_chunks WHERE segment_id = ?1 + AND (segment_offset < 0 OR size < 0 OR segment_offset + size + 8 > ?2)", + params![ + segment_id, + sqlite_i64("recovered segment boundary", max_offset)? + ], + )?; + Self::prune_unowned_segment_payloads(&tx)?; + tx.commit()?; Ok(()) } @@ -7682,6 +7989,7 @@ impl Index { /// # Errors /// /// Returns [`StagingError::Internal`] on SQLite failure. + #[cfg(test)] pub fn locate_pending(&self, chunk_hash: &[u8; 32]) -> Result> { let hash_slice: &[u8] = chunk_hash; self.conn @@ -7703,135 +8011,44 @@ impl Index { .map_err(|e| StagingError::Internal(format!("failed to locate pending chunk: {e}"))) } - /// Batch locator lookup across `chunks` and `pending_chunks`. - /// - /// Issues bounded `WHERE chunk_hash IN (?, ?, ...)` queries per table - /// instead of one round-trip per hash. Returns locators in the same - /// order as `hashes`, with `None` for chunks absent from both tables. - /// - /// Committed rows in `chunks` take precedence over pending rows for - /// the same hash — matching the fallback order in `get_chunk`. - /// - /// # Errors - /// - /// Returns [`StagingError::Internal`] on SQLite failure. + /// Resolve requested segment payloads in bounded batches, preserving order and repeats. pub fn locate_batch(&self, hashes: &[[u8; 32]]) -> Result>> { - if hashes.is_empty() { - return Ok(Vec::new()); - } - - // Bucket input indices by chunk_hash. Duplicate hashes in the - // input all get the same locator. - let mut index_by_hash: std::collections::HashMap<[u8; 32], Vec> = - std::collections::HashMap::with_capacity(hashes.len()); - for (i, h) in hashes.iter().enumerate() { - index_by_hash.entry(*h).or_default().push(i); + let mut positions = HashMap::<[u8; 32], Vec>::with_capacity(hashes.len()); + for (position, hash) in hashes.iter().enumerate() { + positions.entry(*hash).or_default().push(position); } - - let unique_hashes = index_by_hash.keys().copied().collect::>(); - let mut out: Vec> = vec![None; hashes.len()]; - const LOCATE_QUERY_BATCH: usize = 400; - - // Query committed chunks first. - for hash_batch in unique_hashes.chunks(LOCATE_QUERY_BATCH) { - let placeholders = vec!["?"; hash_batch.len()].join(","); + let unique = positions.keys().copied().collect::>(); + let mut out = vec![None; hashes.len()]; + for batch in unique.chunks(400) { + let placeholders = vec!["?"; batch.len()].join(","); let sql = format!( "SELECT chunk_hash, segment_id, segment_offset, size - FROM chunks - WHERE chunk_hash IN ({placeholders})" + FROM chunk_payloads WHERE chunk_hash IN ({placeholders})" ); - let mut stmt = self - .conn - .prepare(&sql) - .map_err(|e| StagingError::Internal(format!("prepare locate_batch: {e}")))?; - let rows = stmt - .query_map( - params_from_iter(hash_batch.iter().map(|h| h.as_slice())), - |row| { - let blob: Vec = row.get(0)?; - let seg_id: u64 = row.get(1)?; - let offset: u64 = row.get(2)?; - let length: u32 = row.get(3)?; - Ok((blob, seg_id, offset, length)) - }, - ) - .map_err(|e| StagingError::Internal(format!("query locate_batch: {e}")))?; - - for row in rows { - let (blob, seg_id, offset, length) = - row.map_err(|e| StagingError::Internal(format!("read locate_batch row: {e}")))?; - if blob.len() != 32 { - continue; - } - let mut h = [0u8; 32]; - h.copy_from_slice(&blob); - if let Some(indices) = index_by_hash.get(&h) { - let loc = ChunkLocator { - segment_id: seg_id, - offset, - length, - }; - for &i in indices { - out[i] = Some(loc); - } - } - } - } - - // Query pending chunks for hashes still missing. - let missing_hashes: Vec<[u8; 32]> = index_by_hash - .iter() - .filter(|(_, indices)| indices.iter().any(|&i| out[i].is_none())) - .map(|(h, _)| *h) - .collect(); - - for hash_batch in missing_hashes.chunks(LOCATE_QUERY_BATCH) { - let pending_placeholders = vec!["?"; hash_batch.len()].join(","); - let sql = format!( - "SELECT chunk_hash, segment_id, segment_offset, size - FROM pending_chunks - WHERE chunk_hash IN ({pending_placeholders})" - ); - let mut stmt = self.conn.prepare(&sql).map_err(|e| { - StagingError::Internal(format!("prepare locate_batch pending: {e}")) - })?; - let rows = stmt - .query_map( - params_from_iter(hash_batch.iter().map(|h| h.as_slice())), - |row| { - let blob: Vec = row.get(0)?; - let seg_id: u64 = row.get(1)?; - let offset: u64 = row.get(2)?; - let length: u32 = row.get(3)?; - Ok((blob, seg_id, offset, length)) - }, - ) - .map_err(|e| StagingError::Internal(format!("query locate_batch pending: {e}")))?; - + let mut statement = self.conn.prepare_cached(&sql)?; + let rows = statement.query_map( + params_from_iter(batch.iter().map(|hash| hash.as_slice())), + |row| { + Ok(( + row.get::<_, Vec>(0)?, + ChunkLocator { + segment_id: row.get(1)?, + offset: row.get(2)?, + length: row.get(3)?, + }, + )) + }, + )?; for row in rows { - let (blob, seg_id, offset, length) = row.map_err(|e| { - StagingError::Internal(format!("read locate_batch pending row: {e}")) - })?; - if blob.len() != 32 { - continue; - } - let mut h = [0u8; 32]; - h.copy_from_slice(&blob); - if let Some(indices) = index_by_hash.get(&h) { - let loc = ChunkLocator { - segment_id: seg_id, - offset, - length, - }; - for &i in indices { - if out[i].is_none() { - out[i] = Some(loc); - } + let (hash, locator) = row?; + let hash = decode_hash_blob("segment payload hash", hash)?; + if let Some(indices) = positions.get(&hash) { + for &position in indices { + out[position] = Some(locator); } } } } - Ok(out) } @@ -7954,6 +8171,27 @@ impl Index { .collect::>(); let segment_locators = self.locate_batch(&hashes)?; let prepared_locators = self.locate_prepared_batch(&hashes)?; + // Released push-plan repair repacks from surviving segment occurrences + // when a prepared body is corrupt. Preserve that preference; otherwise + // use prepared source groups without mistaking retained inventory for rows. + let mut occurrence_backed = HashSet::new(); + for batch in hashes.chunks(400) { + let placeholders = (1..=batch.len()) + .map(|position| format!("?{position}")) + .collect::>() + .join(","); + let mut statement = self.conn.prepare_cached(&format!( + "SELECT chunk_hash FROM chunks WHERE chunk_hash IN ({placeholders}) + UNION SELECT chunk_hash FROM pending_chunks WHERE chunk_hash IN ({placeholders})" + ))?; + let rows = statement.query_map( + params_from_iter(batch.iter().map(|hash| hash.as_slice())), + |row| row.get::<_, Vec>(0), + )?; + for row in rows { + occurrence_backed.insert(decode_hash_blob("segment occurrence hash", row?)?); + } + } let tx = self.conn.unchecked_transaction().map_err(|error| { StagingError::Internal(format!( "failed to begin coalesced residual request append: {error}" @@ -7979,7 +8217,9 @@ impl Index { for (((sequence, hash, expected_size, context), segment), prepared) in requests.iter().zip(segment_locators).zip(prepared_locators) { - if let Some(locator) = segment { + if let Some(locator) = segment + && (prepared.is_none() || occurrence_backed.contains(hash)) + { let actual_size = u64::from(locator.length); if actual_size != *expected_size { return Err(StagingError::StagingCorrupt(format!( @@ -8324,83 +8564,19 @@ impl Index { Ok(()) } - /// Classify requested chunk occurrences by existing payload locator or absence. - /// - /// Committed locators take precedence over pending locators. Every requested - /// occurrence is retained, including repeated hashes and cross-file reuse. + /// Classify every requested occurrence through the canonical segment payload inventory. pub fn batch_dedup_check(&self, hashes: &[(usize, [u8; 32])]) -> Result { - let mut indices_by_hash: std::collections::HashMap<[u8; 32], Vec> = - std::collections::HashMap::with_capacity(hashes.len()); - for (index, hash) in hashes { - indices_by_hash.entry(*hash).or_default().push(*index); - } - let mut existing_by_hash = std::collections::HashMap::<[u8; 32], ChunkLocator>::new(); - - // Membership is recorded separately for every occurrence. Computing an - // unused per-file membership flag here made each reused chunk scan the - // growing destination file, turning repeated-file ingestion quadratic. - for table in ["chunks", "pending_chunks"] { - let missing = indices_by_hash - .keys() - .filter(|hash| !existing_by_hash.contains_key(*hash)) - .copied() - .collect::>(); - for batch in missing.chunks(400) { - let placeholders = vec!["?"; batch.len()].join(","); - let sql = format!( - "SELECT picked.chunk_hash, c.segment_id, c.segment_offset, c.size - FROM ( - SELECT chunk_hash, MIN(rowid) AS picked_rowid - FROM {table} - WHERE chunk_hash IN ({placeholders}) - GROUP BY chunk_hash - ) picked - INNER JOIN {table} c ON c.rowid = picked.picked_rowid" - ); - let mut statement = self.conn.prepare(&sql).map_err(|error| { - StagingError::Internal(format!("prepare {table} dedup lookup: {error}")) - })?; - let rows = statement - .query_map( - params_from_iter(batch.iter().map(|hash| hash.as_slice())), - |row| { - Ok(( - row.get::<_, Vec>(0)?, - ChunkLocator { - segment_id: row.get(1)?, - offset: row.get(2)?, - length: row.get(3)?, - }, - )) - }, - ) - .map_err(|error| { - StagingError::Internal(format!("query {table} dedup lookup: {error}")) - })?; - for row in rows { - let (blob, locator) = row.map_err(|error| { - StagingError::Internal(format!("read {table} dedup locator: {error}")) - })?; - let mut hash = [0; 32]; - if blob.len() == 32 { - hash.copy_from_slice(&blob); - } - existing_by_hash.insert(hash, locator); - } - } - } - - let new_indices = hashes - .iter() - .filter_map(|(index, hash)| (!existing_by_hash.contains_key(hash)).then_some(*index)) - .collect(); + let requested = hashes.iter().map(|(_, hash)| *hash).collect::>(); + let locators = self.locate_batch(&requested)?; let mut existing = Vec::new(); - for (hash, locator) in existing_by_hash { - if let Some(indices) = indices_by_hash.get(&hash) { - existing.extend(indices.iter().map(|index| (*index, hash, locator))); + let mut missing = Vec::new(); + for ((position, hash), locator) in hashes.iter().zip(locators) { + match locator { + Some(locator) => existing.push((*position, *hash, locator)), + None => missing.push(*position), } } - Ok((existing, new_indices)) + Ok((existing, missing)) } /// Return sealed segment ids whose dead-byte ratio exceeds `dead_ratio`. @@ -8413,7 +8589,7 @@ impl Index { /// /// Returns [`StagingError::Internal`] on SQLite failure. pub fn compaction_candidates(&self, dead_ratio: f64) -> Result> { - // Compute live_bytes per segment from the chunks table, then + // Count each retained physical payload once, then // compare against segments.size_bytes. let mut stmt = self .conn @@ -8422,7 +8598,7 @@ impl Index { FROM segments s LEFT JOIN ( SELECT segment_id, SUM(size + 8) AS live_bytes - FROM chunks + FROM chunk_payloads GROUP BY segment_id ) c ON s.segment_id = c.segment_id WHERE s.sealed_at IS NOT NULL @@ -8447,162 +8623,104 @@ impl Index { Ok(ids) } - /// Return live chunks for a segment, ordered by segment offset. - /// - /// Each tuple is `(chunk_hash, file_hash, chunk_index, size, segment_offset)`. - /// - /// # Errors - /// - /// Returns [`StagingError::Internal`] on SQLite failure. - #[expect( - clippy::type_complexity, - reason = "tuple matches the design's specified return shape" - )] + /// Return canonical segment payloads once each, ordered by physical offset. pub fn live_chunks_for_segment( &self, segment_id: u64, - ) -> Result> { - let mut stmt = self - .conn - .prepare( - "SELECT chunk_hash, file_hash, chunk_index, size, segment_offset - FROM chunks - WHERE segment_id = ?1 - ORDER BY segment_offset ASC", - ) - .map_err(|e| { - StagingError::Internal(format!( - "failed to prepare live_chunks query for segment {segment_id}: {e}" - )) - })?; - - let rows = stmt - .query_map(params![segment_id], |row| { - let ch_blob: Vec = row.get(0)?; - let fh_blob: Vec = row.get(1)?; - let mut ch = [0u8; 32]; - let mut fh = [0u8; 32]; - if ch_blob.len() == 32 { - ch.copy_from_slice(&ch_blob); - } - if fh_blob.len() == 32 { - fh.copy_from_slice(&fh_blob); - } - Ok(( - ch, - fh, - row.get::<_, i64>(2)?, - row.get::<_, i64>(3)?, - row.get::<_, u64>(4)?, - )) - }) - .map_err(|e| { - StagingError::Internal(format!( - "failed to query live chunks for segment {segment_id}: {e}" - )) - })? - .collect::, _>>() - .map_err(|e| { - StagingError::Internal(format!( - "failed to collect live chunks for segment {segment_id}: {e}" - )) - })?; + ) -> Result> { + let mut statement = self.conn.prepare_cached( + "SELECT chunk_hash, segment_id, segment_offset, size FROM chunk_payloads + WHERE segment_id = ?1 ORDER BY segment_offset", + )?; + let rows = statement.query_map(params![segment_id], |row| { + Ok(( + row.get::<_, Vec>(0)?, + ChunkLocator { + segment_id: row.get(1)?, + offset: row.get(2)?, + length: row.get(3)?, + }, + )) + })?; + rows.map(|row| { + let (hash, locator) = row?; + Ok((decode_hash_blob("compacted chunk hash", hash)?, locator)) + }) + .collect() + } - Ok(rows) + /// Reserve compaction output without committing an extra current segment. + pub fn begin_compaction(&self) -> Result<(rusqlite::Transaction<'_>, u64)> { + let tx = self.conn.unchecked_transaction()?; + tx.execute("INSERT INTO segments (size_bytes) VALUES (0)", [])?; + let id = nonnegative_count("compaction segment id", tx.last_insert_rowid())?; + Ok((tx, id)) } - /// Atomically swap chunk locators from an old segment to a new one. - /// - /// In a single transaction: updates each chunk's `segment_id` and - /// `segment_offset`, inserts the new segment row, and zeroes the old - /// segment's `live_chunk_count` (marking it for sweep). - /// - /// # Errors - /// - /// Returns [`StagingError::Internal`] on SQLite failure. - #[expect(clippy::cast_possible_wrap, reason = "sizes fit in i64 for SQLite")] + /// Move physical authority and all segment occurrence locators together. pub fn swap_locators( - &self, + tx: rusqlite::Transaction<'_>, old_segment_id: u64, new_segment_id: u64, new_size_bytes: u64, - updates: &[([u8; 32], [u8; 32], i64, u64)], // (chunk_hash, file_hash, chunk_index, new_offset) + updates: &[([u8; 32], u64)], ) -> Result<()> { - let tx = self.conn.unchecked_transaction().map_err(|e| { - StagingError::Internal(format!("failed to begin swap_locators tx: {e}")) - })?; - - // Update each chunk's locator to point at the new segment. - { - let mut stmt = tx - .prepare_cached( - "UPDATE chunks - SET segment_id = ?1, segment_offset = ?2 - WHERE chunk_hash = ?3 AND file_hash = ?4 AND chunk_index = ?5", - ) - .map_err(|e| { - StagingError::Internal(format!("failed to prepare swap update: {e}")) - })?; - - for (chunk_hash, file_hash, chunk_index, new_offset) in updates { - let ch: &[u8] = chunk_hash; - let fh: &[u8] = file_hash; - stmt.execute(params![ + for (chunk_hash, new_offset) in updates { + let moved = tx.execute( + "UPDATE chunk_payloads SET segment_id = ?1, segment_offset = ?2 + WHERE chunk_hash = ?3 AND segment_id = ?4", + params![ new_segment_id, - *new_offset as i64, - ch, - fh, - chunk_index, - ]) - .map_err(|e| { - StagingError::Internal(format!("failed to swap locator for chunk: {e}")) - })?; + sqlite_i64("compacted record offset", *new_offset)?, + chunk_hash.as_slice(), + old_segment_id + ], + )?; + if moved != 1 { + return Err(StagingError::StagingCorrupt( + "compaction lost canonical payload authority".to_owned(), + )); + } + for table in ["chunks", "pending_chunks"] { + tx.execute( + &format!( + "UPDATE {table} SET segment_id = ?1, segment_offset = ?2 + WHERE chunk_hash = ?3 AND segment_id = ?4" + ), + params![ + new_segment_id, + sqlite_i64("compacted record offset", *new_offset)?, + chunk_hash.as_slice(), + old_segment_id + ], + )?; } } - - // Insert the new segment as sealed. - let chunk_count = updates.len() as i64; + let remaining: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM chunk_payloads WHERE segment_id = ?1)", + params![old_segment_id], + |row| row.get(0), + )?; + if remaining { + return Err(StagingError::StagingCorrupt( + "compaction did not cover every retained payload".to_owned(), + )); + } tx.execute( - "UPDATE segments - SET sealed_at = datetime('now'), - size_bytes = ?1, - chunk_count = ?2, - live_chunk_count = ?3 - WHERE segment_id = ?4", + "UPDATE segments SET sealed_at = datetime('now'), size_bytes = ?1, chunk_count = ?2, + live_chunk_count = (SELECT COUNT(*) FROM chunks WHERE segment_id = ?3) + WHERE segment_id = ?3", params![ - new_size_bytes as i64, - chunk_count, - chunk_count, + sqlite_i64("compacted segment bytes", new_size_bytes)?, + updates.len(), new_segment_id ], - ) - .map_err(|e| { - StagingError::Internal(format!( - "failed to update new segment {new_segment_id}: {e}" - )) - })?; - - // Zero out the old segment's live_chunk_count so it becomes a sweep candidate. + )?; tx.execute( - "UPDATE segments SET live_chunk_count = 0 WHERE segment_id = ?1", + "DELETE FROM segments WHERE segment_id = ?1", params![old_segment_id], - ) - .map_err(|e| { - StagingError::Internal(format!( - "failed to zero live_chunk_count for old segment {old_segment_id}: {e}" - )) - })?; - - tx.commit().map_err(|e| { - StagingError::Internal(format!("failed to commit swap_locators tx: {e}")) - })?; - - debug!( - old_segment_id, - new_segment_id, - chunks = updates.len(), - "swapped locators" - ); + )?; + tx.commit()?; Ok(()) } @@ -8740,7 +8858,9 @@ impl Index { AND NOT EXISTS ( SELECT 1 FROM publication_intent_entries AS entry WHERE entry.batch_id = batch.batch_id - )", + ) + AND NOT EXISTS (SELECT 1 FROM external_publications AS external + WHERE external.batch_id = batch.batch_id)", )?, open_push_snapshots: count("SELECT COUNT(*) FROM push_snapshots WHERE state = 'open'")?, committed_push_snapshots: count( @@ -8943,6 +9063,253 @@ mod tests { assert_eq!(recipes.get(&missing_hash), Some(&None)); } + #[test] + fn delegation_freezes_recipe_membership_and_recording_terms() { + let idx = open_in_memory(); + let path = b"model.bin"; + let original = insert_test_recipe_lease(&idx, "original", path, 0x71, 0x72); + let replacement = insert_test_recipe_lease(&idx, "replacement", path, 0x73, 0x74); + idx.delegate_publication(&[("original".to_owned(), path.to_vec())]) + .unwrap(); + let replace = idx.insert_recipe_lease( + "original", + path, + &replacement, + RecipeVerification::CallerVerified, + ); + let append = idx.append_recipe_recording_terms( + "original", + 0, + 0, + &[(crab_xet::hash::MerkleHash::from(test_hash(0x74)), 8)], + ); + let leases = idx.delegated_publications().unwrap(); + assert!( + replace.is_err() + && append.is_err() + && leases.len() == 1 + && leases[0].file_hash == original.file_hash() + ); + } + + #[test] + fn delegation_rejects_unfinished_recording_and_active_preparation() { + for preparation in [false, true] { + let idx = open_in_memory(); + let recipe = insert_test_recipe_lease(&idx, "unfinished", b"model", 0x75, 0x76); + if preparation { + idx.insert_add_preparation("active").unwrap(); + idx.attach_add_preparation_batch("active", "unfinished") + .unwrap(); + } else { + idx.append_recipe_recording_terms( + "unfinished", + 0, + 0, + &[(crab_xet::hash::MerkleHash::from(test_hash(0x76)), 8)], + ) + .unwrap(); + } + let handed = idx.delegate_publication(&[("unfinished".to_owned(), b"model".to_vec())]); + assert!( + handed.is_err() + && idx.delegated_publications().unwrap().is_empty() + && idx.has_file_owner(&recipe.file_hash().into()).unwrap() + ); + } + } + + #[test] + fn delegation_requires_complete_batch_handoff_and_publication_consumes_markers() { + let idx = open_in_memory(); + let first = insert_test_recipe_lease(&idx, "shared", b"first", 0x81, 0x82); + let second = insert_test_recipe_lease(&idx, "source", b"second", 0x83, 0x84); + idx.insert_recipe_lease( + "shared", + b"second", + &second, + RecipeVerification::CallerVerified, + ) + .unwrap(); + let partial = idx.delegate_publication(&[("shared".to_owned(), b"first".to_vec())]); + assert!(partial.is_err() && idx.delegated_publications().unwrap().is_empty()); + idx.delegate_publication(&[ + ("shared".to_owned(), b"first".to_vec()), + ("shared".to_owned(), b"second".to_vec()), + ]) + .unwrap(); + idx.mark_batch_published("shared").unwrap(); + assert!( + idx.delegated_publications().unwrap().is_empty() + && idx + .published_recipe_for_file(&first.file_hash().into()) + .unwrap() + == Some(first) + && idx + .published_recipe_for_file(&second.file_hash().into()) + .unwrap() + == Some(second) + ); + } + + #[test] + fn delegated_listing_rejects_partial_or_unverified_ownership() { + for unverified in [false, true] { + let idx = open_in_memory(); + let recipe = insert_test_recipe_lease(&idx, "shared", b"first", 0x91, 0x92); + if unverified { + idx.conn + .execute( + "DELETE FROM verified_recipes WHERE recipe_hash = ?1", + params![recipe.hash().as_slice()], + ) + .unwrap(); + } else { + idx.insert_recipe_lease( + "shared", + b"second", + &recipe, + RecipeVerification::CallerVerified, + ) + .unwrap(); + } + idx.conn.execute("INSERT INTO external_publications (batch_id, path_bytes) VALUES ('shared', ?1)", + params![b"first".as_slice()]).unwrap(); + assert!(idx.delegated_publications().is_err()); + } + } + + #[test] + fn external_snapshot_retires_only_captured_handoffs_and_keeps_native_owners() { + let idx = open_in_memory(); + let path = b"model.bin"; + let recipe = insert_test_recipe_lease(&idx, "external", path, 0x51, 0x52); + let file_hash: [u8; 32] = recipe.file_hash().into(); + idx.delegate_publication(&[("external".to_owned(), path.to_vec())]) + .unwrap(); + idx.insert_batch("native").unwrap(); + idx.insert_recipe_lease("native", path, &recipe, RecipeVerification::CallerVerified) + .unwrap(); + assert!(idx.published_recipe_for_file(&file_hash).unwrap().is_none()); + assert_eq!( + idx.external_recipes_for_files(&[file_hash]).unwrap()[&file_hash], + Some(recipe.clone()) + ); + idx.create_push_snapshot("first", std::slice::from_ref(&recipe)) + .unwrap(); + idx.insert_batch("late").unwrap(); + idx.insert_recipe_lease("late", path, &recipe, RecipeVerification::CallerVerified) + .unwrap(); + idx.delegate_publication(&[("late".to_owned(), path.to_vec())]) + .unwrap(); + idx.commit_push_snapshot("first").unwrap(); + let retired = idx.retire_push_snapshot("first").unwrap(); + let leases: Vec = idx + .conn + .prepare("SELECT batch_id FROM path_leases ORDER BY batch_id") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert!(retired.is_empty() && leases == ["late", "native"]); + idx.rollback_batch("native").unwrap(); + idx.create_push_snapshot("second", std::slice::from_ref(&recipe)) + .unwrap(); + idx.commit_push_snapshot("second").unwrap(); + assert_eq!(idx.retire_push_snapshot("second").unwrap(), vec![file_hash]); + } + + #[test] + fn external_delegation_is_atomic_and_cannot_share_a_native_intent() { + let idx = open_in_memory(); + let path = b"model.bin"; + let recipe = insert_test_recipe_lease(&idx, "native", path, 0x61, 0x62); + idx.create_publication_intent( + "native-intent", + &[( + "native".to_owned(), + path.to_vec(), + "oid".to_owned(), + "prior".to_owned(), + )], + ) + .unwrap(); + idx.insert_batch("external").unwrap(); + idx.insert_recipe_lease( + "external", + path, + &recipe, + RecipeVerification::CallerVerified, + ) + .unwrap(); + let failed = idx.delegate_publication(&[ + ("external".to_owned(), path.to_vec()), + ("native".to_owned(), path.to_vec()), + ]); + let count: i64 = idx + .conn + .query_row("SELECT count(*) FROM external_publications", [], |row| { + row.get(0) + }) + .unwrap(); + assert!( + failed.is_err() + && count == 0 + && idx.unresolved_publication_intents().unwrap().len() == 1 + ); + idx.delegate_publication(&[("external".to_owned(), path.to_vec())]) + .unwrap(); + assert!( + idx.create_publication_intent( + "wrong-native", + &[( + "external".to_owned(), + path.to_vec(), + "oid".to_owned(), + "prior".to_owned(), + )] + ) + .is_err() + ); + } + + #[test] + fn publishing_a_path_preserves_another_callers_open_lease() { + for through_intent in [false, true] { + let idx = open_in_memory(); + let path = b"model.bin"; + let pending = insert_test_recipe_lease(&idx, "pending", path, 0x31, 0x32); + let current = insert_test_recipe_lease(&idx, "current", path, 0x41, 0x42); + let retired = if through_intent { + idx.create_publication_intent( + "current-intent", + &[( + "current".to_owned(), + path.to_vec(), + "current-oid".to_owned(), + "prior".to_owned(), + )], + ) + .unwrap(); + idx.publish_publication_intent("current-intent").unwrap() + } else { + idx.mark_batch_published("current").unwrap() + }; + let pending_hash: [u8; 32] = pending.file_hash().into(); + let current_hash: [u8; 32] = current.file_hash().into(); + let retained = idx.has_file_owner(&pending_hash).unwrap(); + let released = idx.rollback_batch("pending").unwrap(); + assert!( + retired.is_empty() + && retained + && released == vec![pending_hash] + && idx.published_recipe_for_file(¤t_hash).unwrap() == Some(current), + "publication must leave another active caller's lease for that caller to release" + ); + } + } + #[test] fn publication_intent_atomically_replaces_canonical_path_head() { let idx = open_in_memory(); @@ -9056,6 +9423,49 @@ mod tests { ); } + #[test] + fn failed_prepared_reuse_claim_does_not_leave_retention() { + for insertion_failure in [false, true] { + let idx = open_in_memory(); + idx.insert_add_preparation("reuse").expect("preparation"); + idx.insert_batch("reuse-batch").expect("batch"); + idx.attach_add_preparation_batch("reuse", "reuse-batch") + .expect("attach"); + let xorb = test_hash(0xA1); + idx.conn.execute("INSERT INTO prepared_payloads (xorb_hash, payload_hash, bytes) VALUES (?1, ?2, 16)", params![xorb.as_slice(), test_hash(0xA2).as_slice()]).expect("prepared payload"); + for (position, chunk) in [test_hash(0xA3), test_hash(0xA4)].iter().enumerate() { + idx.conn.execute("INSERT INTO prepared_payload_chunks (xorb_hash, chunk_index, chunk_hash, uncompressed_size) VALUES (?1, ?2, ?3, 8)", params![xorb.as_slice(), position, chunk.as_slice()]).expect("prepared placement"); + } + if insertion_failure { + idx.conn.execute_batch("CREATE TEMP TRIGGER reject_reuse_retention BEFORE INSERT ON preparation_payloads BEGIN SELECT RAISE(ABORT, 'retention failure'); END;").expect("failure injection"); + } + let error = idx + .claim_prepared_chunks( + "reuse", + "reuse-batch", + &[(test_hash(0xA3), 8), (test_hash(0xA4), 9)], + ) + .expect_err("claim fails"); + let leases: i64 = idx + .conn + .query_row( + "SELECT COUNT(*) FROM preparation_payloads WHERE preparation_id = 'reuse'", + [], + |row| row.get(0), + ) + .expect("retention count"); + let expected_error = if insertion_failure { + matches!(error, StagingError::Io(ref source) if source.get_ref().is_some_and(|source| source.is::())) + } else { + matches!(error, StagingError::StagingCorrupt(_)) + }; + assert!( + expected_error && leases == 0, + "failed claim must roll back retention and preserve the cause" + ); + } + } + #[test] fn repeated_chunk_claim_has_one_winning_occurrence() { let idx = open_in_memory(); @@ -9317,45 +9727,6 @@ mod tests { assert_eq!(counts, (1, 1, 1)); } - #[test] - fn published_replacement_removes_superseded_empty_open_batch() { - let idx = open_in_memory(); - let path = b"models/prepared.bin"; - let prepared = insert_test_recipe_lease(&idx, "batch-prepared", path, 0x31, 0x32); - idx.insert_batch("batch-published") - .expect("published batch"); - idx.insert_recipe_lease( - "batch-published", - path, - &prepared, - RecipeVerification::CallerVerified, - ) - .expect("published lease"); - - assert!( - idx.mark_batch_published("batch-published") - .expect("publish prepared replacement") - .is_empty(), - "the published owner keeps the same prepared recipe live" - ); - - let prepared_batch_exists: bool = idx - .conn - .query_row( - "SELECT EXISTS(SELECT 1 FROM staging_batches WHERE batch_id = 'batch-prepared')", - [], - |row| row.get(0), - ) - .expect("prepared batch existence"); - assert!(!prepared_batch_exists); - let published_file_hash: [u8; 32] = prepared.file_hash().into(); - assert_eq!( - idx.published_recipe_for_file(&published_file_hash) - .expect("published recipe"), - Some(prepared) - ); - } - #[test] fn rollback_preserves_snapshot_pin_and_rejects_published_batch() { let idx = open_in_memory(); @@ -9422,6 +9793,176 @@ mod tests { ); } + #[test] + fn segment_recording_retention_releases_only_after_last_batch() { + let idx = open_in_memory(); + let file = test_hash(0xB1); + let chunk = test_hash(0xB2); + let segment = idx.allocate_segment_id().expect("segment"); + insert_test_file(&idx, &file, 8); + idx.insert_pending(&[PendingRow { + chunk_hash: chunk, + file_hash: file, + chunk_index: 0, + size: 8, + segment_id: segment, + segment_offset: 0, + }]) + .expect("payload"); + for batch in ["first", "second"] { + idx.insert_batch(batch).expect("batch"); + idx.append_recipe_recording_terms(batch, 0, 0, &[(chunk.into(), 8)]) + .expect("record"); + } + idx.remove_files(&[file]).expect("retire original"); + idx.rollback_batch("first").expect("retire first recording"); + let retained = idx.locate(&chunk).expect("lookup retained"); + idx.rollback_batch("second") + .expect("retire second recording"); + assert!(retained.is_some() && idx.locate(&chunk).expect("lookup retired").is_none()); + } + + #[test] + fn incomplete_compaction_rolls_back_inventory_and_occurrences() { + let idx = open_in_memory(); + let file = test_hash(0xB3); + let old = idx.allocate_segment_id().expect("old segment"); + insert_test_file(&idx, &file, 16); + let chunks = [test_hash(0xB4), test_hash(0xB5)]; + for (position, hash) in chunks.iter().enumerate() { + idx.insert_pending(&[PendingRow { + chunk_hash: *hash, + file_hash: file, + chunk_index: position as i64, + size: 8, + segment_id: old, + segment_offset: position as u64 * 16, + }]) + .expect("payload"); + } + let before = idx.locate_batch(&chunks).expect("before"); + let (transaction, new) = idx.begin_compaction().expect("reserve output"); + let error = Index::swap_locators(transaction, old, new, 16, &[(chunks[0], 0)]) + .expect_err("partial coverage"); + let pending_segment: u64 = idx + .conn + .query_row( + "SELECT segment_id FROM pending_chunks WHERE chunk_hash = ?1", + params![chunks[0].as_slice()], + |row| row.get(0), + ) + .expect("occurrence"); + assert!( + matches!(error, StagingError::StagingCorrupt(_)) + && idx.locate_batch(&chunks).expect("after") == before + && pending_segment == old + ); + } + + #[test] + fn released_compaction_inventory_upgrades_on_writable_and_shared_open() { + for shared in [false, true] { + let temp = tempfile::NamedTempFile::new().expect("database"); + let idx = Index::open(temp.path()).expect("index"); + insert_test_recipe_lease(&idx, "source", b"source", 0xB6, 0xB7); + let chunk = test_hash(0xB7); + let old = idx + .locate(&chunk) + .expect("old locator") + .expect("payload") + .segment_id; + let new = idx.allocate_segment_id().expect("new segment"); + // The released compactor updated chunks, then removed the old + // segment, leaving payload inventory at its original locator. + idx.conn + .execute( + "UPDATE chunks SET segment_id = ?1, segment_offset = 16 WHERE chunk_hash = ?2", + params![new, chunk.as_slice()], + ) + .expect("released relocation"); + idx.drop_segment(old).expect("released retirement"); + idx.conn + .execute_batch("DROP TABLE external_publications;") + .unwrap(); + idx.conn + .execute( + "UPDATE staging_meta SET value = '1' WHERE key = 'layout_version'", + [], + ) + .expect("released admission"); + drop(idx); + let reopened = if shared { + Index::open_readonly(temp.path()) + } else { + Index::open(temp.path()) + } + .expect("upgrade"); + let locator = reopened + .locate(&chunk) + .expect("repaired locator") + .expect("payload"); + let version: String = reopened + .conn + .query_row( + "SELECT value FROM staging_meta WHERE key = 'layout_version'", + [], + |row| row.get(0), + ) + .expect("admission"); + assert!(locator.segment_id == new && locator.offset == 16 && version == LAYOUT_VERSION); + } + } + + #[test] + fn failed_inventory_upgrade_preserves_released_admission_and_locator() { + for fail_late in [false, true] { + let mut idx = open_in_memory(); + insert_test_recipe_lease(&idx, "source", b"source", 0xB8, 0xB9); + let chunk = test_hash(0xB9); + let before = idx.locate(&chunk).expect("before"); + idx.conn + .execute( + "UPDATE chunks SET segment_offset = 16 WHERE chunk_hash = ?1", + params![chunk.as_slice()], + ) + .expect("released relocation"); + idx.conn + .execute_batch("DROP TABLE external_publications;") + .unwrap(); + idx.conn + .execute( + "UPDATE staging_meta SET value = '1' WHERE key = 'layout_version'", + [], + ) + .expect("released admission"); + let table = if fail_late { + "staging_meta" + } else { + "chunk_payloads" + }; + idx.conn.execute_batch(&format!("CREATE TEMP TRIGGER reject_upgrade BEFORE UPDATE ON {table} BEGIN SELECT RAISE(ABORT, 'upgrade failure'); END;")).expect("inject failure"); + let error = idx.open_canonical_schema().expect_err("upgrade fails"); + let version: String = idx + .conn + .query_row( + "SELECT value FROM staging_meta WHERE key = 'layout_version'", + [], + |row| row.get(0), + ) + .expect("admission"); + assert!( + matches!(error, StagingError::Io(ref source) if source.get_ref().is_some_and(|source| source.is::())) + && version == "1" + && idx.locate(&chunk).expect("after") == before + && !idx + .application_objects("table") + .unwrap() + .iter() + .any(|name| name == "external_publications") + ); + } + } + #[test] fn canonical_schema_reopens_without_change() { let tmp = tempfile::NamedTempFile::new().expect("tempfile"); @@ -9442,6 +9983,21 @@ mod tests { assert_eq!(version, LAYOUT_VERSION); } + #[test] + fn canonical_schema_rejects_external_handoffs_without_lease_cascade() { + let temp = tempfile::NamedTempFile::new().unwrap(); + let idx = Index::open(temp.path()).unwrap(); + idx.conn + .execute_batch( + "DROP TABLE external_publications; + CREATE TABLE external_publications(batch_id TEXT NOT NULL, path_bytes BLOB NOT NULL, + PRIMARY KEY(batch_id, path_bytes));", + ) + .unwrap(); + drop(idx); + assert_staging_corrupt_contains(Index::open(temp.path()), "canonical v3"); + } + #[test] fn canonical_schema_rejects_missing_required_index() { let tmp = tempfile::NamedTempFile::new().expect("tempfile"); @@ -9453,7 +10009,7 @@ mod tests { .expect("drop chunk lease index"); drop(idx); - assert_staging_corrupt_contains(Index::open(path), "canonical v1"); + assert_staging_corrupt_contains(Index::open(path), "canonical v3"); } #[test] @@ -9479,7 +10035,7 @@ mod tests { .expect("install retired publication table"); drop(idx); - assert_staging_corrupt_contains(Index::open(path), "canonical v1"); + assert_staging_corrupt_contains(Index::open(path), "canonical v3"); } #[test] @@ -9496,7 +10052,7 @@ mod tests { .expect("install retired remote authority table"); drop(idx); - assert_staging_corrupt_contains(Index::open(path), "canonical v1"); + assert_staging_corrupt_contains(Index::open(path), "canonical v3"); } #[test] @@ -10048,7 +10604,7 @@ mod tests { .expect("create retired schema"); } - assert_staging_corrupt_contains(Index::open(&path), "canonical v1"); + assert_staging_corrupt_contains(Index::open(&path), "canonical v3"); let connection = Connection::open(&path).expect("reopen retired db"); let tables: Vec = connection diff --git a/crates/crab-staging/src/lib.rs b/crates/crab-staging/src/lib.rs index f9f1185b1..025bbbc30 100644 --- a/crates/crab-staging/src/lib.rs +++ b/crates/crab-staging/src/lib.rs @@ -1,6 +1,6 @@ //! Segment-based staging area for chunks awaiting upload. //! -//! Canonical layout (v1): +//! Canonical layout (v3, with transactional upgrade from released v1): //! ```text //! {root}/ //! ├── segments/ @@ -44,6 +44,7 @@ mod index; pub mod metrics; pub mod multipart_resume; pub mod push_plan; +mod push_snapshot; pub mod recipe; mod recovery; mod segment; @@ -69,6 +70,7 @@ mod prop_torn_tail_recovery; pub use config::StagingConfig; pub use error::{Result, StagingError}; pub use metrics::StagingMetrics; +pub use push_snapshot::StagingPushSnapshot; pub use stats::{ CompactionStats, RetireStats, StagingCleanStats, StagingLifecycleHealth, StagingStats, StagingVerifyStats, @@ -92,7 +94,7 @@ use self::index::{ ExistingChunkWrite, FileChunkLocator, Index, IndexedCoalescedReadGroup, PendingRow, PreparedChunkLocator, PreparedXorbPlacementWrite, PreparedXorbWrite, RecipeVerification, }; -use self::segment::{ChunkLocator, PreparedRecord, ReaderPool, SegmentWriter}; +use self::segment::{ChunkLocator, PreparedRecord, ReaderPool, SegmentReader, SegmentWriter}; /// Default flush threshold in bytes (256 MiB). When pending bytes exceed /// this, the current durability boundary is recorded or the segment is sealed. @@ -165,6 +167,15 @@ pub struct PublicationIntentEntry { pub previous_index_state: String, } +/// Immutable external lease identity for Git-aware maintenance recovery. +#[derive(Debug, Clone)] +pub struct DelegatedPublication { + pub batch_id: StagingBatchId, + pub path_bytes: Vec, + pub file_hash: MerkleHash, + pub size: u64, +} + /// Durable unresolved publication state returned for product-layer recovery. #[derive(Debug, Clone)] pub struct UnresolvedPublicationIntent { @@ -247,6 +258,97 @@ fn unresolved_publication_intents( .collect() } +fn publish_verified_recipe_owner( + root: &Path, + index: &Mutex, + owner: &Path, + recipe: &crate::recipe::FileRecipe, + display_path: Option<&str>, +) -> Result { + let batch_id = new_staging_batch_id(); + let file_hash: [u8; 32] = recipe.file_hash().into(); + let result = (|| { + let guard = lock_index(index)?; + guard.insert_batch(batch_id.as_str())?; + guard.insert_file(&file_hash, recipe.file_size())?; + guard.insert_recipe_lease( + batch_id.as_str(), + &staging_path_bytes(owner), + recipe, + RecipeVerification::CallerVerified, + )?; + if let Some(display_path) = display_path { + guard.insert_file_path(&file_hash, display_path)?; + } + guard.mark_batch_published(batch_id.as_str()) + })(); + if let Err(error) = result.and_then(|unleased| remove_indexed_files(root, index, &unleased)) { + let unowned = lock_index(index).and_then(|guard| { + guard.rollback_batch(batch_id.as_str())?; + guard.has_file_owner(&file_hash).map(|owned| !owned) + }); + if matches!(unowned, Ok(true)) { + let _ = remove_indexed_files(root, index, &[file_hash]); + } + return Err(error); + } + Ok(batch_id) +} + +fn preserve_publication_history( + root: &Path, + index: &Mutex, + intent_id: &PublicationIntentId, + recipe_hashes: &[MerkleHash], + cancel: &tokio_util::sync::CancellationToken, +) -> Result<()> { + if cancel.is_cancelled() { + return Err(StagingError::Cancelled); + } + let intent = unresolved_publication_intents(index)? + .into_iter() + .find(|intent| &intent.intent_id == intent_id) + .ok_or_else(|| StagingError::NotFound { + path: format!("publication intent {}", intent_id.as_str()), + })?; + let members = intent + .entries + .iter() + .map(|entry| entry.recipe_hash) + .collect::>(); + let recipes = { + let guard = lock_index(index)?; + recipe_hashes + .iter() + .copied() + .collect::>() + .into_iter() + .map(|hash| { + if !members.contains(&hash) { + return Err(StagingError::StagingCorrupt( + "history recipe is not owned by publication intent".to_owned(), + )); + } + guard.verified_local_recipe(&hash.into())?.ok_or_else(|| { + StagingError::StagingCorrupt( + "publication history recipe is not verified".to_owned(), + ) + }) + }) + .collect::>>()? + }; + // Commit every history owner before rollback can remove native leases. + // An interruption leaves the original intent and its remaining payloads owned. + for recipe in recipes { + if cancel.is_cancelled() { + return Err(StagingError::Cancelled); + } + let owner = PathBuf::from(".crab/history").join(recipe.file_hash().hex()); + publish_verified_recipe_owner(root, index, &owner, &recipe, None)?; + } + Ok(()) +} + #[cfg(unix)] fn staging_path_bytes(path: &Path) -> Vec { use std::os::unix::ffi::OsStrExt; @@ -1379,47 +1481,42 @@ async fn read_staged_chunks_batch( let hash_bytes: Vec<[u8; 32]> = hashes.iter().map(|h| <[u8; 32]>::from(*h)).collect(); - let (locators, prepared_locators) = { + let (segment_chunks, segment_positions, prepared_chunks, pinned_groups) = { let idx = lock_index(index)?; - ( - idx.locate_batch(&hash_bytes)?, - idx.locate_prepared_batch(&hash_bytes)?, - ) - }; - - for (i, (locator, prepared)) in locators.iter().zip(&prepared_locators).enumerate() { - if locator.is_none() && prepared.is_none() { - return Err(StagingError::ChunkNotFound { - hash: hashes[i].hex(), - }); - } - } - - let mut segment_chunks = Vec::new(); - let mut segment_positions = Vec::new(); - let mut prepared_chunks = Vec::new(); - for (position, ((hash, locator), prepared)) in hashes - .iter() - .zip(locators) - .zip(prepared_locators) - .enumerate() - { - if let Some(locator) = locator { - segment_positions.push(position); - segment_chunks.push(StagedChunkLocator { - hash: *hash, - size: u64::from(locator.length), - locator, - }); - } else if let Some(locator) = prepared { - prepared_chunks.push((position, *hash, locator)); + let locators = idx.locate_batch(&hash_bytes)?; + let prepared_locators = idx.locate_prepared_batch(&hash_bytes)?; + let mut segment_chunks = Vec::new(); + let mut segment_positions = Vec::new(); + let mut prepared_chunks = Vec::new(); + for (position, ((hash, locator), prepared)) in hashes + .iter() + .zip(locators) + .zip(prepared_locators) + .enumerate() + { + if let Some(locator) = locator { + segment_positions.push(position); + segment_chunks.push(StagedChunkLocator { + hash: *hash, + size: u64::from(locator.length), + locator, + }); + } else if let Some(locator) = prepared { + prepared_chunks.push((position, *hash, locator)); + } else { + return Err(StagingError::ChunkNotFound { hash: hash.hex() }); + } } - } + // Capture file handles before releasing locator ownership. A compact + // or sweep can unlink the old name, but cannot invalidate these reads. + let groups = pin_segment_read_groups(readers, &segment_chunks)?; + (segment_chunks, segment_positions, prepared_chunks, groups) + }; let mut out = vec![None; hashes.len()]; for (position, (_, data)) in segment_positions .into_iter() - .zip(read_located_staged_chunks_batch(readers, metrics, &segment_chunks).await?) + .zip(read_located_staged_chunks_batch(pinned_groups, metrics, &segment_chunks).await?) { out[position] = Some(data); } @@ -1555,7 +1652,12 @@ async fn read_coalesced_chunk_group( .iter() .map(|(_, locator)| *locator) .collect::>(); - let payloads = read_located_staged_chunks_batch(readers, metrics, &locators).await?; + let payloads = read_located_staged_chunks_batch( + pin_segment_read_groups(readers, &locators)?, + metrics, + &locators, + ) + .await?; Ok(chunks .into_iter() .zip(payloads) @@ -1711,15 +1813,12 @@ async fn complete_recipe_payload_validation( lock_index(index)?.finish_recipe_payload_validation() } -async fn read_located_staged_chunks_batch( +type SegmentReadGroup = (SegmentReader, Vec<(usize, u64, u32)>); + +fn pin_segment_read_groups( readers: &ReaderPool, - metrics: Option<&dyn StagingMetrics>, chunks: &[StagedChunkLocator], -) -> Result> { - if chunks.is_empty() { - return Ok(Vec::new()); - } - +) -> Result> { let mut by_segment: HashMap> = HashMap::new(); for (i, chunk) in chunks.iter().enumerate() { let loc = chunk.locator; @@ -1732,10 +1831,20 @@ async fn read_located_staged_chunks_batch( group.sort_by_key(|&(_, offset, _)| offset); } + by_segment + .into_iter() + .map(|(segment_id, group)| Ok((readers.get(segment_id)?, group))) + .collect() +} + +async fn read_located_staged_chunks_batch( + groups: Vec, + metrics: Option<&dyn StagingMetrics>, + chunks: &[StagedChunkLocator], +) -> Result> { let mut out: Vec> = (0..chunks.len()).map(|_| None).collect(); - let mut tasks = Vec::with_capacity(by_segment.len()); - for (segment_id, group) in by_segment { - let reader = readers.get(segment_id)?; + let mut tasks = Vec::with_capacity(groups.len()); + for (reader, group) in groups { tasks.push(tokio::task::spawn_blocking(move || { reader.read_many_sorted(&group) })); @@ -2329,9 +2438,16 @@ impl StagingArea { .locate_prepared_batch(&[hash_bytes])? .first() .is_some_and(Option::is_some); - (locator, prepared_exists) + let located = locator + .map(|locator| { + self.readers + .get(locator.segment_id) + .map(|reader| (locator, reader)) + }) + .transpose()?; + (located, prepared_exists) }; - let Some(locator) = locator else { + let Some((locator, reader)) = locator else { if !prepared_exists { return Ok(None); } @@ -2341,9 +2457,6 @@ impl StagingArea { .map(|mut chunks| chunks.pop().map(|(_, data)| data)); }; - // Get a reader from the pool. - let reader = self.readers.get(locator.segment_id)?; - // pread + CRC verification in spawn_blocking. let offset = locator.offset; let length = locator.length; @@ -2613,6 +2726,24 @@ impl StagingArea { Ok(()) } + /// Persist a completed lease handoff without claiming that Git accepted it. + /// + /// Payloads must already be flushed, and every path in a batch must be handed off. + /// Delegation freezes batch membership and excludes subsequent native + /// index intents; an observed Git pointer permits push snapshot capture. + pub fn delegate_publication(&self, entries: &[(&StagingBatchId, &Path)]) -> Result<()> { + let entries = entries + .iter() + .map(|(batch, path)| (batch.as_str().to_owned(), staging_path_bytes(path))) + .collect::>(); + lock_index(&self.index)?.delegate_publication(&entries) + } + + /// List immutable delegated leases for explicit maintenance reconciliation. + pub fn delegated_publications(&self) -> Result> { + lock_index(&self.index)?.delegated_publications() + } + /// Mark a batch as published after its Git index replacement commits. pub fn mark_batch_published(&self, batch_id: &StagingBatchId) -> Result<()> { let unleased = lock_index(&self.index)?.mark_batch_published(batch_id.as_str())?; @@ -2669,6 +2800,16 @@ impl StagingArea { unresolved_publication_intents(&self.index) } + /// Preserve caller-attested committed recipe roots before rolling back a native intent. + pub fn preserve_publication_history( + &self, + intent_id: &PublicationIntentId, + recipe_hashes: &[MerkleHash], + cancel: &tokio_util::sync::CancellationToken, + ) -> Result<()> { + preserve_publication_history(&self.root, &self.index, intent_id, recipe_hashes, cancel) + } + /// Atomically remove an unresolved intent and every open batch it owns. pub fn rollback_publication_intent( &self, @@ -2688,31 +2829,13 @@ impl StagingArea { path: &Path, recipe: &crate::recipe::FileRecipe, ) -> Result { - self.publish_verified_recipe_owner(path, recipe, Some(&path.to_string_lossy())) - } - - fn publish_verified_recipe_owner( - &self, - owner: &Path, - recipe: &crate::recipe::FileRecipe, - display_path: Option<&str>, - ) -> Result { - let batch_id = self.create_batch()?; - let file_hash = recipe.file_hash(); - let publish = (|| -> Result<()> { - self.pre_register_file(&file_hash, recipe.file_size())?; - self.record_verified_recipe_lease(&batch_id, owner, recipe)?; - if let Some(display_path) = display_path { - self.record_file_path(&file_hash, display_path)?; - } - self.mark_batch_published(&batch_id) - })(); - if let Err(error) = publish { - let _ = self.rollback_batch(&batch_id); - let _ = self.retire_file_if_unleased(&file_hash); - return Err(error); - } - Ok(batch_id) + publish_verified_recipe_owner( + &self.root, + &self.index, + path, + recipe, + Some(&path.to_string_lossy()), + ) } /// Publish one immutable recipe needed by not-yet-pushed Git history. @@ -2725,7 +2848,7 @@ impl StagingArea { recipe: &crate::recipe::FileRecipe, ) -> Result { let owner = PathBuf::from(".crab/history").join(recipe.file_hash().hex()); - self.publish_verified_recipe_owner(&owner, recipe, None) + publish_verified_recipe_owner(&self.root, &self.index, &owner, recipe, None) } /// Preserve a locally published recipe referenced by not-yet-pushed Git history. @@ -2807,7 +2930,10 @@ impl StagingArea { /// Return the ordered list of chunk hashes staged for a given file. pub fn chunks_for_file(&self, file_hash: &MerkleHash) -> Result> { - if let Some(recipe) = self.published_recipe_for_file(file_hash)? { + // Ordered occurrence visibility follows retained versions, including + // native/delegated preparations. It does not attest Git publication. + let recipe = lock_index(&self.index)?.retained_recipe_for_file(&(*file_hash).into())?; + if let Some(recipe) = recipe { return collect_recipe_chunks(&self.index, &recipe) .map(|chunks| chunks.into_iter().map(|(hash, _)| hash).collect()); } @@ -2821,7 +2947,10 @@ impl StagingArea { &self, file_hash: &MerkleHash, ) -> Result> { - if let Some(recipe) = self.published_recipe_for_file(file_hash)? { + // Ordered occurrence visibility follows retained versions, including + // native/delegated preparations. It does not attest Git publication. + let recipe = lock_index(&self.index)?.retained_recipe_for_file(&(*file_hash).into())?; + if let Some(recipe) = recipe { return collect_recipe_chunks(&self.index, &recipe); } let fh: [u8; 32] = (*file_hash).into(); @@ -3022,8 +3151,8 @@ impl StagingArea { /// Register a file and its chunks in the index. /// - /// For each chunk in the list, verifies it exists in `chunks` or - /// `pending_chunks`, then links the file to those locators in + /// For each chunk in the list, verifies canonical segment authority, + /// then links the file to those locators in /// `chunks`. Segment `live_chunk_count` accounting is updated in /// the same transaction that replaces the chunk rows. /// @@ -3285,34 +3414,28 @@ impl StagingArea { } } - let (current_id, current_write_offset) = { - let writer = self.writer.lock().await; - (writer.segment_id(), writer.write_offset()) - }; - + let mut writer = self.writer.lock().await; + let current_id = writer.segment_id(); + let current_write_offset = writer.write_offset(); let segments_dir = self.root.join("segments"); - - let (candidates, current_is_abandoned) = { - let idx = lock_index(&self.index)?; - let candidates = idx.abandoned_segments(current_id)?; - // The current segment counts as abandoned only when it has - // no locator rows at all and still has bytes on disk. Pending - // rows may back a pointer already written to Git's index, so - // preserve them until retire_file removes them explicitly. - let committed = idx.segment_committed_chunk_count(current_id)?; - let pending = idx.segment_pending_chunk_count(current_id)?; - let (persisted_size, _) = idx.segment_info(current_id).unwrap_or((0, 0)); - let current_size = persisted_size.max(current_write_offset); - let current_is_abandoned = committed == 0 && pending == 0 && current_size > 0; - (candidates, current_is_abandoned) - }; + // Preserve writer -> index lock ordering. Capture and deletion stay in + // one synchronous section so a new claim cannot race reclamation. + let idx = lock_index(&self.index)?; + let candidates = idx.abandoned_segments(current_id)?; + let committed = idx.segment_committed_chunk_count(current_id)?; + let pending = idx.segment_pending_chunk_count(current_id)?; + let current_size = idx.segment_info(current_id)?.0.max(current_write_offset); + let current_is_abandoned = committed == 0 + && pending == 0 + && !idx.segment_has_payloads(current_id)? + && current_size > 0; let mut segments_removed: u64 = 0; let mut bytes_reclaimed: u64 = 0; let mut pending_removed: u64 = 0; for (seg_id, size_bytes) in &candidates { - let n = lock_index(&self.index)?.delete_pending_for_segment(*seg_id)?; + let n = idx.delete_pending_for_segment(*seg_id)?; pending_removed += n; let seg_path = segments_dir.join(format!("{seg_id:016x}.seg")); @@ -3322,7 +3445,7 @@ impl StagingArea { Err(e) => return Err(e.into()), } - lock_index(&self.index)?.drop_segment(*seg_id)?; + idx.drop_segment(*seg_id)?; segments_removed += 1; bytes_reclaimed += size_bytes; @@ -3332,14 +3455,9 @@ impl StagingArea { // locator rows. Keep the writer running by rolling it onto a // fresh segment id with an empty `current.seg`. if current_is_abandoned { - let (old_size, n_pending) = { - let idx = lock_index(&self.index)?; - let persisted_size = idx.segment_info(current_id).map(|(s, _)| s).unwrap_or(0); - let size = persisted_size.max(current_write_offset); - let n = idx.delete_pending_for_segment(current_id)?; - idx.drop_segment(current_id)?; - (size, n) - }; + let old_size = current_size; + let n_pending = idx.delete_pending_for_segment(current_id)?; + idx.drop_segment(current_id)?; // Drop the existing current.seg, reallocate a fresh id // under the writer lock, and rotate the writer onto it. @@ -3350,14 +3468,8 @@ impl StagingArea { Err(e) => return Err(e.into()), } - let new_id = { - let idx = lock_index(&self.index)?; - let id = idx.allocate_segment_id()?; - idx.register_current_segment(id)?; - id - }; - - let mut writer = self.writer.lock().await; + let new_id = idx.allocate_segment_id()?; + idx.register_current_segment(new_id)?; *writer = SegmentWriter::new( &segments_dir, new_id, @@ -3387,10 +3499,7 @@ impl StagingArea { // a previous run that missed the pending cleanup. Safe because // no segment row → no .seg file → no way to ever read those // chunks back. - let orphan_pending = { - let idx = lock_index(&self.index)?; - idx.delete_orphan_pending_rows()? - }; + let orphan_pending = idx.delete_orphan_pending_rows()?; pending_removed += orphan_pending; debug!( @@ -3511,14 +3620,20 @@ impl StagingArea { old_segment_id: u64, segments_dir: &Path, ) -> Result { - let live_chunks = lock_index(&self.index)?.live_chunks_for_segment(old_segment_id)?; + // Keep capture, locator transfer and unlink under one owner lock. + // Writers/retirement cannot invalidate the captured payload set while + // the physical records are copied; no await occurs in this section. + let index = lock_index(&self.index)?; + let live_chunks = index.live_chunks_for_segment(old_segment_id)?; if live_chunks.is_empty() { return Ok(CompactionStats::default()); } // Allocate a new segment id for the compacted output. - let new_segment_id = lock_index(&self.index)?.allocate_segment_id()?; + // Allocation is provisional until durable bytes and every locator move + // commit together. A copy/SQL failure cannot publish a second current. + let (transaction, new_segment_id) = index.begin_compaction()?; // Open the old segment for reading. let old_reader = self.readers.get(old_segment_id)?; @@ -3535,25 +3650,16 @@ impl StagingArea { .open(&tmp_path)?; let mut new_offset: u64 = 0; - let mut updates: Vec<([u8; 32], [u8; 32], i64, u64)> = - Vec::with_capacity(live_chunks.len()); - - for (chunk_hash, file_hash, chunk_index, size, old_offset) in &live_chunks { - #[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - reason = "size is always non-negative and bounded by segment hard cap" - )] - let length = *size as u32; + let mut updates = Vec::with_capacity(live_chunks.len()); - // Read the full record from the old segment (CRC-verified). - let data = old_reader.read(*old_offset, length)?; + for (chunk_hash, locator) in &live_chunks { + let data = old_reader.read(locator.offset, locator.length)?; // Re-encode and write to the new segment. let record = segment::encode_record(&data); std::io::Write::write_all(&mut new_file, &record)?; - updates.push((*chunk_hash, *file_hash, *chunk_index, new_offset)); + updates.push((*chunk_hash, new_offset)); new_offset += record.len() as u64; } @@ -3570,7 +3676,8 @@ impl StagingArea { dir_fd.sync_all()?; // Atomic locator swap in SQLite. - lock_index(&self.index)?.swap_locators( + Index::swap_locators( + transaction, old_segment_id, new_segment_id, new_offset, @@ -3591,9 +3698,6 @@ impl StagingArea { let dir_fd = File::open(segments_dir)?; dir_fd.sync_all()?; - // Drop the old segment from the index (it has 0 live chunks now). - lock_index(&self.index)?.drop_segment(old_segment_id)?; - let bytes_reclaimed = old_size.saturating_sub(new_offset); debug!( @@ -3754,9 +3858,16 @@ impl StagingAreaReadOnly { .locate_prepared_batch(&[hash_bytes])? .first() .is_some_and(Option::is_some); - (locator, prepared_exists) + let located = locator + .map(|locator| { + self.readers + .get(locator.segment_id) + .map(|reader| (locator, reader)) + }) + .transpose()?; + (located, prepared_exists) }; - let Some(locator) = locator else { + let Some((locator, reader)) = locator else { if !prepared_exists { return Ok(None); } @@ -3766,7 +3877,6 @@ impl StagingAreaReadOnly { .map(|mut chunks| chunks.pop().map(|(_, data)| data)); }; - let reader = self.readers.get(locator.segment_id)?; let offset = locator.offset; let length = locator.length; let data = tokio::task::spawn_blocking(move || reader.read(offset, length)) @@ -3841,7 +3951,10 @@ impl StagingAreaReadOnly { /// Return the ordered list of chunk hashes staged for a given file. pub fn chunks_for_file(&self, file_hash: &MerkleHash) -> Result> { - if let Some(recipe) = self.published_recipe_for_file(file_hash)? { + // Ordered occurrence visibility follows retained versions, including + // native/delegated preparations. It does not attest Git publication. + let recipe = lock_index(&self.index)?.retained_recipe_for_file(&(*file_hash).into())?; + if let Some(recipe) = recipe { return collect_recipe_chunks(&self.index, &recipe) .map(|chunks| chunks.into_iter().map(|(hash, _)| hash).collect()); } @@ -3855,7 +3968,10 @@ impl StagingAreaReadOnly { &self, file_hash: &MerkleHash, ) -> Result> { - if let Some(recipe) = self.published_recipe_for_file(file_hash)? { + // Ordered occurrence visibility follows retained versions, including + // native/delegated preparations. It does not attest Git publication. + let recipe = lock_index(&self.index)?.retained_recipe_for_file(&(*file_hash).into())?; + if let Some(recipe) = recipe { return collect_recipe_chunks(&self.index, &recipe); } let fh: [u8; 32] = (*file_hash).into(); @@ -3935,6 +4051,22 @@ impl StagingAreaReadOnly { .collect()) } + /// Load delegated recipes only for the Git pointers selected by the caller. + pub fn external_recipes_for_files( + &self, + file_hashes: &[MerkleHash], + ) -> Result>> { + let hashes = file_hashes + .iter() + .map(|hash| (*hash).into()) + .collect::>(); + Ok(lock_index(&self.index)? + .external_recipes_for_files(&hashes)? + .into_iter() + .map(|(hash, recipe)| (MerkleHash::from(hash), recipe)) + .collect()) + } + /// Read one bounded page from an indexed immutable recipe. pub fn recipe_page( &self, @@ -4087,6 +4219,16 @@ impl StagingAreaReadOnly { Ok(()) } + /// Preserve caller-attested committed recipe roots before rolling back a native intent. + pub fn preserve_publication_history( + &self, + intent_id: &PublicationIntentId, + recipe_hashes: &[MerkleHash], + cancel: &tokio_util::sync::CancellationToken, + ) -> Result<()> { + preserve_publication_history(&self.root, &self.index, intent_id, recipe_hashes, cancel) + } + /// Atomically remove an unresolved intent and every open batch it owns. pub async fn rollback_publication_intent( &self, @@ -4118,6 +4260,10 @@ impl StagingAreaReadOnly { /// Retire the exact recipe ownership pinned by one committed push snapshot. pub async fn retire_push_snapshot(&self, push_id: &str) -> Result> { + self.retire_push_snapshot_now(push_id) + } + + fn retire_push_snapshot_now(&self, push_id: &str) -> Result> { let unleased = lock_index(&self.index)?.retire_push_snapshot(push_id)?; remove_indexed_files(&self.root, &self.index, &unleased) } @@ -4816,6 +4962,369 @@ mod tests { ); } + #[tokio::test] + async fn released_upgrade_and_failed_compaction_preserve_real_payload_reads() { + for fail_commit in [false, true] { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().join("staging"); + let staging = StagingArea::open(root.clone()).await.expect("staging"); + let (hash, data) = make_chunk(123, 4096); + let file = MerkleHash::from(*blake3::hash(&data).as_bytes()); + staging + .pre_register_file(&file, data.len() as u64) + .expect("file"); + staging + .stage_chunks_batch(&[(&hash, data.as_slice())], &file, 0) + .await + .expect("payload"); + staging + .register_file(&file, data.len() as u64, &[(hash, data.len() as u64)]) + .expect("promote"); + let (dead_hash, dead_data) = make_chunk(124, 8192); + let dead_file = MerkleHash::from(*blake3::hash(&dead_data).as_bytes()); + staging + .pre_register_file(&dead_file, dead_data.len() as u64) + .expect("dead file"); + staging + .stage_chunks_batch(&[(&dead_hash, dead_data.as_slice())], &dead_file, 0) + .await + .expect("dead payload"); + let old = lock_index(&staging.index) + .expect("index") + .locate(&hash.into()) + .expect("locator") + .expect("old payload"); + { + let mut writer = staging.writer.lock().await; + seal_current_segment_blocking( + staging.root(), + &staging.index, + &staging.cfg, + None, + &mut writer, + ) + .expect("seal"); + } + staging + .unregister_file(&dead_file) + .expect("retire dead file"); + if fail_commit { + lock_index(&staging.index).expect("index").connection().execute_batch( + "CREATE TEMP TRIGGER reject_compaction BEFORE UPDATE ON chunk_payloads BEGIN SELECT RAISE(ABORT, 'compaction failure'); END;" + ).expect("inject commit failure"); + } + let result = staging.compact_force().await; + let expected_outcome = if fail_commit { + matches!(result, Err(StagingError::Io(ref source)) if source.get_ref().is_some_and(|source| source.is::())) + } else { + result.is_ok_and(|stats| stats.segments_compacted == 1) + }; + let currents = lock_index(&staging.index) + .expect("index") + .all_segments() + .expect("segments") + .into_iter() + .filter(|(_, sealed, _)| sealed.is_none()) + .count(); + staging.close().await.expect("close"); + if !fail_commit { + let connection = + rusqlite::Connection::open(root.join("index.db")).expect("released database"); + connection.execute("UPDATE chunk_payloads SET segment_id = ?1, segment_offset = ?2 WHERE chunk_hash = ?3", rusqlite::params![old.segment_id, old.offset, <[u8; 32]>::from(hash).as_slice()]).expect("released stale inventory"); + connection + .execute_batch("DROP TABLE external_publications;") + .unwrap(); + connection + .execute( + "UPDATE staging_meta SET value = '1' WHERE key = 'layout_version'", + [], + ) + .expect("released admission"); + } + let actual = if fail_commit { + let reopened = StagingArea::open(root.clone()) + .await + .expect("recover failed compaction"); + let bytes = reopened + .get_chunk(&hash) + .await + .expect("read original bytes"); + reopened.close().await.expect("close recovered"); + bytes + } else { + let upgraded = StagingAreaReadOnly::open(root.clone()) + .await + .expect("upgrade shared reader"); + upgraded + .get_chunk(&hash) + .await + .expect("read upgraded bytes") + }; + let sealed_files = std::fs::read_dir(root.join("segments")) + .expect("segments") + .map(|entry| entry.expect("entry").file_name()) + .filter(|name| { + name.to_str() + .is_some_and(|name| name.len() == 20 && name.ends_with(".seg")) + }) + .count(); + assert!( + expected_outcome + && currents == 1 + && sealed_files == 1 + && actual.as_ref().is_some_and(|bytes| bytes.as_ref() == data) + ); + } + } + + #[tokio::test] + async fn segment_reuse_survives_retirement_compaction_and_reopen() { + for (seal_before_retirement, compact) in + [(false, false), (false, true), (true, false), (true, true)] + { + let tmp = tempfile::tempdir().expect("tempdir"); + let source = tmp.path().join("source.bin"); + let content = b"segment reuse lifetime".repeat(64 * 1024); + std::fs::write(&source, &content).expect("source"); + let root = tmp.path().join("staging"); + let staging = StagingArea::open(root.clone()).await.expect("staging"); + let first = stream::stage_file_streaming( + &source, + tmp.path(), + &staging, + stream::StreamStageProgress::default(), + &tokio_util::sync::CancellationToken::new(), + ) + .await + .expect("source recipe"); + if compact { + let mut writer = staging.writer.lock().await; + seal_current_segment_blocking( + staging.root(), + &staging.index, + &staging.cfg, + None, + &mut writer, + ) + .expect("seal source segment"); + } + let term = staging.recipe_page(&first.recipe, 0).expect("page").chunks[0]; + let bytes = &content[term.offset as usize..(term.offset + term.len) as usize]; + let incoming = bytes.repeat(2); + let file_hash = MerkleHash::from(*blake3::hash(&incoming).as_bytes()); + let mut recorder = recipe::RecipeRecorder::new(first.recipe.policy()); + recorder + .record(term.chunk_hash, term.len) + .expect("first occurrence"); + recorder + .record(term.chunk_hash, term.len) + .expect("repeat occurrence"); + let recipe = recorder + .seal(file_hash, incoming.len() as u64) + .expect("recipe"); + let preparation = staging.create_add_preparation().expect("preparation"); + let batch = staging.create_batch().expect("batch"); + staging + .attach_add_preparation_batch(&preparation, &batch) + .expect("attach"); + staging + .append_recipe_recording_terms(&batch, 0, 0, &[(term.chunk_hash, term.len); 2]) + .expect("occurrences"); + let claims = staging + .claim_prepared_chunks(&preparation, &batch, &[(term.chunk_hash, term.len); 2]) + .expect("segment claims"); + if seal_before_retirement { + staging + .pre_register_file(&file_hash, recipe.file_size()) + .expect("file"); + staging + .record_verified_recipe_lease(&batch, Path::new("reuse.bin"), &recipe) + .expect("seal incoming"); + staging + .finalize_add_preparation(&preparation) + .expect("finalize"); + } + staging + .rollback_batch(&first.batch_id) + .expect("retire original"); + let read = staging + .get_chunks_batch(&[term.chunk_hash; 2]) + .await + .expect("read retained payload"); + if !seal_before_retirement { + staging + .pre_register_file(&file_hash, recipe.file_size()) + .expect("file"); + staging + .record_verified_recipe_lease(&batch, Path::new("reuse.bin"), &recipe) + .expect("seal incoming"); + staging + .finalize_add_preparation(&preparation) + .expect("finalize"); + } + staging.mark_batch_published(&batch).expect("publish"); + staging + .clean_abandoned(true) + .await + .expect("preserve owned segment"); + let reader_pool = ReaderPool::new(staging.root().join("segments"), 1); + let locators = { + let index = lock_index(&staging.index).expect("index"); + let locator = index + .locate(&term.chunk_hash.into()) + .expect("locator") + .expect("payload"); + [StagedChunkLocator { + hash: term.chunk_hash, + size: term.len, + locator, + }; 2] + }; + let pinned = pin_segment_read_groups(&reader_pool, &locators).expect("pin readers"); + let compacted = staging.compact_force().await.expect("compact"); + let current_id = staging.writer.lock().await.segment_id(); + reader_pool.get(current_id).expect("evict old pool entry"); + let captured_read = read_located_staged_chunks_batch(pinned, None, &locators) + .await + .expect("read captured bytes after unlink"); + let after_compaction = staging + .get_chunks_batch(&[term.chunk_hash; 2]) + .await + .expect("read compacted"); + staging.close().await.expect("close"); + let reopened = StagingArea::open(root).await.expect("reopen"); + let recovered = reopened + .get_chunks_batch(&[term.chunk_hash; 2]) + .await + .expect("read recovered"); + reopened + .release_published_path(Path::new("reuse.bin"), &file_hash) + .expect("release final owner"); + reopened.sweep_orphans().expect("sweep retired bytes"); + let retired = reopened + .get_chunk(&term.chunk_hash) + .await + .expect("retired lookup"); + reopened.close().await.expect("close recovered"); + assert!( + claims + .iter() + .all(|claim| matches!(claim, index::PreparedChunkClaim::Segment)) + && read.iter().all(|(_, data)| data.as_ref() == bytes) + && after_compaction == read + && captured_read == read + && recovered == read + && compacted.segments_compacted == u64::from(compact) + && retired.is_none() + ); + } + } + + #[tokio::test] + async fn prepared_reuse_survives_original_rollback_until_new_owner_retires() { + for publish in [false, true] { + let tmp = tempfile::tempdir().expect("tempdir"); + let path = tmp.path().join("first.bin"); + let content = b"prepared reuse lifetime".repeat(64 * 1024); + std::fs::write(&path, &content).expect("source file"); + let staging = StagingArea::open(tmp.path().join("staging")) + .await + .expect("staging"); + let first = stream::stage_file_streaming( + &path, + tmp.path(), + &staging, + stream::StreamStageProgress { + xorb_builder: Some(stream::StreamStageXorbBuilder::new( + 1, + crab_xet::xorb::builder::XorbBuilder::new, + )), + ..stream::StreamStageProgress::default() + }, + &tokio_util::sync::CancellationToken::new(), + ) + .await + .expect("first prepared recipe"); + let terms = staging + .recipe_page(&first.recipe, 0) + .expect("recipe page") + .chunks; + let chunks = terms + .iter() + .map(|term| (term.chunk_hash, term.len)) + .collect::>(); + let preparation = staging + .create_add_preparation() + .expect("reusing preparation"); + let batch = staging.create_batch().expect("reusing batch"); + staging + .attach_add_preparation_batch(&preparation, &batch) + .expect("attach batch"); + staging + .append_recipe_recording_terms(&batch, 0, 0, &chunks) + .expect("record occurrences"); + let claims = staging + .claim_prepared_chunks(&preparation, &batch, &chunks) + .expect("claim reusable bytes"); + staging + .rollback_batch(&first.batch_id) + .expect("retire original recipe"); + let hashes = chunks.iter().map(|(hash, _)| *hash).collect::>(); + let retained = staging.get_chunks_batch(&hashes).await; + let sealed_retained = if publish { + let file_hash = first.recipe.file_hash(); + staging + .pre_register_file(&file_hash, first.recipe.file_size()) + .expect("register reused file"); + staging + .record_verified_recipe_lease(&batch, Path::new("second.bin"), &first.recipe) + .expect("seal reused recipe"); + staging + .finalize_add_preparation(&preparation) + .expect("transfer preparation ownership"); + staging + .mark_batch_published(&batch) + .expect("publish reused recipe"); + let sealed = staging + .get_chunks_batch(&hashes) + .await + .expect("published reused bytes"); + let unchanged = sealed == *retained.as_ref().expect("retained reusable bytes"); + staging + .release_published_path(Path::new("second.bin"), &file_hash) + .expect("release published owner"); + unchanged + } else { + staging + .abort_add_preparation(&preparation) + .expect("abort reusing preparation"); + true + }; + let retired = staging + .get_chunk(&hashes[0]) + .await + .expect("read after retirement"); + staging.close().await.expect("close staging"); + let expected = terms + .iter() + .map(|term| &content[term.offset as usize..(term.offset + term.len) as usize]) + .collect::>(); + assert!( + claims + .iter() + .all(|claim| matches!(claim, index::PreparedChunkClaim::Prepared(_))) + && retained + .expect("reused payload must survive original rollback") + .iter() + .map(|(_, bytes)| bytes.as_ref()) + .collect::>() + == expected + && sealed_retained + && retired.is_none() + ); + } + } + fn prepared_plan_for_first_chunk( file_hash: MerkleHash, total_bytes: u64, diff --git a/crates/crab-staging/src/push_snapshot.rs b/crates/crab-staging/src/push_snapshot.rs new file mode 100644 index 000000000..2437a4634 --- /dev/null +++ b/crates/crab-staging/src/push_snapshot.rs @@ -0,0 +1,70 @@ +use std::sync::Arc; + +use crate::{Result, StagingAreaReadOnly, recipe::FileRecipe}; + +/// Own one push reader's marker and exact immutable recipe snapshot. +#[must_use = "retain the snapshot through remote publication"] +pub struct StagingPushSnapshot { + staging: Arc, + push_id: String, + committed: bool, +} + +impl StagingPushSnapshot { + /// Fence retirement before selecting recipes for a uniquely identified push. + /// + /// The caller must supply a unique push ID. + /// Failure or drop preserves their original leases; another reader can defer cleanup. + pub fn begin(staging: Arc, push_id: String) -> Result { + let snapshot = Self { + staging, + push_id, + committed: false, + }; + snapshot.staging.mark_push_inflight(&snapshot.push_id)?; + Ok(snapshot) + } + + /// Pin published/delegated recipes selected from verified Git pointers. + pub fn capture(&self, recipes: &[FileRecipe]) -> Result<()> { + self.staging.create_push_snapshot(&self.push_id, recipes) + } + + /// Record confirmed remote publication and retire only captured local owners. + /// + /// The caller must establish remote commit before calling this method. Contention + /// leaves a committed journal for exclusive maintenance; cleanup errors retain it. + pub fn commit(mut self) -> Result<()> { + // Remote commit is already authoritative. Even if the local transition + // fails, Drop must retain its journal rather than treat it as a failed push. + self.committed = true; + self.staging.commit_push_snapshot(&self.push_id)?; + let Some(_retirement) = self.staging.begin_retirement(&self.push_id)? else { + return Ok(()); + }; + self.staging.retire_push_snapshot_now(&self.push_id)?; + self.staging.sweep_orphans()?; + self.staging.remove_push_snapshot(&self.push_id) + } +} + +impl Drop for StagingPushSnapshot { + fn drop(&mut self) { + if !self.committed { + let discard = (|| -> Result<()> { + // A sibling may still read a version this snapshot alone pins. + // Exclusive maintenance consumes the journal if cleanup is deferred. + if let Some(_retirement) = self.staging.begin_retirement(&self.push_id)? { + self.staging.discard_open_push_snapshot_now(&self.push_id)?; + } + Ok(()) + })(); + if let Err(error) = discard { + tracing::warn!(push_id = %self.push_id, %error, "staging snapshot cleanup needs maintenance"); + } + } + if let Err(error) = self.staging.clear_push_inflight(&self.push_id) { + tracing::warn!(push_id = %self.push_id, %error, "staging push marker cleanup failed"); + } + } +} diff --git a/crates/crab-staging/src/recovery.rs b/crates/crab-staging/src/recovery.rs index 8b8d83424..6f7e5e135 100644 --- a/crates/crab-staging/src/recovery.rs +++ b/crates/crab-staging/src/recovery.rs @@ -31,7 +31,7 @@ pub struct RecoveredSegment { /// committed boundary and actual file length, fsync when truncated, and /// delete pending rows that no longer fit. /// -/// Finally, removes any orphan `current.seg.tmp` files. +/// Finally, removes orphan temporary and unindexed sealed segment bodies. /// /// Returns `Some(RecoveredSegment)` if an existing current segment was /// found and recovered, or `None` if no current segment existed (fresh @@ -60,6 +60,7 @@ pub fn recover(root: &Path, index: &Index) -> Result> { } cleanup_orphan_temps(&segments_dir)?; + cleanup_orphan_segments(&segments_dir, index)?; debug!("crash recovery complete"); Ok(recovered) @@ -170,6 +171,46 @@ fn cleanup_orphan_temps(segments_dir: &Path) -> Result<()> { } } +fn cleanup_orphan_segments(segments_dir: &Path, index: &Index) -> Result<()> { + let owned = index + .all_segments()? + .into_iter() + .map(|(id, _, _)| id) + .collect::>(); + let mut removed = false; + // Compaction can crash after installing a body but before its SQL commit, + // or after transferring ownership but before unlink. Under the exclusive + // open lock, unnamed-by-index sealed bodies cannot belong to a writer. + for entry in fs::read_dir(segments_dir)? { + let entry = entry?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let Some(identity) = name.strip_suffix(".seg").filter(|identity| { + identity.len() == 16 && identity.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) else { + continue; + }; + let Ok(id) = u64::from_str_radix(identity, 16) else { + continue; + }; + if !owned.contains(&id) { + if index.segment_has_payloads(id)? { + return Err(StagingError::StagingCorrupt(format!( + "payload inventory references unregistered segment {id:016x}" + ))); + } + fs::remove_file(entry.path())?; + removed = true; + } + } + if removed { + fs::File::open(segments_dir)?.sync_all()?; + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -184,6 +225,29 @@ mod tests { (root, index) } + #[test] + fn orphan_cleanup_never_deletes_a_referenced_unregistered_body() { + let temp = tempfile::tempdir().expect("tempdir"); + let (root, index) = setup_staging(&temp); + let file = [1; 32]; + index.insert_file(&file, 8).expect("file"); + index + .insert_pending(&[PendingRow { + chunk_hash: [2; 32], + file_hash: file, + chunk_index: 0, + size: 8, + segment_id: 123, + segment_offset: 0, + }]) + .expect("referenced inventory"); + let body = root.join("segments/000000000000007b.seg"); + fs::write(&body, [0; 16]).expect("body"); + let error = cleanup_orphan_segments(&root.join("segments"), &index) + .expect_err("missing registry is corruption"); + assert!(matches!(error, StagingError::StagingCorrupt(_)) && body.exists()); + } + #[cfg(unix)] #[test] fn current_lookup_error_preserves_pending_rows_and_boundary() { diff --git a/crates/crab-staging/src/stream.rs b/crates/crab-staging/src/stream.rs index 35458ee40..f304afe84 100644 --- a/crates/crab-staging/src/stream.rs +++ b/crates/crab-staging/src/stream.rs @@ -1,4 +1,4 @@ -//! Bounded-memory local-file staging shared by `crab add` and `crab adopt`. +//! Bounded-memory byte ingestion with native descriptor validation at its boundary. //! //! The staging schema keys chunk rows by file hash. Local files are read once //! into a provisional staging key, then the staged rows are adopted under the @@ -10,7 +10,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering::Relaxed}; use std::time::{Duration, Instant}; use bytes::{Bytes, BytesMut}; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncRead, AsyncReadExt}; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc}; use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; @@ -640,6 +640,94 @@ async fn stage_file_streaming_inner( }); } let mut read_file = tokio::fs::File::from_std(descriptor.try_clone()?); + let result = stage_reader_inner( + &mut read_file, + abs_path, + &rel_path, + staging, + progress, + cancel, + hooks, + |file_hash, size| { + let after_descriptor_stat = VerifiedIndexStat::from_file(&descriptor); + let after_path_stat = VerifiedIndexStat::from_path_no_follow(abs_path); + if after_descriptor_stat != Some(before_stream_stat) + || after_path_stat != after_descriptor_stat + { + return Err(CrabError::FileChangedDuringStaging { + path: abs_path.display().to_string(), + first_hash: file_hash.hex(), + second_hash: "unread".to_owned(), + first_size: size, + second_size: after_path_stat.map_or(0, |stat| stat.len), + }); + } + Ok(after_descriptor_stat.filter(|stat| stat.len == size)) + }, + ) + .await; + match result { + Ok(mut result) => { + result.duration_ms = start.elapsed().as_millis() as u64; + Ok(result) + } + Err(CrabError::FileChangedDuringStaging { + first_hash, + first_size, + .. + }) => { + let second = stream_hash_file(abs_path, None, cancel) + .await + .map_err(|error| with_path(abs_path, error))?; + Err(CrabError::FileChangedDuringStaging { + path: abs_path.display().to_string(), + first_hash, + second_hash: MerkleHash::from(second.hash).hex(), + first_size, + second_size: second.size, + }) + } + Err(error) => Err(error), + } +} + +/// Stage supplied bytes without opening, seeking or trusting a source path. +/// +/// `path` identifies the recipe lease only. The returned version has durable +/// payloads and an open batch; the caller owns publication or rollback. +/// `abs_path` preserves this logical identity and `index_stat` is always absent. +/// Cancellation must use the token while awaiting the operation so cleanup runs. +pub async fn stage_reader_streaming( + reader: &mut R, + path: &Path, + staging: &StagingArea, + progress: StreamStageProgress, + cancel: &CancellationToken, +) -> Result { + stage_reader_inner( + reader, + path, + path, + staging, + progress, + cancel, + StreamStageHooks::default(), + |_, _| Ok(None), + ) + .await +} + +async fn stage_reader_inner( + reader: &mut R, + abs_path: &Path, + rel_path: &Path, + staging: &StagingArea, + progress: StreamStageProgress, + cancel: &CancellationToken, + hooks: StreamStageHooks, + validate: impl FnOnce(MerkleHash, u64) -> Result>, +) -> Result { + let start = Instant::now(); let batch_id = staging.create_batch()?; let mut xorb_builder = progress.xorb_builder.clone(); let owned_preparation = if let Some(builder) = xorb_builder.as_mut() @@ -659,7 +747,7 @@ async fn stage_file_streaming_inner( let _ = staging.rollback_batch(&batch_id); return Err(with_path(abs_path, error)); } - let provisional_merkle = provisional_file_hash(&rel_path, &batch_id); + let provisional_merkle = provisional_file_hash(rel_path, &batch_id); let retired = staging.retire_file(&provisional_merkle)?; if retired.rows_deleted > 0 { debug!( @@ -676,7 +764,7 @@ async fn stage_file_streaming_inner( } staging.pre_register_file_with_path(&provisional_merkle, 0, &rel_path.to_string_lossy())?; let stage_result = stream_chunk_and_stage( - &mut read_file, + reader, &provisional_merkle, staging, &batch_id, @@ -723,30 +811,21 @@ async fn stage_file_streaming_inner( return Err(with_path(abs_path, error)); } }; - let after_descriptor_stat = VerifiedIndexStat::from_file(&descriptor); - let after_path_stat = VerifiedIndexStat::from_path_no_follow(abs_path); - if after_descriptor_stat != Some(before_stream_stat) || after_path_stat != after_descriptor_stat - { - fail_stream_preparation(staging, xorb_builder.as_ref(), owned_preparation.as_ref()); - cleanup_staged_file( - staging, - &provisional_merkle, - abs_path, - "file changed during stream staging", - ); - cleanup_stream_prepared_xorb_dir(staging.root(), &provisional_merkle, abs_path); - let _ = staging.rollback_batch(&batch_id); - let second = stream_hash_file(abs_path, None, cancel) - .await - .map_err(|e| with_path(abs_path, e))?; - return Err(CrabError::FileChangedDuringStaging { - path: abs_path.display().to_string(), - first_hash: file_merkle.hex(), - second_hash: MerkleHash::from(second.hash).hex(), - first_size: stage_stats.size, - second_size: second.size, - }); - } + let index_stat = match validate(file_merkle, stage_stats.size) { + Ok(stat) => stat, + Err(error) => { + fail_stream_preparation(staging, xorb_builder.as_ref(), owned_preparation.as_ref()); + cleanup_staged_file( + staging, + &provisional_merkle, + abs_path, + "source validation failed", + ); + cleanup_stream_prepared_xorb_dir(staging.root(), &provisional_merkle, abs_path); + let _ = staging.rollback_batch(&batch_id); + return Err(with_path(abs_path, error)); + } + }; // A renamed or copied path can resolve entirely to proven remote chunks // while the same canonical recipe is still indexed locally. Its partial @@ -821,7 +900,16 @@ async fn stage_file_streaming_inner( return Err(with_path(abs_path, err)); } - if let Err(err) = staging.record_verified_recipe_lease(&batch_id, &rel_path, &recipe) { + // A supplied stream cannot be reopened after its pointer is returned. + // Flush segment authority before sealing the caller-owned version lease. + if let Err(error) = staging.flush_pending().await { + fail_stream_preparation(staging, xorb_builder.as_ref(), owned_preparation.as_ref()); + let _ = staging.retire_file_if_unleased(&file_merkle); + let _ = staging.rollback_batch(&batch_id); + return Err(with_path(abs_path, error)); + } + + if let Err(err) = staging.record_verified_recipe_lease(&batch_id, rel_path, &recipe) { if let Some(builder) = &xorb_builder { builder.fail_preparation(); } @@ -849,14 +937,6 @@ async fn stage_file_streaming_inner( return Err(with_path(abs_path, err)); } - let index_stat = match after_descriptor_stat { - Some(after) if after.len == stage_stats.size => Some(after), - _ => { - debug!(path = %rel_path.display(), "leaving Git stat cache unpopulated"); - None - } - }; - Ok(StreamStageResult { batch_id, abs_path: abs_path.to_path_buf(), @@ -980,8 +1060,8 @@ pub async fn stream_hash_file( clippy::too_many_arguments, reason = "streaming state is explicit to keep chunking and staging allocation-free" )] -async fn stream_chunk_and_stage( - file: &mut tokio::fs::File, +async fn stream_chunk_and_stage( + file: &mut R, file_hash: &MerkleHash, staging: &StagingArea, batch_id: &crate::StagingBatchId, @@ -1049,8 +1129,8 @@ async fn stream_chunk_and_stage( clippy::too_many_arguments, reason = "producer state stays explicit while writer lifecycle is owned by its caller" )] -async fn stream_chunk_and_stage_producer( - file: &mut tokio::fs::File, +async fn stream_chunk_and_stage_producer( + file: &mut R, file_hash: &MerkleHash, staging: &StagingArea, batch_id: &crate::StagingBatchId, @@ -1087,7 +1167,19 @@ async fn stream_chunk_and_stage_producer( loop { check_cancelled(cancel)?; let mut buf = BytesMut::with_capacity(READ_BUF_SIZE); - let n = file.read_buf(&mut buf).await?; + // Short packet/prefix reads must not retain a full allocation per + // fragment in zero-copy chunks. Fill one bounded block before CDC. + while buf.len() < READ_BUF_SIZE { + let n = tokio::select! { + biased; + () = cancel.cancelled() => return Err(CrabError::Cancelled), + read = file.read_buf(&mut buf) => read?, + }; + if n == 0 { + break; + } + } + let n = buf.len(); if n == 0 { break; } @@ -2253,6 +2345,138 @@ mod tests { assert_eq!(bytes, STAGE_BATCH_CHUNKS as u64); } + #[tokio::test] + async fn cancellation_interrupts_pending_stream_read_and_retires_attempt() { + use std::future::{Future, poll_fn}; + use std::pin::Pin; + use std::task::{Context, Poll}; + use tokio::io::ReadBuf; + + struct StalledReader { + started: Arc, + supplied_prefix: bool, + } + impl AsyncRead for StalledReader { + fn poll_read( + mut self: Pin<&mut Self>, + _: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + if !self.supplied_prefix { + buffer.put_slice(&[17; 4096]); + self.supplied_prefix = true; + return Poll::Ready(Ok(())); + } + self.started.store(true, Relaxed); + Poll::Pending + } + } + let root = tempfile::tempdir().expect("staging root"); + let staging = StagingArea::open(root.path().join("staging")) + .await + .expect("staging"); + let started = Arc::new(AtomicBool::new(false)); + let mut reader = StalledReader { + started: started.clone(), + supplied_prefix: false, + }; + let cancel = CancellationToken::new(); + let result = { + let stage = stage_reader_streaming( + &mut reader, + Path::new("stream.bin"), + &staging, + StreamStageProgress::default(), + &cancel, + ); + tokio::pin!(stage); + poll_fn(|context| { + assert!( + stage.as_mut().poll(context).is_pending(), + "reader must still be waiting" + ); + if started.load(Relaxed) { + Poll::Ready(()) + } else { + Poll::Pending + } + }) + .await; + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(1), stage) + .await + .expect("cancellation must wake the read") + }; + let health = staging.lifecycle_health().expect("health"); + let files = staging.list_files().expect("files"); + staging.close().await.expect("close"); + assert!( + matches!(result, Err(CrabError::Cancelled)) + && health.open_batches_without_publication == 0 + && files.is_empty() + ); + } + + #[tokio::test] + async fn nonseeking_stream_returns_durable_recipe_without_source_metadata() { + use tokio::io::AsyncWriteExt; + + for prepared in [false, true] { + let root = tempfile::tempdir().expect("staging root"); + let logical_path = root.path().join("never-created.bin"); + let staging_path = root.path().join("staging"); + let staging = StagingArea::open(staging_path.clone()) + .await + .expect("staging"); + let input = (0..2 * READ_BUF_SIZE) + .map(|index| ((index * 41) ^ (index >> 8)) as u8) + .collect::>(); + let expected = input.clone(); + let (mut writer, mut reader) = tokio::io::duplex(4096); + let source = tokio::spawn(async move { + writer.write_all(&input).await.expect("pipe write"); + writer.shutdown().await.expect("pipe EOF"); + }); + let result = stage_reader_streaming( + &mut reader, + &logical_path, + &staging, + StreamStageProgress { + xorb_builder: prepared + .then(|| StreamStageXorbBuilder::new(1, XorbBuilder::new)), + ..StreamStageProgress::default() + }, + &CancellationToken::new(), + ) + .await + .expect("ingest pipe"); + source.await.expect("source task"); + let hashes = recipe_pairs(&staging, &result) + .into_iter() + .map(|(hash, _)| hash) + .collect::>(); + // The API itself promises durability; do not call close's flush. + drop(staging); + let reopened = StagingArea::open(staging_path) + .await + .expect("recover returned version"); + let actual = reopened + .get_chunks_batch(&hashes) + .await + .expect("durable bytes") + .into_iter() + .flat_map(|(_, data)| data) + .collect::>(); + reopened.close().await.expect("close recovered"); + assert!( + actual == expected + && result.file_hash == *blake3::hash(&expected).as_bytes() + && result.index_stat.is_none() + && !logical_path.exists() + ); + } + } + #[tokio::test] async fn empty_file_produces_empty_hash_and_zero_chunks() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/crab-staging/tests/publication_history_recovery.rs b/crates/crab-staging/tests/publication_history_recovery.rs new file mode 100644 index 000000000..7afe7de79 --- /dev/null +++ b/crates/crab-staging/tests/publication_history_recovery.rs @@ -0,0 +1,128 @@ +//! Committed history ownership must outlive rollback of its native index intent. + +#![expect(clippy::unwrap_used, reason = "test assertions")] + +use std::path::Path; +use std::sync::Arc; + +use crab_staging::stream::{StreamStageProgress, StreamStageXorbBuilder, stage_reader_streaming}; +use crab_staging::{ + PublicationIntentEntry, StagingArea, StagingAreaReadOnly, StagingError, StagingPushSnapshot, +}; +use crab_xet::hash::MerkleHash; +use tokio_util::sync::CancellationToken; + +#[tokio::test(flavor = "multi_thread")] +async fn committed_history_preserves_only_selected_intent_recipes_through_rollback() { + let temp = tempfile::tempdir().unwrap(); + let staging = StagingArea::open(temp.path().to_owned()).await.unwrap(); + let mut staged = Vec::new(); + for (path, byte) in [("committed", 0x71), ("uncommitted", 0x72)] { + staged.push( + stage_reader_streaming( + &mut std::io::Cursor::new(vec![byte; 2 * 1024 * 1024]), + Path::new(path), + &staging, + StreamStageProgress { + xorb_builder: Some(StreamStageXorbBuilder::new( + 1, + crab_xet::xorb::builder::XorbBuilder::new, + )), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await + .unwrap(), + ); + } + let entries = staged + .iter() + .enumerate() + .map(|(index, staged)| PublicationIntentEntry { + batch_id: staged.batch_id.clone(), + path: if index == 0 { + "committed" + } else { + "uncommitted" + } + .into(), + expected_pointer_oid: format!("{}", index + 1).repeat(40), + previous_index_state: "0".repeat(64), + }) + .collect::>(); + let intent = staging.create_publication_intent(&entries).unwrap(); + let recipe_hash = MerkleHash::from(staged[0].recipe.hash()); + let invalid = staging.preserve_publication_history( + &intent, + &[recipe_hash, MerkleHash::from([0xff; 32])], + &CancellationToken::new(), + ); + assert!(invalid.is_err() && staging.lifecycle_health().unwrap().path_heads == 0); + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert!(matches!( + staging.preserve_publication_history(&intent, &[recipe_hash], &cancelled), + Err(StagingError::Cancelled) + )); + for _ in 0..2 { + staging + .preserve_publication_history(&intent, &[recipe_hash], &CancellationToken::new()) + .unwrap(); + } + let health = staging.lifecycle_health().unwrap(); + assert!( + health.path_heads == 1 + && health.path_leases == 3 + && health.payloads == 0 + && health.current_head_prepared_bytes > 0, + "{health:?}" + ); + staging.rollback_publication_intent(&intent).unwrap(); + assert!( + staging + .published_recipe_for_file(&staged[0].recipe.file_hash()) + .unwrap() + == Some(staged[0].recipe.clone()) + && staging + .chunks_for_file(&staged[0].recipe.file_hash()) + .unwrap() + .len() + == 16 + && staging + .chunks_for_file(&staged[1].recipe.file_hash()) + .unwrap() + .is_empty() + ); + staging.close().await.unwrap(); + let reader = Arc::new( + StagingAreaReadOnly::open(temp.path().to_owned()) + .await + .unwrap(), + ); + reader + .verify_file_reconstruction(&staged[0].recipe.file_hash(), staged[0].size) + .await + .unwrap(); + let prepared = reader.prepared_xorbs_for_recipes(&[recipe_hash]).unwrap(); + let bodies = prepared + .values() + .flatten() + .map(|xorb| { + crab_staging::push_plan::prepared_xorb_path( + temp.path(), + &MerkleHash::from_hex(&xorb.hash).unwrap(), + ) + }) + .collect::>(); + assert!(!bodies.is_empty() && bodies.iter().all(|path| path.is_file())); + let snapshot = + StagingPushSnapshot::begin(Arc::clone(&reader), "history-push".to_owned()).unwrap(); + snapshot + .capture(std::slice::from_ref(&staged[0].recipe)) + .unwrap(); + snapshot.commit().unwrap(); + let health = reader.lifecycle_health().unwrap(); + assert!(health.path_heads == 0 && health.path_leases == 0 && health.payloads == 0); + assert!(bodies.iter().all(|path| !path.try_exists().unwrap())); +} diff --git a/crates/crab-staging/tests/push_snapshot_guard.rs b/crates/crab-staging/tests/push_snapshot_guard.rs new file mode 100644 index 000000000..909622b41 --- /dev/null +++ b/crates/crab-staging/tests/push_snapshot_guard.rs @@ -0,0 +1,174 @@ +//! Reader fences and exact lease cleanup across push completion and cancellation. + +#![expect(clippy::unwrap_used, reason = "test assertions")] + +use std::path::Path; +use std::sync::Arc; + +use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; +use crab_staging::{StagingArea, StagingAreaReadOnly, StagingPushSnapshot}; +use tokio_util::sync::CancellationToken; + +async fn fixture( + root: &Path, + native_owner: bool, +) -> ( + Arc, + crab_staging::stream::StreamStageResult, + Option, +) { + let staging = StagingArea::open(root.to_owned()).await.unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(b"shared fixture payload"), + Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + staging + .delegate_publication(&[(&staged.batch_id, Path::new("model.bin"))]) + .unwrap(); + let native = if native_owner { + let batch = staging.create_batch().unwrap(); + staging + .record_verified_recipe_lease(&batch, Path::new("native.bin"), &staged.recipe) + .unwrap(); + Some(batch) + } else { + None + }; + staging.close().await.unwrap(); + ( + Arc::new(StagingAreaReadOnly::open(root.to_owned()).await.unwrap()), + staged, + native, + ) +} + +#[tokio::test(flavor = "multi_thread")] +async fn dropped_snapshot_keeps_delegated_payload_and_clears_its_reader() { + let temp = tempfile::tempdir().unwrap(); + let (staging, staged, _) = fixture(temp.path(), false).await; + let snapshot = StagingPushSnapshot::begin(Arc::clone(&staging), "failed".to_owned()).unwrap(); + snapshot + .capture(std::slice::from_ref(&staged.recipe)) + .unwrap(); + drop(snapshot); + let health = staging.lifecycle_health().unwrap(); + assert!( + health.open_push_snapshots == 0 + && health.path_leases == 1 + && health.payloads == 1 + && staging.list_inflight().unwrap().is_empty() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn committed_snapshot_preserves_active_native_owner_of_shared_payload() { + let temp = tempfile::tempdir().unwrap(); + let (staging, staged, native) = fixture(temp.path(), true).await; + let snapshot = + StagingPushSnapshot::begin(Arc::clone(&staging), "committed".to_owned()).unwrap(); + snapshot + .capture(std::slice::from_ref(&staged.recipe)) + .unwrap(); + snapshot.commit().unwrap(); + let health = staging.lifecycle_health().unwrap(); + assert!( + health.path_leases == 1 + && health.payloads == 1 + && health.open_push_snapshots == 0 + && health.committed_push_snapshots == 0 + && staging.list_inflight().unwrap().is_empty() + ); + drop(staging); + let writer = StagingArea::open(temp.path().to_owned()).await.unwrap(); + writer.rollback_batch(&native.unwrap()).unwrap(); + assert!( + writer + .chunks_for_file(&staged.recipe.file_hash()) + .unwrap() + .is_empty() + ); + writer.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn sibling_reader_defers_retirement_until_exclusive_maintenance() { + let temp = tempfile::tempdir().unwrap(); + let (staging, staged, _) = fixture(temp.path(), false).await; + let first = StagingPushSnapshot::begin(Arc::clone(&staging), "first".to_owned()).unwrap(); + first.capture(std::slice::from_ref(&staged.recipe)).unwrap(); + let second = StagingPushSnapshot::begin(Arc::clone(&staging), "second".to_owned()).unwrap(); + second + .capture(std::slice::from_ref(&staged.recipe)) + .unwrap(); + first.commit().unwrap(); + let health = staging.lifecycle_health().unwrap(); + assert!( + health.committed_push_snapshots == 1 + && health.open_push_snapshots == 1 + && health.payloads == 1 + ); + drop(second); + drop(staging); + let writer = StagingArea::open(temp.path().to_owned()).await.unwrap(); + writer.clean().unwrap(); + let health = writer.lifecycle_health().unwrap(); + assert!( + health.path_leases == 0 + && health.payloads == 0 + && health.open_push_snapshots == 0 + && health.committed_push_snapshots == 0 + && writer.list_inflight().unwrap().is_empty() + ); + writer.close().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn task_abort_drops_snapshot_without_retiring_unpublished_lease() { + let temp = tempfile::tempdir().unwrap(); + let (staging, staged, _) = fixture(temp.path(), false).await; + let reader = Arc::clone(&staging); + let (ready, received) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let snapshot = StagingPushSnapshot::begin(reader, "aborted".to_owned()).unwrap(); + snapshot + .capture(std::slice::from_ref(&staged.recipe)) + .unwrap(); + ready.send(()).unwrap(); + std::future::pending::<()>().await; + drop(snapshot); + }); + received.await.unwrap(); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + let health = staging.lifecycle_health().unwrap(); + assert!( + health.path_leases == 1 + && health.open_push_snapshots == 0 + && health.payloads == 1 + && staging.list_inflight().unwrap().is_empty() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn local_commit_failure_releases_marker_and_preserves_original_lease() { + let temp = tempfile::tempdir().unwrap(); + let (staging, staged, _) = fixture(temp.path(), false).await; + let snapshot = + StagingPushSnapshot::begin(Arc::clone(&staging), "missing-journal".to_owned()).unwrap(); + snapshot + .capture(std::slice::from_ref(&staged.recipe)) + .unwrap(); + staging.remove_push_snapshot("missing-journal").unwrap(); + assert!(snapshot.commit().is_err()); + let health = staging.lifecycle_health().unwrap(); + assert!( + health.path_leases == 1 + && health.payloads == 1 + && staging.list_inflight().unwrap().is_empty() + ); +} diff --git a/crates/crab-staging/tests/retained_recipe_visibility.rs b/crates/crab-staging/tests/retained_recipe_visibility.rs new file mode 100644 index 000000000..0827d2b26 --- /dev/null +++ b/crates/crab-staging/tests/retained_recipe_visibility.rs @@ -0,0 +1,90 @@ +//! Public occurrence enumeration must not depend on Git publication state. + +use crab_staging::stream::{StreamStageProgress, StreamStageXorbBuilder, stage_reader_streaming}; +use crab_staging::{StagingArea, StagingAreaReadOnly}; +use crab_xet::chunker::GearChunker; +use crab_xet::hash::MerkleHash; +use std::path::Path; +use tokio_util::sync::CancellationToken; + +#[tokio::test(flavor = "multi_thread")] +#[allow(clippy::unwrap_used, reason = "test assertions")] +async fn prepared_occurrences_are_visible_to_both_handles_before_publication() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("staging"); + let bytes = vec![0x71; 2 * 1024 * 1024]; + let mut chunker = GearChunker::new(); + let mut chunks = chunker.feed(&bytes); + chunks.extend(chunker.finalize()); + let expected = chunks + .into_iter() + .map(|chunk| { + ( + MerkleHash::from(<[u8; 32]>::from(chunk.hash)), + chunk.data.len() as u64, + ) + }) + .collect::>(); + assert!(expected.len() > 1 && expected.windows(2).any(|pair| pair[0] == pair[1])); + let staging = StagingArea::open(root.clone()).await.unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(&bytes), + Path::new("model.bin"), + &staging, + StreamStageProgress { + xorb_builder: Some(StreamStageXorbBuilder::new( + 1, + crab_xet::xorb::builder::XorbBuilder::new, + )), + ..StreamStageProgress::default() + }, + &CancellationToken::new(), + ) + .await + .unwrap(); + let hash = staged.recipe.file_hash(); + let hashes = expected.iter().map(|(hash, _)| *hash).collect::>(); + assert!(staging.published_recipe_for_file(&hash).unwrap().is_none()); + assert_eq!(staging.chunks_for_file(&hash).unwrap(), hashes); + assert_eq!(staging.chunks_for_file_with_sizes(&hash).unwrap(), expected); + staging.close().await.unwrap(); + { + let shared = StagingAreaReadOnly::open(root.clone()).await.unwrap(); + assert!(shared.published_recipe_for_file(&hash).unwrap().is_none()); + assert_eq!(shared.chunks_for_file(&hash).unwrap(), hashes); + assert_eq!(shared.chunks_for_file_with_sizes(&hash).unwrap(), expected); + } + let staging = StagingArea::open(root.clone()).await.unwrap(); + staging + .delegate_publication(&[(&staged.batch_id, Path::new("model.bin"))]) + .unwrap(); + assert!(staging.published_recipe_for_file(&hash).unwrap().is_none()); + assert_eq!(staging.chunks_for_file(&hash).unwrap(), hashes); + assert_eq!(staging.chunks_for_file_with_sizes(&hash).unwrap(), expected); + staging.close().await.unwrap(); + let shared = StagingAreaReadOnly::open(root.clone()).await.unwrap(); + assert!(shared.published_recipe_for_file(&hash).unwrap().is_none()); + assert_eq!(shared.chunks_for_file(&hash).unwrap(), hashes); + assert_eq!(shared.chunks_for_file_with_sizes(&hash).unwrap(), expected); + shared + .create_push_snapshot("captured-pointer", std::slice::from_ref(&staged.recipe)) + .unwrap(); + drop(shared); + let staging = StagingArea::open(root.clone()).await.unwrap(); + staging.rollback_batch(&staged.batch_id).unwrap(); + assert!(staging.published_recipe_for_file(&hash).unwrap().is_none()); + assert_eq!(staging.chunks_for_file(&hash).unwrap(), hashes); + assert_eq!(staging.chunks_for_file_with_sizes(&hash).unwrap(), expected); + staging.close().await.unwrap(); + let shared = StagingAreaReadOnly::open(root).await.unwrap(); + assert_eq!(shared.chunks_for_file(&hash).unwrap(), hashes); + assert_eq!(shared.chunks_for_file_with_sizes(&hash).unwrap(), expected); + shared + .discard_open_push_snapshot("captured-pointer") + .await + .unwrap(); + assert!( + shared.chunks_for_file(&hash).unwrap().is_empty() + && shared.chunks_for_file_with_sizes(&hash).unwrap().is_empty() + ); +} diff --git a/crates/crab-staging/tests/unit/prop_compaction_preserves_reads.rs b/crates/crab-staging/tests/unit/prop_compaction_preserves_reads.rs index 2d3863dae..a94f588c2 100644 --- a/crates/crab-staging/tests/unit/prop_compaction_preserves_reads.rs +++ b/crates/crab-staging/tests/unit/prop_compaction_preserves_reads.rs @@ -225,7 +225,7 @@ async fn run_compaction_test( continue; } - let new_seg_id = index.allocate_segment_id().expect("alloc new"); + let (transaction, new_seg_id) = index.begin_compaction().expect("reserve output"); let old_reader = SegmentReader::open( &segments_dir.join(format!("{old_seg_id:016x}.seg")), *old_seg_id, @@ -245,17 +245,13 @@ async fn run_compaction_test( let mut new_offset: u64 = 0; let mut updates = Vec::new(); - for (chunk_hash, file_hash, chunk_index, size, old_offset) in &live_chunks { - #[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - reason = "size bounded by segment cap" - )] - let length = *size as u32; - let data = old_reader.read(*old_offset, length).expect("read"); + for (chunk_hash, locator) in &live_chunks { + let data = old_reader + .read(locator.offset, locator.length) + .expect("read"); let record = segment::encode_record(&data); std::io::Write::write_all(&mut new_file, &record).expect("write"); - updates.push((*chunk_hash, *file_hash, *chunk_index, new_offset)); + updates.push((*chunk_hash, new_offset)); new_offset += record.len() as u64; } @@ -263,13 +259,11 @@ async fn run_compaction_test( drop(new_file); fs::rename(&tmp_path, &new_seg_path).expect("rename"); - index - .swap_locators(*old_seg_id, new_seg_id, new_offset, &updates) + Index::swap_locators(transaction, *old_seg_id, new_seg_id, new_offset, &updates) .expect("swap"); let old_path = segments_dir.join(format!("{old_seg_id:016x}.seg")); let _ = fs::remove_file(&old_path); - index.drop_segment(*old_seg_id).expect("drop old"); } // Wait for concurrent readers. From 39c9509599411037dbe4e1c6d8c14877cf01e8f2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:09:29 -0700 Subject: [PATCH 051/127] perf(read): page pinned hydration lookups Keep captured catalog roots in the shared file lookup instead of copying all file bindings. Mount capture uses the paged Git view so checkpoint bodies stay cold. Preserve close ownership and authenticate the selected dependency closure. --- crab/src/cmd/mount.rs | 11 +- crab/src/read/mod.rs | 6 +- crates/crab-metadata/README.md | 8 +- crates/crab-metadata/src/file_index_lookup.rs | 114 +++++++------- .../src/file_index_lookup/catalog_tests.rs | 140 ++++++++++++++++++ crates/crab-read/README.md | 5 + crates/crab-read/src/hydrator.rs | 88 ++++++++++- 7 files changed, 302 insertions(+), 70 deletions(-) create mode 100644 crates/crab-metadata/src/file_index_lookup/catalog_tests.rs diff --git a/crab/src/cmd/mount.rs b/crab/src/cmd/mount.rs index 0ed8c170a..6f414255d 100644 --- a/crab/src/cmd/mount.rs +++ b/crab/src/cmd/mount.rs @@ -2077,7 +2077,9 @@ async fn resolve_mount_read_context_from_remote_url( } else { config.uploadpack_max_egress_bytes }; - let view = crab_read::capsule_protocol::open_view_from_root_with_control( + // The mount needs captured catalog roots, not complete checkpoint bodies. + // Keep pages cold until hydration requests their dependency closure. + let view = crab_read::capsule_protocol::open_git_view_from_root( &read_layout, root, crab_read::capsule_protocol::CapsuleReadLimits { @@ -2087,13 +2089,12 @@ async fn resolve_mount_read_context_from_remote_url( ) .await .ok()?; - let catalog = view.pointer_catalog().await.ok()?; + let catalog = view.pointer_catalog_lookup(&read_layout, maximum).ok()?; Some( crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( read_layout.clone(), - &catalog, - ) - .ok()?, + catalog, + ), ) } Err(crab_metadata::error::MetadataError::Storage { diff --git a/crab/src/read/mod.rs b/crab/src/read/mod.rs index ee5a83395..0a5c729d9 100644 --- a/crab/src/read/mod.rs +++ b/crab/src/read/mod.rs @@ -531,12 +531,12 @@ impl Inner { &self.cancel, ) .await?; - let catalog = view.pointer_catalog().await?; + let catalog = view.pointer_catalog_lookup(&read_layout, maximum)?; let file_index_lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( read_layout, - &catalog, - )?; + catalog, + ); Ok((resolved, git_dir, Some(file_index_lookup))) } diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index c7752c5a2..84a0a6de5 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -59,7 +59,13 @@ its footer. `LayeredCheckpoint::pointer_catalog_lookup` captures these roots; `PointerCatalogLookup` batches file/chunk/xorb keys through one bounded page cache and byte budget, validates selected dependency closure and keeps immutable descriptor conflicts distinct from absent keys. Full inspection remains an -explicit maintenance operation. All 19 audited local release tags omit these +explicit maintenance operation. `SharedFileIndexLookup::for_pointer_catalog` +accepts captured lookup roots and resolves requested files with their authenticated +shard/xorb closure. Point and batch calls share the page cache and retain the +session read guard through I/O, so close waits for active reads. Missing catalog +pages remain errors rather than reopening the mutable file-index database. +The current-state opener still materializes a complete v2 catalog; callers with +an authenticated view should pass its captured roots. All 19 audited local release tags omit these capsule checkpoint formats; unshipped format 6 is rejected without a fallback. Publication deltas are still materialized by the caller's captured view; their run-level aggregation and lazy capture remain required for the complete gate. diff --git a/crates/crab-metadata/src/file_index_lookup.rs b/crates/crab-metadata/src/file_index_lookup.rs index 86fef2ef5..872418e4b 100644 --- a/crates/crab-metadata/src/file_index_lookup.rs +++ b/crates/crab-metadata/src/file_index_lookup.rs @@ -224,6 +224,7 @@ async fn lookup_committed_record( /// Opens the repo's `file_index_db` once, serves one or many point lookups, and /// closes the underlying SlateDB reader when consumed. pub struct FileIndexLookupSession { + catalog: Option>, static_entries: Option>>, reader: Option>, anchor: Option, @@ -244,6 +245,7 @@ impl FileIndexLookupSession { snapshot: &RepositorySnapshot, ) -> Result { Ok(Self { + catalog: None, static_entries: None, reader: None, anchor: CommittedShardAnchor::from_snapshot(snapshot)?, @@ -307,6 +309,7 @@ impl FileIndexLookupSession { )?; let anchor = CommittedShardAnchor::from_snapshot(snapshot)?; Ok(Self { + catalog: None, static_entries: None, reader: None, anchor, @@ -367,6 +370,7 @@ impl FileIndexLookupSession { }) }; Ok(Self { + catalog: None, static_entries: None, reader: None, anchor, @@ -414,6 +418,7 @@ impl FileIndexLookupSession { }) .collect::>>()?; return Ok(Self { + catalog: None, static_entries: Some(Arc::new(entries)), reader: None, anchor: None, @@ -438,6 +443,7 @@ impl FileIndexLookupSession { Err(error) => return Err(error), }; let mut session = Self { + catalog: None, static_entries: None, reader: None, anchor, @@ -477,6 +483,14 @@ impl FileIndexLookupSession { /// Look up one file hash in the open session. pub async fn lookup(&self, file_hash: &MerkleHash) -> Result> { + if self.catalog.is_some() { + return Ok(self + .lookup_batch(&[*file_hash]) + .await? + .into_iter() + .next() + .flatten()); + } if let Some(entries) = &self.static_entries { return Ok(entries.get(file_hash).copied()); } @@ -505,6 +519,30 @@ impl FileIndexLookupSession { if file_hashes.is_empty() { return Ok(Vec::new()); } + if let Some(lookup) = &self.catalog { + let hashes = file_hashes.iter().map(MerkleHash::hex).collect::>(); + let selected = lookup.files(&self.storage, &hashes).await?; + return hashes + .iter() + .map(|hash| { + selected + .files() + .get(hash) + .map(|entry| { + MerkleHash::from_hex(entry.shard_hash()).map_err(|source| { + MetadataError::CorruptObject { + path: "capsule-protocol pointer catalog".to_owned(), + reason: format!( + "invalid shard hash {}: {source}", + entry.shard_hash() + ), + } + }) + }) + .transpose() + }) + .collect(); + } if let Some(entries) = &self.static_entries { return Ok(file_hashes .iter() @@ -739,9 +777,9 @@ fn spawn_shard_parse( } enum LookupSource { - Static { + Catalog { router: crab_storage::StoreLayout, - entries: Arc>, + lookup: Arc, }, Current { store: crab_storage::Store, @@ -759,8 +797,9 @@ enum LookupSource { impl LookupSource { async fn open(&self) -> Result { match self { - Self::Static { router, entries } => Ok(FileIndexLookupSession { - static_entries: Some(Arc::clone(entries)), + Self::Catalog { router, lookup } => Ok(FileIndexLookupSession { + catalog: Some(Arc::clone(lookup)), + static_entries: None, reader: None, anchor: None, storage: router.store().clone(), @@ -807,45 +846,25 @@ pub struct SharedFileIndexLookup { } impl SharedFileIndexLookup { - /// Lazily resolve files from an already authenticated pointer catalog. + /// Resolve files lazily from the caller's captured authenticated catalog roots. /// - /// This binds the lookup to the catalog captured by a caller's immutable - /// v2 view. Lookups never open the mutable file-index database or widen to - /// a later repository state. + /// The caller retains these roots against GC. Lookups never open mutable + /// metadata or widen to a later repository state; dependency errors stay errors. + #[must_use] pub fn for_pointer_catalog( router: crab_storage::StoreLayout, - catalog: &crate::capsule_protocol::PointerCatalog, - ) -> Result { - let entries = catalog - .files() - .iter() - .map(|(file_hash, entry)| { - Ok(( - MerkleHash::from_hex(file_hash).map_err(|source| { - MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!("invalid file hash {file_hash}: {source}"), - } - })?, - MerkleHash::from_hex(entry.shard_hash()).map_err(|source| { - MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!("invalid shard hash {}: {source}", entry.shard_hash()), - } - })?, - )) - }) - .collect::>>()?; - Ok(Self { + lookup: crate::capsule_protocol::PointerCatalogLookup, + ) -> Self { + Self { inner: Arc::new(SharedFileIndexLookupInner { - source: LookupSource::Static { + source: LookupSource::Catalog { router, - entries: Arc::new(entries), + lookup: Arc::new(lookup), }, session: tokio::sync::RwLock::new(tokio::sync::OnceCell::new()), closed: AtomicBool::new(false), }), - }) + } } /// Lazily resolve files from one captured immutable shard-index root without writes. @@ -976,6 +995,8 @@ pub async fn resolve_file_hash_to_shard( result } +#[cfg(test)] +mod catalog_tests; #[cfg(test)] mod shared_tests; #[cfg(test)] @@ -1832,29 +1853,4 @@ mod tests { .expect_err("closed clone must reject future lookups"); assert!(matches!(err, MetadataError::Internal(_))); } - - #[tokio::test] - async fn shared_lookup_from_pointer_catalog_never_reads_latest_index() { - let store: Arc = Arc::new(InMemory::new()); - let storage = crab_storage::Store::new(Arc::clone(&store)); - let router = crab_storage::StoreLayout::new(storage, "org/pinned".to_owned()); - let file_hash = hash_from_seed(91); - let shard_hash = hash_from_seed(92); - let mut catalog = crate::capsule_protocol::PointerCatalog::new(); - catalog - .insert_file( - file_hash.hex(), - crate::capsule_protocol::FileCatalogEntry::new(16, shard_hash.hex()), - ) - .unwrap(); - - let lookup = SharedFileIndexLookup::for_pointer_catalog(router, &catalog).unwrap(); - assert_eq!(lookup.lookup(&file_hash).await.unwrap(), Some(shard_hash)); - assert_eq!( - lookup.lookup(&hash_from_seed(93)).await.unwrap(), - None, - "the captured catalog is authoritative for this mount" - ); - lookup.close().await.unwrap(); - } } diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs new file mode 100644 index 000000000..a035cf9cb --- /dev/null +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -0,0 +1,140 @@ +use super::*; +use crate::capsule_protocol::{ + CapsuleGitPack, FileCatalogEntry, LayeredCheckpoint, PackLayer, PointerCatalog, + PointerCatalogLookup, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, +}; +use bytes::Bytes; +use object_store::memory::InMemory; +use std::sync::atomic::AtomicUsize; + +async fn capture( + layout: &crab_storage::StoreLayout, + catalog: PointerCatalog, +) -> (PointerCatalogLookup, object_store::path::Path) { + let pack = CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "1".repeat(40), + 1, + ) + .unwrap(); + let source = PackLayer::build(&pack) + .unwrap() + .source_descriptor() + .unwrap(); + let checkpoint = + LayeredCheckpoint::build(1, &"3".repeat(64), vec![source], catalog, None).unwrap(); + let path = layout.capsule_checkpoint_path(checkpoint.hash()); + layout + .store() + .put_overwrite(&path, checkpoint.bytes().clone()) + .await + .unwrap(); + let control = LayeredCheckpoint::decode_control( + checkpoint + .bytes() + .slice(checkpoint.control_offset() as usize..), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.hash(), + checkpoint.footer_hash(), + ) + .unwrap(); + ( + control.pointer_catalog_lookup(layout, 512 * 1024).unwrap(), + path, + ) +} + +fn catalog(count: usize) -> PointerCatalog { + let mut result = PointerCatalog::new(); + for index in 0..count { + let file = super::tests::hash_from_seed(index as u64); + let shard = super::tests::hash_from_seed(1_000_000 + index as u64); + let xorb = super::tests::hash_from_seed(2_000_000 + index as u64); + result + .insert_file(file.hex(), FileCatalogEntry::new(16, shard.hex())) + .unwrap(); + result + .insert_shard(shard.hex(), ShardCatalogEntry::new(80, vec![xorb.hex()])) + .unwrap(); + result + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 99, + "e".repeat(64), + vec![XorbChunkEntry::new( + super::tests::hash_from_seed(3_000_000 + index as u64).hex(), + 16, + )], + ), + ) + .unwrap(); + } + result +} + +#[tokio::test] +async fn pinned_batches_preserve_order_misses_and_page_bounds_as_history_grows() { + for count in [100, 20_000] { + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let storage = crab_storage::Store::new(Arc::new(InMemory::new())) + .with_read_request_observer(Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + })); + let layout = crab_storage::StoreLayout::new(storage, "pinned-lookup".to_owned()); + let (captured, _) = capture(&layout, catalog(count)).await; + let lookup = SharedFileIndexLookup::for_pointer_catalog(layout.clone(), captured); + assert_eq!( + reads.load(Ordering::SeqCst), + 0, + "construction must leave pages cold" + ); + // A later authenticated catalog must not change an existing capture. + capture(&layout, catalog(count + 1)).await; + let file = super::tests::hash_from_seed(42); + let later = super::tests::hash_from_seed(count as u64); + let shard = super::tests::hash_from_seed(1_000_042); + assert_eq!( + lookup.lookup_batch(&[file, later, file]).await.unwrap(), + vec![Some(shard), None, Some(shard)] + ); + let cold = reads.load(Ordering::SeqCst); + assert!( + cold <= 24, + "fixed file lookup exceeded its page bound: {cold}" + ); + assert_eq!(lookup.lookup(&file).await.unwrap(), Some(shard)); + assert_eq!( + reads.load(Ordering::SeqCst), + cold, + "warm lookup reread pages" + ); + let clone = lookup.clone(); + lookup.close().await.unwrap(); + assert!(clone.lookup(&file).await.is_err()); + } +} + +#[tokio::test] +async fn missing_captured_pages_are_errors_even_when_a_new_catalog_exists() { + let layout = crab_storage::StoreLayout::new( + crab_storage::Store::new(Arc::new(InMemory::new())), + "missing-page".to_owned(), + ); + let (captured, path) = capture(&layout, catalog(100)).await; + capture(&layout, catalog(101)).await; + layout.store().delete(&path).await.unwrap(); + let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); + assert!( + lookup + .lookup(&super::tests::hash_from_seed(42)) + .await + .is_err() + ); + lookup.close().await.unwrap(); +} diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 6fe09c3ad..22f08e250 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -61,6 +61,11 @@ work. `git_visibility_index`, `candidate_git_visibility` and `tip_bound_transitions` asynchronously materialize captured controls only when bulk inspection needs them. They do not recapture refs or switch origins. +Pinned hydration and remote-mount setup pass captured paged catalog roots into +metadata's shared file lookup. A lookup loads only its selected file dependency +closure, preserving the captured publication order and immutable descriptor +checks. It never substitutes a later catalog or mutable file-index session. + `ShardHydrator` provides memory, file, and half-open byte-range reconstruction. Before full-file reconstruction it checks that the pointer's shard terms are covered; after reconstruction it verifies the requested file hash. A shared diff --git a/crates/crab-read/src/hydrator.rs b/crates/crab-read/src/hydrator.rs index 8c1adcd21..c2a1cf139 100644 --- a/crates/crab-read/src/hydrator.rs +++ b/crates/crab-read/src/hydrator.rs @@ -1209,10 +1209,94 @@ mod tests { hydrator.router.repo_prefix().to_owned(), hydrator.router.global_prefix().to_owned(), ); - let lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( - router, &catalog, + let (body, _) = router + .store() + .get_with_etag(&router.shard_path(&shard_hash.into())) + .await + .unwrap(); + let mut chunks = std::collections::BTreeMap::<_, Vec<_>>::new(); + crab_xet::shard_parse::visit_xorb_chunks_from_reader( + &mut std::io::Cursor::new(&body), + |xorb, _, chunk, size| { + chunks.entry(xorb).or_default().push( + crab_metadata::capsule_protocol::XorbChunkEntry::new(chunk.hex(), size), + ); + Ok(()) + }, ) .unwrap(); + let xorb_hashes = chunks.keys().map(|hash| hash.hex()).collect(); + catalog + .insert_shard( + crab_xet::hash::MerkleHash::from(shard_hash).hex(), + crab_metadata::capsule_protocol::ShardCatalogEntry::new( + body.len() as u64, + xorb_hashes, + ), + ) + .unwrap(); + for (hash, entries) in chunks { + let (body, _) = router + .store() + .get_with_etag(&router.xorb_path(&hash)) + .await + .unwrap(); + catalog + .insert_xorb( + hash.hex(), + crab_metadata::capsule_protocol::XorbCatalogEntry::new( + body.len() as u64, + blake3::hash(&body).to_hex().to_string(), + entries, + ), + ) + .unwrap(); + } + let pack = crab_metadata::capsule_protocol::CapsuleGitPack::new( + bytes::Bytes::from_static(b"pack"), + bytes::Bytes::from_static(b"index"), + bytes::Bytes::from_static(b"reverse"), + bytes::Bytes::from_static(b"locator"), + "1".repeat(40), + 1, + ) + .unwrap(); + let source = crab_metadata::capsule_protocol::PackLayer::build(&pack) + .unwrap() + .source_descriptor() + .unwrap(); + let checkpoint = crab_metadata::capsule_protocol::LayeredCheckpoint::build( + 1, + &"3".repeat(64), + vec![source], + catalog, + None, + ) + .unwrap(); + router + .store() + .put_overwrite( + &router.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + let control = crab_metadata::capsule_protocol::LayeredCheckpoint::decode_control( + checkpoint + .bytes() + .slice(checkpoint.control_offset() as usize..), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.hash(), + checkpoint.footer_hash(), + ) + .unwrap(); + let captured = control + .pointer_catalog_lookup(&router, 1024 * 1024) + .unwrap(); + let lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( + router, captured, + ); let hydrator = hydrator.with_file_index_lookup(lookup); assert_eq!( From 138c89fd72fed2f03adcd24a81ddffc30e2fe531 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:21:23 -0700 Subject: [PATCH 052/127] test(read): name the fixture shard hash type --- crates/crab-read/src/hydrator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/crab-read/src/hydrator.rs b/crates/crab-read/src/hydrator.rs index c2a1cf139..d5e48f4d8 100644 --- a/crates/crab-read/src/hydrator.rs +++ b/crates/crab-read/src/hydrator.rs @@ -1211,7 +1211,7 @@ mod tests { ); let (body, _) = router .store() - .get_with_etag(&router.shard_path(&shard_hash.into())) + .get_with_etag(&router.shard_path(&crab_xet::hash::MerkleHash::from(shard_hash))) .await .unwrap(); let mut chunks = std::collections::BTreeMap::<_, Vec<_>>::new(); From 0438cbd47eba916efaeeefb5c9e3551c42149f66 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 21:59:17 -0700 Subject: [PATCH 053/127] perf(remote): select publication dependencies --- crates/crab-metadata/README.md | 21 +- .../capsule_protocol/pointer/paged/reader.rs | 33 ++- .../pointer/paged/reader/tests.rs | 101 +++++++ crates/crab-remote/README.md | 9 + crates/crab-remote/src/prepare.rs | 34 ++- crates/crab-remote/tests/checkpoint.rs | 3 + .../tests/checkpoint/publication_catalog.rs | 267 ++++++++++++++++++ 7 files changed, 453 insertions(+), 15 deletions(-) create mode 100644 crates/crab-remote/tests/checkpoint/publication_catalog.rs diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 84a0a6de5..4955ae875 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -57,9 +57,11 @@ remain rejected. Checkpoint format 7 (`CRBCKP07`) carries authenticated catalog page roots in its footer. `LayeredCheckpoint::pointer_catalog_lookup` captures these roots; `PointerCatalogLookup` batches file/chunk/xorb keys through one bounded page -cache and byte budget, validates selected dependency closure and keeps immutable -descriptor conflicts distinct from absent keys. Full inspection remains an -explicit maintenance operation. `SharedFileIndexLookup::for_pointer_catalog` +cache and byte budget. Its `dependencies` query selects files plus explicit +shard/xorb identities in one aggregate budget, including complete dependency +closure. Missing candidate identities remain misses; missing referenced +artifacts and conflicting visible descriptors fail. Full inspection remains +an explicit maintenance operation. `SharedFileIndexLookup::for_pointer_catalog` accepts captured lookup roots and resolves requested files with their authenticated shard/xorb closure. Point and batch calls share the page cache and retain the session read guard through I/O, so close waits for active reads. Missing catalog @@ -67,8 +69,10 @@ pages remain errors rather than reopening the mutable file-index database. The current-state opener still materializes a complete v2 catalog; callers with an authenticated view should pass its captured roots. All 19 audited local release tags omit these capsule checkpoint formats; unshipped format 6 is rejected without a fallback. -Publication deltas are still materialized by the caller's captured view; their -run-level aggregation and lazy capture remain required for the complete gate. +Captured run-v4 roots provide the same selected lookup over exact logical +capsule occurrences; catalog bodies remain cold until their keys are selected. +CLI and shared capsule-publication preparation use these captured queries. +Protected receive and hydration still require separate consumer integration. External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and @@ -79,9 +83,10 @@ canonical page layout. Delta inspection defers base-owned closure to `apply`, which retains later file bindings at equal size and rejects immutable shard/xorb conflicts. Encoded catalogs are limited to 2 GiB and descriptors to 4 KiB. The retired monolithic JSON encoding was absent from all 19 audited release tags -and is rejected. This format foundation still uses complete materialization in -foreground callers; lazy lookup and bounded historical amplification remain -unfinished and must not be inferred from the page representation alone. +and is rejected. Foreground consumers must use captured selected lookups rather +than infer bounded work from the page encoding alone. Maintenance retains explicit full +inspection; shared/protected hydration and lifecycle qualification remain +separate gates. `v2/browse-indexes` is a bounded, mutable derived record, not ref authority. Its version 3 record binds a durable immutable commit graph, optional complete diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index 1f96eb221..14449fba8 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -233,8 +233,24 @@ impl PointerCatalogLookup { /// Select requested files and their complete shard/xorb dependency closure. pub async fn files(&self, store: &Store, hashes: &[String]) -> Result { + self.dependencies(store, hashes, &[], &[]).await + } + + /// Select files and explicit shard/xorb identities with their complete closure. + /// + /// All keys and dependency pages share one query budget. Absent requested + /// identities are misses; a present file or shard with missing dependencies fails. + pub async fn dependencies( + &self, + store: &Store, + files: &[String], + shards: &[String], + xorbs: &[String], + ) -> Result { let mut work = self.work(); - let hashes = requested(hashes, &mut work)?; + let hashes = requested(files, &mut work)?; + let mut shards = requested(shards, &mut work)?; + let mut xorbs = requested(xorbs, &mut work)?; let root = self .checkpoint .as_ref() @@ -243,7 +259,6 @@ impl PointerCatalogLookup { .rows(store, root, Kind::CatalogFiles, &keys(&hashes)?, &mut work) .await?; let mut catalog = PointerCatalog::new(); - let mut shards = BTreeSet::new(); for (hash, row) in hashes.iter().zip(rows) { let mut selected: Option<(usize, FileCatalogEntry)> = None; for value in row { @@ -279,7 +294,6 @@ impl PointerCatalogLookup { let rows = self .rows(store, root, Kind::CatalogShards, &keys(&shards)?, &mut work) .await?; - let mut xorbs = BTreeSet::new(); for (hash, row) in shards.iter().zip(rows) { for value in row { let row: ShardRow = from_json(&value.bytes)?; @@ -305,12 +319,19 @@ impl PointerCatalogLookup { verify_identity(&value, Kind::CatalogShards, &json(&entry)?)?; catalog.insert_shard(hash.clone(), entry)?; } - let entry = catalog + } + for file in catalog.files.values() { + catalog .shards - .get(hash) + .get(&file.shard_hash) .ok_or_else(|| corrupt("selected file references an absent shard"))?; - xorbs.extend(entry.xorb_hashes.iter().cloned()); } + xorbs.extend( + catalog + .shards + .values() + .flat_map(|entry| entry.xorb_hashes.iter().cloned()), + ); let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; for (hash, entry) in selected.xorbs { catalog.insert_xorb(hash, entry)?; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs index 610dce41b..308031030 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs @@ -123,6 +123,107 @@ async fn fixed_file_and_chunk_queries_read_selected_closure_as_history_grows() { } } +#[tokio::test] +async fn mixed_dependency_queries_keep_unrelated_history_cold() { + for count in [100, 20_000] { + let full = catalog(count); + let mut expected = PointerCatalog::new(); + let file = format!("{:064x}", 42); + expected + .insert_file(file.clone(), full.files[&file].clone()) + .unwrap(); + for index in [7, 42] { + let hash = format!("{:064x}", 3_000_000 + index); + expected + .insert_shard(hash.clone(), full.shards[&hash].clone()) + .unwrap(); + } + for index in [7, 9, 42] { + let hash = format!("{:064x}", 1_000_000 + index); + expected + .insert_xorb(hash.clone(), full.xorbs[&hash].clone()) + .unwrap(); + } + let (layout, _, lookup, reads) = fixture(full, 512 * 1024).await; + let files = [file.clone(), file]; + let shards = [format!("{:064x}", 3_000_007), "f".repeat(64)]; + let xorbs = [format!("{:064x}", 1_000_009)]; + let selected = lookup + .dependencies(layout.store(), &files, &shards, &xorbs) + .await + .unwrap(); + assert_eq!(selected, expected); + let cold = reads.load(Ordering::SeqCst); + assert!(cold <= 24, "fixed mixed query exceeded its page bound"); + lookup + .dependencies(layout.store(), &files, &shards, &xorbs) + .await + .unwrap(); + assert_eq!(reads.load(Ordering::SeqCst), cold); + } +} + +#[tokio::test] +async fn explicit_identity_misses_do_not_hide_missing_file_dependencies() { + let (layout, _, lookup, _) = fixture(catalog(1), 1024 * 1024).await; + let missing = ["f".repeat(64)]; + assert!( + lookup + .dependencies(layout.store(), &missing, &missing, &missing) + .await + .unwrap() + .is_empty() + ); + let mut delta = PointerCatalog::new(); + delta + .insert_file(missing[0].clone(), FileCatalogEntry::new(7, "e".repeat(64))) + .unwrap(); + let lookup = append_run(&layout, lookup, vec![delta]).await; + assert!( + lookup + .dependencies(layout.store(), &missing, &[], &[]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn explicit_shard_selection_rejects_conflicting_visible_descriptors() { + let (layout, _, lookup, _) = fixture(catalog(1), 1024 * 1024).await; + let shard = format!("{:064x}", 3_000_000); + let mut delta = PointerCatalog::new(); + delta + .insert_shard( + shard.clone(), + ShardCatalogEntry::new(81, vec![format!("{:064x}", 1_000_000)]), + ) + .unwrap(); + let lookup = append_run(&layout, lookup, vec![delta]).await; + assert!( + lookup + .dependencies(layout.store(), &[], &[shard], &[]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn mixed_query_keys_share_admission_before_origin_reads() { + let (layout, _, lookup, reads) = fixture(catalog(1), 191).await; + assert!( + lookup + .dependencies( + layout.store(), + &[format!("{:064x}", 0)], + &[format!("{:064x}", 3_000_000)], + &[], + ) + .await + .is_err() + ); + assert_eq!(reads.load(Ordering::SeqCst), 0); +} + #[tokio::test] async fn ordered_runs_override_file_bindings_but_preserve_descriptor_conflicts() { let (layout, _, lookup, _) = fixture(catalog(2), 1024 * 1024).await; diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 07fabe32e..20c9bf085 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -63,6 +63,15 @@ commit encoding, and Git object identities. `prepare` owns normalized packs and hydrated content artifacts used by remote commits and recovery. File bodies and caller admission policy stay outside the object helpers. +Capsule publication preparation queries only outgoing file bindings and +candidate shard/xorb identities from its captured catalog roots. All three key +sets and their dependency pages share one query budget. Candidate identities +still need collision checks when a new file reuses historical artifacts; they +cannot be inferred only from existing file bindings. The caller retains its +ref leases and GC fences through lookup, payload verification and commitment. +This does not change physical content-proof limits or authorize a newer view. +Full catalog inspection remains an explicit maintenance operation. + The `local` feature owns explicit Git and Crab subprocesses, capabilities handshake, bounded I/O, cancellation, hook isolation, repository leases, filter configuration, and exact command-path quoting. `transfer` owns direct clone and diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index ddd0e4dec..3c526c14e 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -837,7 +837,39 @@ impl Prepared { "Capsule view differs from the prepared repository version", )); } - let base_catalog = view.pointer_catalog().await?; + let lookup = view.pointer_catalog_lookup(&self.layout, 2 * 1024 * 1024 * 1024)?; + let local = self.content.as_ref().map(PreparedContent::pointer_catalog); + let mut files = self + .plan + .pointers() + .iter() + .filter_map(|dependency| match &dependency.pointer { + crab_git::pointer_detect::PointerKind::Crab(pointer) => { + Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) + } + _ => None, + }) + .collect::>(); + files.extend( + local + .into_iter() + .flat_map(|catalog| catalog.files().keys().cloned()), + ); + let files = files.into_iter().collect::>(); + let shards = local + .into_iter() + .flat_map(|catalog| catalog.shards().keys().cloned()) + .collect::>(); + let xorbs = local + .into_iter() + .flat_map(|catalog| catalog.xorbs().keys().cloned()) + .collect::>(); + // Local artifacts can collide with historical identities even when + // none of their files are present. Select those keys and the outgoing + // file closure together; unrelated history must stay cold. + let base_catalog = lookup + .dependencies(self.layout.store(), &files, &shards, &xorbs) + .await?; let prepared_files = self .content .as_ref() diff --git a/crates/crab-remote/tests/checkpoint.rs b/crates/crab-remote/tests/checkpoint.rs index ce24a3c17..b3030b58f 100644 --- a/crates/crab-remote/tests/checkpoint.rs +++ b/crates/crab-remote/tests/checkpoint.rs @@ -9,6 +9,9 @@ mod frontier_admission; #[path = "checkpoint/native_cache.rs"] mod native_cache; +#[path = "checkpoint/publication_catalog.rs"] +mod publication_catalog; + use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; diff --git a/crates/crab-remote/tests/checkpoint/publication_catalog.rs b/crates/crab-remote/tests/checkpoint/publication_catalog.rs new file mode 100644 index 000000000..55e9964e1 --- /dev/null +++ b/crates/crab-remote/tests/checkpoint/publication_catalog.rs @@ -0,0 +1,267 @@ +use super::*; +use crab_metadata::capsule_protocol::{ + FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, +}; +use crab_remote::prepare::{ContentArtifact, ContentFile, PreparedContent}; +use crab_xet::hash::MerkleHash; + +async fn native_content( + layout: &StoreLayout, + directory: &std::path::Path, +) -> (PreparedContent, Vec, String) { + let data = Bytes::from(vec![7; 8192]); + let hash = *blake3::hash(&data).as_bytes(); + let chunk = crab_xet::xorb::format::Chunk::new(data.clone()); + let mut builder = crab_xet::xorb::builder::XorbBuilder::new(); + builder + .push(&chunk, crab_xet::xorb::builder::RunId(0)) + .unwrap(); + let xorb = builder.finalize().unwrap().remove(0); + let placements = xorb + .placements + .iter() + .map(|entry| (entry.chunk_hash, entry.clone())) + .collect(); + let file = crab_xet::shard::file_info_from_placements( + MerkleHash::from(hash), + &[chunk.hash], + &placements, + ) + .unwrap(); + let dependency = + crab_xet::shard::xorb_info_from_placements(xorb.hash, &xorb.placements).unwrap(); + let mut shards = crab_xet::shard::PushShardSession::new(); + shards + .add_file_bundle(file, &[Arc::new(dependency)]) + .unwrap(); + let (shard, shard_hash) = shards.finalize().unwrap().remove(0); + let shard = Bytes::from(shard); + let xorb_path = directory.join("xorb"); + let shard_path = directory.join("shard"); + std::fs::write(&xorb_path, &xorb.bytes).unwrap(); + std::fs::write(&shard_path, &shard).unwrap(); + layout + .store() + .put_overwrite(&layout.xorb_path(&xorb.hash), xorb.bytes.clone()) + .await + .unwrap(); + layout + .store() + .put_overwrite(&layout.shard_path(&shard_hash), shard.clone()) + .await + .unwrap(); + let content = PreparedContent::new( + vec![ContentArtifact::new( + xorb.hash.into(), + *blake3::hash(&xorb.bytes).as_bytes(), + xorb.bytes.len() as u64, + xorb_path, + )], + vec![ContentArtifact::new( + shard_hash.into(), + *blake3::hash(&shard).as_bytes(), + shard.len() as u64, + shard_path, + )], + vec![ContentFile::new(hash, data.len() as u64, shard_hash.into())], + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\n", + blake3::Hash::from(hash).to_hex(), + data.len() + ) + .into_bytes(); + (content, pointer, MerkleHash::from(hash).hex()) +} + +async fn publish_history(layout: &StoreLayout, catalog: PointerCatalog) { + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + let (oid, pack) = blob_pack(b"history marker"); + let name = "refs/tags/history"; + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new(name, None, Some(oid.clone()), None)], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + name.to_owned(), + GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid]), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![pack], + vec![ + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + ), + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + ), + ], + ) + .unwrap(); + crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + .await + .unwrap(); +} + +fn dependency_limits() -> crab_read::dependency_proof::DependencyProofLimits { + crab_read::dependency_proof::DependencyProofLimits { + max_dependencies: 16, + max_total_file_bytes: LIMIT, + max_duration: std::time::Duration::from_secs(30), + lookup: crab_metadata::file_index_lookup::FileIndexLookupLimits { + max_files: 16, + max_shard_visits: 16, + max_shard_bytes: LIMIT, + max_recipe_entries: 1024, + }, + content: crab_read::pointer_proof::PointerProofLimits { + max_file_bytes: LIMIT, + max_shard_bytes: LIMIT, + max_xorb_bytes: LIMIT, + max_read_bytes: LIMIT, + max_chunks: 1024, + max_duration: std::time::Duration::from_secs(30), + }, + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn fixed_pointer_publication_keeps_unrelated_catalog_history_cold() { + for count in [100, 10_000] { + for checkpointed in [false, true] { + for supplied in [false, true] { + let (layout, observations) = empty_fixture().await; + let directory = tempfile::tempdir().unwrap(); + let (content, pointer, file_hash) = native_content(&layout, directory.path()).await; + let mut catalog = PointerCatalog::new(); + for index in 0..count { + let shard = format!("{:064x}", index + 1_000_000); + let xorb = format!("{:064x}", index + 2_000_000); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 99, + "e".repeat(64), + vec![XorbChunkEntry::new( + format!("{:064x}", index + 3_000_000), + 7, + )], + ), + ) + .unwrap(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + catalog + .insert_file(format!("{index:064x}"), FileCatalogEntry::new(7, shard)) + .unwrap(); + } + // A new file may reuse already-published shard/xorb identities. + // Omitting its binding makes explicit artifact selection necessary. + for (hash, entry) in content.pointer_catalog().xorbs() { + catalog.insert_xorb(hash.clone(), entry.clone()).unwrap(); + } + for (hash, entry) in content.pointer_catalog().shards() { + catalog.insert_shard(hash.clone(), entry.clone()).unwrap(); + } + if !supplied { + for (hash, entry) in content.pointer_catalog().files() { + catalog.insert_file(hash.clone(), entry.clone()).unwrap(); + } + } + publish_history(&layout, catalog.clone()).await; + if checkpointed { + assert!( + crab_remote::checkpoint::publish_capsule_checkpoint( + &layout, + 0, + 64 * LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap() + .published + ); + } + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let cancel = CancellationToken::new(); + let target = "refs/tags/pointer".to_owned(); + let lease_store = layout.store().clone(); + let lease_layout = layout.clone(); + let reader_runtime = runtime.clone(); + crab_remote::publication::with_leases( + &lease_store, &lease_layout, [target.clone()], std::time::Duration::from_secs(60), &cancel, + move |_holders, cancel| async move { + let view = open_history_view(&layout).await; + assert_eq!(view.pointer_catalog().await?, catalog); + let repository = view.git_repository_from_store( + layout.clone(), + crab_remote_git::RepositoryIdentity::new("memory", "selected-publication", 1).unwrap(), + reader_runtime, crab_remote_git::RepositoryOptions::default(), LIMIT, &cancel, + ).await?; + let (oid, pack) = blob_pack(&pointer); + let path = directory.path().join("incoming.pack"); + std::fs::write(&path, pack.pack_bytes())?; + let mut prepared = crab_remote::prepare::prepare_capsule( + repository, view.git_visibility_index().await?, + crab_git::incoming_pack::ScratchSpace::new(directory.path(), 64 * LIMIT), + Some(std::io::BufReader::new(std::fs::File::open(path)?)), + vec![crab_git::receive_plan::RefUpdate { name: target.clone(), old: None, new: Some(gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap()) }], + BTreeMap::new(), &cancel, + crab_remote::prepare::Options { + layout: layout.clone(), + graph: crab_git::receive_plan::GraphLimits { max_ref_updates: 16, max_graph_steps: 1024, max_object_bytes: LIMIT as usize, max_read_bytes: LIMIT }, + pack: crab_git::incoming_pack::ReceiveLimits { max_pack_bytes: LIMIT, max_objects: 1024, max_object_bytes: LIMIT as usize, max_inflated_bytes: LIMIT, max_delta_depth: 128 }, + policy: |_: &str| crab_git::receive_plan::RefPolicy { allow_delete: true, allow_non_fast_forward: false }, + }, + ).await?; + assert_eq!(prepared.plan().pointers().len(), 1); + if supplied { prepared = prepared.attach_content(content)?; } + observations.0.lock().unwrap().clear(); + let artifacts = prepared.upload_capsule(&view, dependency_limits(), &cancel).await?; + let requests = observations.0.lock().unwrap().clone(); + assert!(requests.iter().all(|read| read.operation != StorageOperation::Put), "reused dependencies must not upload payloads"); + let reads = requests.iter().filter(|r| matches!(r.operation, StorageOperation::Get | StorageOperation::Range)).collect::>(); + assert!(reads.len() <= 24 && reads.iter().map(|r| r.bytes_read).sum::() <= 512 * 1024, "fixed publication exceeds selected dependency bounds: count={count}, checkpoint={checkpointed}, supplied={supplied}"); + let outcome = artifacts.commit(None, std::time::Duration::from_secs(60), &cancel).await?; + assert!(matches!(outcome, crab_remote::prepare::CapsuleCommitOutcome::Committed { .. })); + let after = open_history_view(&layout).await; + assert_eq!(after.refs()[&target], oid); + assert!(after.pointer_catalog_lookup(&layout, LIMIT)?.files(layout.store(), &[file_hash.clone()]).await?.files().contains_key(&file_hash)); + drop(prepared); + Ok::<_, Box>(()) + }, + ).await.unwrap(); + runtime.shutdown().await; + } + } + } +} + +async fn open_history_view(layout: &StoreLayout) -> CapsuleRepositoryView { + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + crab_read::capsule_protocol::open_git_view_from_root( + layout, + root, + CapsuleReadLimits { + max_capsule_bytes: 64 * LIMIT, + max_frontier_bytes: 64 * LIMIT, + }, + ) + .await + .unwrap() +} From 8d403c69d669de25e186897c50f09e03f8818c5c Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:03:01 -0700 Subject: [PATCH 054/127] test(remote): borrow selected file hash --- crates/crab-remote/tests/checkpoint/publication_catalog.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/crab-remote/tests/checkpoint/publication_catalog.rs b/crates/crab-remote/tests/checkpoint/publication_catalog.rs index 55e9964e1..40ccb2116 100644 --- a/crates/crab-remote/tests/checkpoint/publication_catalog.rs +++ b/crates/crab-remote/tests/checkpoint/publication_catalog.rs @@ -239,7 +239,7 @@ async fn fixed_pointer_publication_keeps_unrelated_catalog_history_cold() { assert!(matches!(outcome, crab_remote::prepare::CapsuleCommitOutcome::Committed { .. })); let after = open_history_view(&layout).await; assert_eq!(after.refs()[&target], oid); - assert!(after.pointer_catalog_lookup(&layout, LIMIT)?.files(layout.store(), &[file_hash.clone()]).await?.files().contains_key(&file_hash)); + assert!(after.pointer_catalog_lookup(&layout, LIMIT)?.files(layout.store(), std::slice::from_ref(&file_hash)).await?.files().contains_key(&file_hash)); drop(prepared); Ok::<_, Box>(()) }, From 4eccbdcb194bb414cb5dc042e83e3ebc3b9f61f0 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:28:09 -0700 Subject: [PATCH 055/127] fix(xet): enforce xorb chunk-count limit --- crates/crab-xet/README.md | 7 +- crates/crab-xet/src/xorb/builder.rs | 140 +++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 2 deletions(-) diff --git a/crates/crab-xet/README.md b/crates/crab-xet/README.md index bab7af39f..30ac02bb3 100644 --- a/crates/crab-xet/README.md +++ b/crates/crab-xet/README.md @@ -88,7 +88,12 @@ the upstream BG4 reader would buffer before reaching a caller's sink. Decoder block buffers, vector capacity, and the BG4 regrouping allocation are additional memory costs, so this is not a whole-process memory limit. -The builder and parser share the `u32` decoded-offset limit. Range assembly +The builder and parser share the `u32` decoded-offset and Xet chunk-count +limits. The builder seals before a new unique chunk would exceed either limit, +including when the payload is still below its target size. Serial, +precompressed, batch and admitted streaming writes share this boundary; +deduplication happens first, and denied rollover admission leaves staged data +unsealed for retry. Range assembly grows after validating decoded chunks instead of reserving an untrusted advertised total. These format checks are separate from request admission and whole-process memory limits. diff --git a/crates/crab-xet/src/xorb/builder.rs b/crates/crab-xet/src/xorb/builder.rs index 524f3864c..bdf366fa3 100644 --- a/crates/crab-xet/src/xorb/builder.rs +++ b/crates/crab-xet/src/xorb/builder.rs @@ -563,7 +563,11 @@ impl XorbBuilder { .checked_add(uncompressed_len) .ok_or_else(|| xorb_layout_overflow("uncompressed xorb size", uncompressed_len))?; let mut rolled = false; - if next_uncompressed > u64::from(u32::MAX) && !self.staged.is_empty() { + // Format limits override run continuity: small or highly compressed + // chunks must not produce a xorb that the parser will reject. + if (next_uncompressed > u64::from(u32::MAX) || self.staged.len() >= *MAX_XORB_CHUNKS) + && !self.staged.is_empty() + { self.finalize_current_after_admission(before_rollover)?; rolled = true; } @@ -2365,6 +2369,140 @@ mod tests { ); } + #[test] + fn chunk_count_limit_produces_verified_xorbs_for_every_ingestion_path() { + let chunks = (0..*MAX_XORB_CHUNKS + 2) + .map(|index| Chunk::new(Bytes::from((index as u64).to_le_bytes().to_vec()))) + .collect::>(); + let mut batch = chunks[..*MAX_XORB_CHUNKS] + .iter() + .cloned() + .map(|chunk| (chunk, RunId(0))) + .collect::>(); + batch.push((chunks[0].clone(), RunId(0))); + batch.extend( + chunks[*MAX_XORB_CHUNKS..] + .iter() + .cloned() + .map(|chunk| (chunk, RunId(0))), + ); + let mut expected = None; + for mode in ["serial", "precompressed", "batch", "admitted"] { + let mut builder = + XorbBuilder::with_policy(Arc::new(FixedCompression::new(CompressionScheme::None)) + as Arc); + let mut xorbs = Vec::new(); + match mode { + "serial" => { + for (chunk, run) in &batch { + builder.push(chunk, *run).unwrap(); + } + } + "precompressed" => { + for (chunk, run) in &batch { + builder + .push_precompressed( + chunk, + CompressionScheme::None, + chunk.data.to_vec(), + *run, + ) + .unwrap(); + } + } + "batch" => builder.push_batch(&batch).unwrap(), + "admitted" => xorbs = push_batch_and_collect(&mut builder, &batch).unwrap(), + _ => unreachable!(), + } + xorbs.extend(builder.finalize().unwrap()); + let mut decoded = Vec::new(); + for xorb in &xorbs { + let parser = crate::xorb::parser::XorbParser::parse(xorb.bytes.clone()).unwrap(); + parser.verify_payload_digest().unwrap(); + parser.verify_all_chunks().unwrap(); + assert_eq!(parser.hash(), xorb.hash); + for (index, placement) in xorb.placements.iter().enumerate() { + assert_eq!(placement.chunk_index, index as u32); + assert_eq!(placement.xorb_hash, xorb.hash); + decoded.push(parser.get_chunk(index as u32).unwrap().data); + } + } + assert_eq!( + decoded, + chunks + .iter() + .map(|chunk| chunk.data.clone()) + .collect::>(), + "{mode}" + ); + assert_eq!( + xorbs + .iter() + .map(|xorb| xorb.placements.len()) + .collect::>(), + vec![*MAX_XORB_CHUNKS, 2], + "{mode}" + ); + let bytes = xorbs + .iter() + .map(|xorb| xorb.bytes.clone()) + .collect::>(); + if let Some(expected) = &expected { + assert_eq!(&bytes, expected, "{mode}"); + } else { + expected = Some(bytes); + } + } + } + + #[test] + fn chunk_count_limit_preserves_staged_chunks_when_admission_fails() { + let batch = (0..=*MAX_XORB_CHUNKS) + .map(|index| { + ( + Chunk::new(Bytes::from((index as u64).to_le_bytes().to_vec())), + RunId(0), + ) + }) + .collect::>(); + let mut builder = + XorbBuilder::with_policy(Arc::new(FixedCompression::new(CompressionScheme::None)) + as Arc); + builder.push_batch(&batch[..*MAX_XORB_CHUNKS]).unwrap(); + builder + .push_batch_with_rollover_admission( + &batch[..1], + || -> Result<()> { panic!("dedup must not request rollover admission") }, + |_, ()| -> Result<()> { panic!("dedup must not seal the current xorb") }, + ) + .unwrap(); + let error = builder + .push_batch_with_rollover_admission( + &batch[*MAX_XORB_CHUNKS..], + || Err::<(), _>(XetError::Internal("admission denied".to_owned())), + |_, ()| -> Result<()> { panic!("failed admission must not publish a result") }, + ) + .unwrap_err(); + assert!(matches!(error, XetError::Internal(message) if message == "admission denied")); + assert_eq!(builder.staged_count(), *MAX_XORB_CHUNKS); + assert_eq!(builder.completed_count(), 0); + assert!(!builder.seen.contains(&batch[*MAX_XORB_CHUNKS].0.hash)); + let mut xorbs = push_batch_and_collect(&mut builder, &batch[*MAX_XORB_CHUNKS..]).unwrap(); + xorbs.extend(builder.finalize().unwrap()); + assert_eq!( + xorbs + .iter() + .map(|xorb| xorb.placements.len()) + .collect::>(), + vec![*MAX_XORB_CHUNKS, 1] + ); + for xorb in xorbs { + let parsed = crate::xorb::parser::XorbParser::parse(xorb.bytes).unwrap(); + parsed.verify_payload_digest().unwrap(); + parsed.verify_all_chunks().unwrap(); + } + } + #[test] fn push_rolls_over_before_uncompressed_xorb_size_exceeds_shard_limit() { let mut builder = XorbBuilder::new(); From ed6b5346cb61d770510072372188ea5f8322252f Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:55:11 -0700 Subject: [PATCH 056/127] fix(auth): isolate proof and page capsule catalogs --- crates/crab-auth-server/README.md | 14 + .../crab-auth-server/src/receive/capsule.rs | 51 ++- .../src/receive/capsule_dependencies.rs | 50 +++ .../src/receive/git_workspace.rs | 94 ++++- .../crab-auth-server/src/receive/session.rs | 15 +- .../crab-auth-server/src/receive/workflow.rs | 398 ++++++++++++++++-- crates/crab-metadata/README.md | 3 +- crates/crab-read/README.md | 8 + crates/crab-read/src/capsule_protocol.rs | 47 ++- .../tests/checkpoint/external_delta.rs | 117 ++++- 10 files changed, 700 insertions(+), 97 deletions(-) diff --git a/crates/crab-auth-server/README.md b/crates/crab-auth-server/README.md index 950e7ea74..05d1223a8 100644 --- a/crates/crab-auth-server/README.md +++ b/crates/crab-auth-server/README.md @@ -35,6 +35,20 @@ ref updates, staged Xet/Git objects, manifests, indexes, receipts, and active-active options. Its `Prepare`, `Verify`, and `Commit` commands make the publish boundary explicit; commit requires the verified plan digest. +Capsule receive captures source and filtered Git/catalog views separately. +Before source materialization, scoped candidates are verified using only the +filtered view and incoming packs. Newly reachable Git objects must be visible +under that capture or supplied by the candidate; retained physical objects do +not grant access. Both receive paths share the graph/visibility verifier. +Receive then queries pointer files and candidate shard/xorb identities under +each capture and applies the candidate delta. +File-only deltas can reference historical filtered artifacts; source-first +file selection and immutable descriptor conflicts remain enforced. Prepare +reads only ref authority. Git workspace intake combines captured packs and +candidate packs under the existing byte and external-delta checks, including +checkpoint-backed views. Physical shard/xorb verification and copy-before-ref +publication remain unchanged. + `crab-auth-view` materializes a filtered view using repeated `--read-path` and `--deny-path` rules, a scope hash, and a source repository URL. It builds the view's Git workspace and Crab metadata and reports the resulting prefixes, diff --git a/crates/crab-auth-server/src/receive/capsule.rs b/crates/crab-auth-server/src/receive/capsule.rs index f8bfc37cf..3a6475ac3 100644 --- a/crates/crab-auth-server/src/receive/capsule.rs +++ b/crates/crab-auth-server/src/receive/capsule.rs @@ -7,7 +7,8 @@ use crab_coordination::write_coordinator::{CommitOutcome, CoordinatedRefUpdate}; use crab_metadata::capsule_protocol::{Capsule, CapsuleRun, CapsuleSection, CapsuleSectionKind}; use super::capsule_dependencies::{ - DependencyCopy, promote_dependency_copies, source_pointer_delta, verify_pointer_dependencies, + DependencyCopy, promote_dependency_copies, select_pointer_catalog, source_pointer_delta, + verify_pointer_dependencies, }; use super::git_workspace::{materialize_capsule_source_push, verify_capsule_git_candidate}; use super::{ @@ -331,8 +332,10 @@ pub(super) async fn verify_capsule_candidate( .source_root_digest .as_deref() .ok_or_else(|| invalid("capsule prepare record is missing its root digest"))?; - let source_view = crab_read::capsule_protocol::open_view( + let source_root = crab_metadata::capsule_protocol::load_root(ctx.router()).await?; + let source_view = crab_read::capsule_protocol::open_git_view_from_root( ctx.router(), + source_root, crab_read::capsule_protocol::CapsuleReadLimits { max_capsule_bytes: MAX_PROTECTED_CAPSULE_BYTES, max_frontier_bytes: MAX_PROTECTED_CAPSULE_BYTES, @@ -354,16 +357,13 @@ pub(super) async fn verify_capsule_candidate( .ok_or_else(|| invalid("protected capsule run is empty"))?; validate_capsule_transaction(plan, candidate)?; + let candidate_delta = candidate.pointer_catalog_delta()?.unwrap_or_default(); let (changed_paths, publication, replication_objects, dependency_copies) = match prepare.view_scope.as_ref() { None => { if prepared_root != plan.base_root_digest { return Err(conflict("capsule base root differs from prepare record")); } - let mut catalog = source_view.pointer_catalog().await?; - if let Some(delta) = candidate.pointer_catalog_delta()? { - catalog.apply(&delta)?; - } let (changed_paths, pointers) = verify_capsule_git_candidate( ctx.store(), ctx.router(), @@ -373,6 +373,15 @@ pub(super) async fn verify_capsule_candidate( MAX_PROTECTED_CAPSULE_BYTES, ) .await?; + let mut catalog = select_pointer_catalog( + &source_view, + ctx.router(), + &pointers, + &candidate_delta, + MAX_PROTECTED_CAPSULE_BYTES, + ) + .await?; + catalog.apply(&candidate_delta)?; let (objects, copies) = verify_pointer_dependencies(ctx, plan, &catalog, &pointers, None).await?; (changed_paths, candidate.clone(), objects, copies) @@ -383,8 +392,11 @@ pub(super) async fn verify_capsule_candidate( scope.repo_prefix.clone(), scope.global_prefix.clone(), ); - let filtered_view = crab_read::capsule_protocol::open_view( + let filtered_root = + crab_metadata::capsule_protocol::load_root(&filtered_router).await?; + let filtered_view = crab_read::capsule_protocol::open_git_view_from_root( &filtered_router, + filtered_root, crab_read::capsule_protocol::CapsuleReadLimits { max_capsule_bytes: MAX_PROTECTED_CAPSULE_BYTES, max_frontier_bytes: MAX_PROTECTED_CAPSULE_BYTES, @@ -395,21 +407,36 @@ pub(super) async fn verify_capsule_candidate( return Err(conflict("filtered capsule root changed since prepare")); } validate_ref_heads(&prepare.view_ref_updates, filtered_view.refs())?; - let candidate_delta = candidate.pointer_catalog_delta()?.unwrap_or_default(); - let mut candidate_catalog = filtered_view.pointer_catalog().await?; - candidate_catalog.apply(&candidate_delta)?; let materialized = materialize_capsule_source_push( ctx.router(), &source_view, &filtered_view, + &filtered_router, candidate, &plan.ref_updates, &prepare.source_ref_updates, - &plan.transaction_id, MAX_PROTECTED_CAPSULE_BYTES, ) .await?; - let source_catalog = source_view.pointer_catalog().await?; + // Source and filtered captures have different authority and namespaces. + // Select each independently; source-first file policy is applied below. + let mut candidate_catalog = select_pointer_catalog( + &filtered_view, + &filtered_router, + &materialized.pointers, + &candidate_delta, + MAX_PROTECTED_CAPSULE_BYTES, + ) + .await?; + candidate_catalog.apply(&candidate_delta)?; + let source_catalog = select_pointer_catalog( + &source_view, + ctx.router(), + &materialized.pointers, + &candidate_catalog, + MAX_PROTECTED_CAPSULE_BYTES, + ) + .await?; let delta = source_pointer_delta( &source_catalog, &candidate_catalog, diff --git a/crates/crab-auth-server/src/receive/capsule_dependencies.rs b/crates/crab-auth-server/src/receive/capsule_dependencies.rs index a8f6697f0..a3d692b3e 100644 --- a/crates/crab-auth-server/src/receive/capsule_dependencies.rs +++ b/crates/crab-auth-server/src/receive/capsule_dependencies.rs @@ -22,6 +22,56 @@ pub(super) struct DependencyCopy { digest: String, } +pub(super) async fn select_pointer_catalog( + view: &crab_read::capsule_protocol::CapsuleRepositoryView, + layout: &crab_storage::StoreLayout, + pointers: &ReachablePointerScan, + candidate: &PointerCatalog, + maximum: u64, +) -> Result { + let files = pointers + .crab_pointers + .iter() + .map(|pointer| MerkleHash::from(pointer.file_hash).hex()) + .chain(candidate.files().keys().cloned()) + .collect::>() + .into_iter() + .collect::>(); + let shards = candidate + .shards() + .keys() + .cloned() + .chain( + candidate + .files() + .values() + .map(|file| file.shard_hash().to_owned()), + ) + .collect::>() + .into_iter() + .collect::>(); + let xorbs = candidate + .xorbs() + .keys() + .cloned() + .chain( + candidate + .shards() + .values() + .flat_map(|shard| shard.xorb_hashes().iter().cloned()), + ) + .collect::>() + .into_iter() + .collect::>(); + // Delta references may be owned by history rather than carried by the delta. + // Query every candidate identity too, so selected reads retain collision checks. + view.pointer_catalog_lookup(layout, maximum)? + .dependencies(layout.store(), &files, &shards, &xorbs) + .await + .map_err(crab_read::ReadError::from) + .map_err(Into::into) +} + pub(super) fn source_pointer_delta( source: &PointerCatalog, candidate: &PointerCatalog, diff --git a/crates/crab-auth-server/src/receive/git_workspace.rs b/crates/crab-auth-server/src/receive/git_workspace.rs index 6a6036163..245180864 100644 --- a/crates/crab-auth-server/src/receive/git_workspace.rs +++ b/crates/crab-auth-server/src/receive/git_workspace.rs @@ -1,6 +1,6 @@ //! Git workspace orchestration for protected-push receive. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fs; use std::io::Write; use std::path::Path; @@ -94,17 +94,30 @@ pub(super) async fn verify_capsule_git_candidate( let temp = tempfile::tempdir()?; let git_dir = temp.path().join("source.git"); run_git(["init", "--bare", path_str(&git_dir)?], None)?; - crab_read::capsule_protocol::install_git_packs_with_candidates( + crab_read::capsule_protocol::install_git_packs_with_candidates_from_store( view, std::slice::from_ref(candidate), + router, &git_dir, max_input_bytes, + None, + &tokio_util::sync::CancellationToken::new(), ) .await?; let workspace = GitReceiveWorkspace::new(store, router, router.repo_prefix()); let paths = workspace.compute_changed_paths_in(&git_dir, ref_updates, &BTreeMap::new(), false)?; + let refs = verify_capsule_candidate_refs(&git_dir, view, candidate).await?; + let pointers = scan_reachable_pointers_from_refs(&git_dir, &refs)?; + Ok((paths, pointers)) +} + +async fn verify_capsule_candidate_refs( + git_dir: &Path, + view: &crab_read::capsule_protocol::CapsuleRepositoryView, + candidate: &Capsule, +) -> Result> { let transaction = candidate.transaction()?; let mut refs = view.refs().clone(); let mut peeled = view.peeled_refs().clone(); @@ -127,25 +140,57 @@ pub(super) async fn verify_capsule_git_candidate( } } let refs = refs.into_iter().collect::>(); - let closures = crab_git::walk_reachable_by_ref_bounded( - &git_dir, + let reachable = crab_git::walk_reachable_by_ref_bounded( + git_dir, &refs, &peeled, usize::try_from(crab_metadata::git_visibility::MAX_GIT_VISIBILITY_OBJECTS) .map_err(|_| invalid("Git visibility object limit does not fit usize"))?, ) - .map_err(|source| AuthServerError::GitVisibilityWalk { source })? - .into_iter() - .map(|(name, reachable)| (name, reachable_object_ids(&reachable))) - .collect::>(); + .map_err(|source| AuthServerError::GitVisibilityWalk { source })?; + // Pack presence is not read authority. Reintroducing an invisible object + // requires its body in the verified candidate, even if a retained base pack + // happens to contain it. + { + let visible = view.git_visibility_index().await?; + let mut allowed = visible + .objects_for_refs(view.refs().keys().map(String::as_str)) + .into_iter() + .collect::>(); + for pack in candidate.git_packs() { + let index = candidate.section_bytes(pack.index_section())?; + let (oids, _) = crab_git::pack_locator::sorted_object_ids_from_index_bytes(&index) + .map_err(std::io::Error::other)?; + for oid in oids { + let oid = <[u8; 20]>::try_from(oid.as_bytes()).map_err(std::io::Error::other)?; + allowed.insert(oid); + } + } + if reachable.values().any(|objects| { + objects + .commits + .iter() + .chain(&objects.trees) + .chain(&objects.blobs) + .chain(&objects.tags) + .any(|oid| !allowed.contains(oid)) + }) { + return Err(invalid( + "candidate Git object is neither visible in the captured view nor supplied by its pack", + )); + } + } + let closures = reachable + .into_iter() + .map(|(name, objects)| (name, reachable_object_ids(&objects))) + .collect::>(); let declared = view.candidate_git_visibility(candidate).await?; if closures != declared { return Err(invalid( "candidate capsule Git visibility differs from its reachable object closure", )); } - let pointers = scan_reachable_pointers_from_refs(&git_dir, &refs)?; - Ok((paths, pointers)) + Ok(refs) } pub(super) struct MaterializedCapsuleSourcePush { @@ -160,10 +205,10 @@ pub(super) async fn materialize_capsule_source_push( source_router: &StoreLayout, source_view: &crab_read::capsule_protocol::CapsuleRepositoryView, filtered_view: &crab_read::capsule_protocol::CapsuleRepositoryView, + filtered_router: &StoreLayout, candidate: &Capsule, view_updates: &[PushRefUpdate], source_updates: &[PushRefUpdate], - candidate_transaction_id: &str, max_input_bytes: u64, ) -> Result { if view_updates.len() != source_updates.len() { @@ -172,18 +217,14 @@ pub(super) async fn materialize_capsule_source_push( let temp = tempfile::tempdir()?; let git_dir = temp.path().join("source.git"); run_git(["init", "--bare", path_str(&git_dir)?], None)?; - crab_read::capsule_protocol::install_git_packs_with_candidates( - source_view, - &[], - &git_dir, - max_input_bytes, - ) - .await?; - crab_read::capsule_protocol::install_git_packs_with_candidates( + crab_read::capsule_protocol::install_git_packs_with_candidates_from_store( filtered_view, std::slice::from_ref(candidate), + filtered_router, &git_dir, max_input_bytes, + None, + &tokio_util::sync::CancellationToken::new(), ) .await?; @@ -194,6 +235,19 @@ pub(super) async fn materialize_capsule_source_push( ); let changed_paths = workspace.compute_changed_paths_in(&git_dir, view_updates, filtered_view.refs(), true)?; + // Prove the candidate using only filtered and incoming objects before the + // private source ODB becomes available for source-commit synthesis. + let _ = verify_capsule_candidate_refs(&git_dir, filtered_view, candidate).await?; + crab_read::capsule_protocol::install_git_packs_with_candidates_from_store( + source_view, + &[], + source_router, + &git_dir, + max_input_bytes, + None, + &tokio_util::sync::CancellationToken::new(), + ) + .await?; let mut final_updates = Vec::with_capacity(source_updates.len()); let mut synthesized_tips = Vec::with_capacity(source_updates.len()); for (view_update, source_update) in view_updates.iter().zip(source_updates) { @@ -251,7 +305,7 @@ pub(super) async fn materialize_capsule_source_push( )?; let transaction = CapsuleTransaction::for_protected_source( source_view.root().digest(), - candidate_transaction_id, + candidate.transaction_id(), final_updates .iter() .map(|update| { diff --git a/crates/crab-auth-server/src/receive/session.rs b/crates/crab-auth-server/src/receive/session.rs index 543bb1b68..a5b73675b 100644 --- a/crates/crab-auth-server/src/receive/session.rs +++ b/crates/crab-auth-server/src/receive/session.rs @@ -224,20 +224,13 @@ impl ReceiveContext { }, other => other.into(), })?; - let view = crab_read::capsule_protocol::open_view_from_root_with_control( - &self.router, - root, - crab_read::capsule_protocol::CapsuleReadLimits { - max_capsule_bytes: 2 * 1024 * 1024 * 1024, - max_frontier_bytes: 2 * 1024 * 1024 * 1024, - }, - ) - .await?; + let view = crab_read::capsule_protocol::open_ref_view_from_root(&self.router, root) + .await?; build_capsule_prepare_record( &self.repo_prefix, &self.push_id, - view.root().root().generation(), - view.root().digest(), + view.root_snapshot().record().root().generation(), + view.root_snapshot().record().digest(), view.refs(), view_ref_updates, view_scope, diff --git a/crates/crab-auth-server/src/receive/workflow.rs b/crates/crab-auth-server/src/receive/workflow.rs index 818e2ac31..e25927e46 100644 --- a/crates/crab-auth-server/src/receive/workflow.rs +++ b/crates/crab-auth-server/src/receive/workflow.rs @@ -1272,8 +1272,11 @@ mod tests { Some(source.path()), )?; run_git(["config", "user.name", "Alice"], Some(source.path()))?; + let xet_content = b"unrestricted protected Xet content\n"; + let xet = xet_fixture(xet_content)?; std::fs::write(source.path().join("tracked.txt"), b"protected capsule\n")?; - run_git(["add", "tracked.txt"], Some(source.path()))?; + std::fs::write(source.path().join("model.bin"), xet.pointer.serialize())?; + run_git(["add", "tracked.txt", "model.bin"], Some(source.path()))?; run_git(["commit", "-m", "capsule candidate"], Some(source.path()))?; let tip = run_git_capture(["-C", path_str(source.path())?, "rev-parse", "HEAD"])? .trim() @@ -1322,15 +1325,28 @@ mod tests { let capsule = Capsule::build( &transaction, vec![pack], - vec![CapsuleSection::new( - CapsuleSectionKind::VisibilityDelta, - visibility.encode()?, - )], + vec![ + CapsuleSection::new(CapsuleSectionKind::VisibilityDelta, visibility.encode()?), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + xet.catalog.encode_delta()?, + ), + ], )?; let run = CapsuleRun::leaf(capsule)?; let run_key = ctx.router().capsule_path(run.hash()).to_string(); let run_object = staged_object(run_key, run.bytes()); put_staged(&ctx, &run_object, run.bytes().to_vec()).await?; + let xorb_object = staged_object( + ctx.router().xorb_path(&xet.xorb_hash).to_string(), + &xet.xorb_bytes, + ); + put_staged(&ctx, &xorb_object, xet.xorb_bytes.to_vec()).await?; + let shard_object = staged_object( + ctx.router().shard_path(&xet.shard_hash).to_string(), + &xet.shard_bytes, + ); + put_staged(&ctx, &shard_object, xet.shard_bytes.clone()).await?; let plan = super::super::ProtectedCapsulePushPlan { schema_version: crab_remote::protected::PROTECTED_CAPSULE_PUSH_PLAN_SCHEMA_VERSION, repo_prefix: ctx.repo_prefix().to_owned(), @@ -1341,15 +1357,24 @@ mod tests { run_hash: run.hash().to_owned(), run_size: run.bytes().len() as u64, ref_updates: vec![update], - staged_objects: vec![run_object], + staged_objects: vec![run_object, xorb_object, shard_object], }; write_plan(&ctx, &plan).await?; let verified = verify_receive(&ctx).await?; assert_eq!(verified.ref_updates, plan.ref_updates); - assert_eq!(verified.verified_changed_paths, vec!["tracked.txt"]); - assert_eq!(verified.verified_staged_bytes, plan.run_size); + assert_eq!( + verified.verified_changed_paths, + vec!["model.bin", "tracked.txt"] + ); + assert_eq!( + verified.verified_staged_bytes, + plan.staged_objects + .iter() + .map(|object| object.size) + .sum::() + ); let response = commit_receive(&ctx, "crab://bucket/org/repo", &verified.plan_digest, None).await?; assert_eq!(response.ref_updates, plan.ref_updates); @@ -1366,6 +1391,21 @@ mod tests { view.visible_ref_transactions().get("refs/heads/main"), Some(&plan.transaction_id) ); + let catalog = view + .pointer_catalog_lookup(ctx.router(), 1024 * 1024)? + .files( + ctx.store(), + &[MerkleHash::from(xet.pointer.file_hash).hex()], + ) + .await?; + assert_eq!(catalog, xet.catalog); + crab_read::verify_capsule_pointer_catalog_objects(ctx.router(), &catalog).await?; + let (body, _) = ctx + .store() + .get_with_etag(&ctx.router().xorb_path(&xet.xorb_hash)) + .await?; + let parsed = crab_xet::xorb::parser::XorbParser::parse(body)?; + assert_eq!(parsed.get_chunk(0)?.data.as_ref(), xet_content); let receipt = crab_metadata::capsule_protocol::resolve_capsule_plan_receipt( ctx.store(), ctx.router(), @@ -1383,7 +1423,165 @@ mod tests { #[tokio::test] async fn protected_capsule_view_push_preserves_hidden_paths_and_dependencies() -> Result<()> { + for count in [100, 10_000] { + protected_capsule_view_push(false, count, None).await?; + } + Ok(()) + } + + #[tokio::test] + async fn checkpointed_capsule_view_push_preserves_hidden_paths_and_dependencies() -> Result<()> + { + for count in [100, 10_000] { + protected_capsule_view_push(true, count, None).await?; + } + Ok(()) + } + + #[tokio::test] + async fn scoped_capsule_candidate_cannot_borrow_hidden_source_git_objects() -> Result<()> { + protected_capsule_view_push(false, 100, Some("source")).await + } + + #[tokio::test] + async fn scoped_capsule_candidate_cannot_borrow_invisible_retained_git_objects() -> Result<()> { + protected_capsule_view_push(true, 100, Some("filtered")).await + } + + #[derive(Default)] + struct CatalogReads(AtomicU64, AtomicU64); + + impl crab_storage::StorageObserver for CatalogReads { + fn started(&self, _: crab_storage::StorageOperation) {} + + fn finished(&self, observation: crab_storage::StorageObservation) { + if matches!( + observation.operation, + crab_storage::StorageOperation::Get | crab_storage::StorageOperation::Range + ) { + self.0.fetch_add(1, Ordering::SeqCst); + self.1.fetch_add(observation.bytes_read, Ordering::SeqCst); + } + } + } + + async fn historical_catalog( + layout: &crab_storage::StoreLayout, + count: usize, + ) -> Result { + let mut builder = XorbBuilder::new(); + let mut files = Vec::new(); + for index in 0..count { + let data = Bytes::from(format!("unrelated historical payload {index:016x}\n")); + let file = MerkleHash::from(*blake3::hash(&data).as_bytes()); + let chunk = crab_xet::xorb::format::Chunk::new(data.clone()); + builder.push(&chunk, RunId(index as u64))?; + files.push((file, chunk.hash, data.len() as u64)); + } + let xorbs = builder.finalize()?; + let placements = xorbs + .iter() + .flat_map(|xorb| xorb.placements.iter()) + .map(|placement| (placement.chunk_hash, placement.clone())) + .collect::(); + let dependencies = xorbs + .iter() + .map(|xorb| { + Ok(( + xorb.hash, + Arc::new(xorb_info_from_placements(xorb.hash, &xorb.placements)?), + )) + }) + .collect::>>()?; + let mut shards = PushShardSession::new(); + let mut file_shards = Vec::new(); + let mut shard_xorbs = BTreeMap::>::new(); + for (hash, chunk, size) in files { + let dependency = &dependencies[&placements[&chunk].xorb_hash]; + let index = shards.add_file_bundle( + file_info_from_placements(hash, &[chunk], &placements)?, + std::slice::from_ref(dependency), + )?; + file_shards.push((hash, size, index)); + shard_xorbs + .entry(index) + .or_default() + .insert(placements[&chunk].xorb_hash.hex()); + } + let shards = shards.finalize()?; + let mut catalog = PointerCatalog::new(); + for xorb in xorbs { + let mut chunks = xorb.placements.iter().collect::>(); + chunks.sort_by_key(|chunk| chunk.chunk_index); + catalog.insert_xorb( + xorb.hash.hex(), + XorbCatalogEntry::new( + xorb.bytes.len() as u64, + blake3_hex(&xorb.bytes), + chunks + .into_iter() + .map(|chunk| { + XorbChunkEntry::new(chunk.chunk_hash.hex(), chunk.uncompressed_size) + }) + .collect(), + ), + )?; + layout + .store() + .put_exact(&layout.xorb_path(&xorb.hash), xorb.bytes) + .await?; + } + for (index, (bytes, hash)) in shards.iter().enumerate() { + catalog.insert_shard( + hash.hex(), + ShardCatalogEntry::new( + bytes.len() as u64, + shard_xorbs[&index].iter().cloned().collect(), + ), + )?; + layout + .store() + .put_exact(&layout.shard_path(hash), Bytes::from(bytes.clone())) + .await?; + } + for (hash, size, index) in file_shards { + catalog.insert_file( + hash.hex(), + FileCatalogEntry::new(size, shards[index].1.hex()), + )?; + } + Ok(catalog) + } + + async fn protected_capsule_view_push( + checkpointed: bool, + count: usize, + borrowed_from: Option<&str>, + ) -> Result<()> { let ctx = capsule_context().await?; + let counts = Arc::new(CatalogReads::default()); + let ctx = ReceiveContext::from_store( + ctx.store().clone().with_storage_observer(counts.clone()), + ctx.repo_prefix().to_owned(), + ctx.push_id().to_owned(), + ); + let history = historical_catalog(ctx.router(), count).await?; + let hidden_content = b"source-only Xet content\n"; + let hidden = xet_fixture(hidden_content)?; + ctx.store() + .put_exact( + &ctx.router().xorb_path(&hidden.xorb_hash), + hidden.xorb_bytes.clone(), + ) + .await?; + ctx.store() + .put_exact( + &ctx.router().shard_path(&hidden.shard_hash), + Bytes::from(hidden.shard_bytes.clone()), + ) + .await?; + let mut source_catalog = history.clone(); + source_catalog.apply(&hidden.catalog)?; let temp = tempfile::tempdir()?; let source = temp.path().join("source"); run_git(["init", "--initial-branch=main", path_str(&source)?], None)?; @@ -1393,11 +1591,20 @@ mod tests { std::fs::create_dir_all(source.join("secret"))?; std::fs::write(source.join("src/app.txt"), b"allowed v1\n")?; std::fs::write(source.join("secret/key.txt"), b"classified\n")?; + std::fs::write(source.join("secret/model.bin"), hidden.pointer.serialize())?; run_git(["add", "."], Some(&source))?; run_git(["commit", "-m", "source base"], Some(&source))?; let source_old = run_git_capture(["-C", path_str(&source)?, "rev-parse", "HEAD"])? .trim() .to_owned(); + let hidden_oid = run_git_capture([ + "-C", + path_str(&source)?, + "rev-parse", + &format!("{source_old}:secret/key.txt"), + ])? + .trim() + .to_owned(); let source_pack_bytes = git_capture_with_input( ["pack-objects", "--stdout", "--revs"], &source, @@ -1428,10 +1635,16 @@ mod tests { let source_capsule = Capsule::build( &source_transaction, vec![source_pack], - vec![CapsuleSection::new( - CapsuleSectionKind::VisibilityDelta, - source_visibility.encode()?, - )], + vec![ + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + source_visibility.encode()?, + ), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + source_catalog.encode_delta()?, + ), + ], )?; crab_write::capsule_protocol::publish( ctx.router(), @@ -1470,15 +1683,29 @@ mod tests { "refs/heads/main", ) .await?; + assert_eq!(historical_catalog(&view_router, count).await?, history); + let view_base_revisions = if borrowed_from == Some("filtered") { + let oid = git_capture_with_input( + ["hash-object", "-w", "--stdin"], + &filtered, + b"classified\n", + )?; + assert_eq!(String::from_utf8(oid).unwrap().trim(), hidden_oid); + // Physical retention is permitted; the base visibility proof below + // deliberately grants no ref access to this extra blob. + format!("{view_old}\n{hidden_oid}\n") + } else { + format!("{view_old}\n") + }; let view_base_pack_bytes = git_capture_with_input( ["pack-objects", "--stdout", "--revs"], &filtered, - format!("{view_old}\n").as_bytes(), + view_base_revisions.as_bytes(), )?; let view_base_pack = capsule_git_pack( &filtered.join(".git"), &view_base_pack_bytes, - git_object_count(&filtered, &format!("{view_old}\n"))?, + git_object_count(&filtered, &view_base_revisions)?, )?; let view_base_transaction = CapsuleTransaction::new( view_root.record().digest(), @@ -1496,22 +1723,6 @@ mod tests { &view_old, &format!("{view_old}\n"), )?; - let view_base_capsule = Capsule::build( - &view_base_transaction, - vec![view_base_pack], - vec![CapsuleSection::new( - CapsuleSectionKind::VisibilityDelta, - view_base_visibility.encode()?, - )], - )?; - crab_write::capsule_protocol::publish( - &view_router, - view_root, - &view_base_transaction, - &view_base_capsule, - ) - .await?; - let xet_content = b"protected view Xet content\n"; let xet = xet_fixture(xet_content)?; ctx.store() @@ -1526,6 +1737,49 @@ mod tests { Bytes::from(xet.shard_bytes.clone()), ) .await?; + let mut filtered_catalog = history.clone(); + for (hash, entry) in xet.catalog.xorbs() { + filtered_catalog.insert_xorb(hash.clone(), entry.clone())?; + } + for (hash, entry) in xet.catalog.shards() { + filtered_catalog.insert_shard(hash.clone(), entry.clone())?; + } + let view_base_capsule = Capsule::build( + &view_base_transaction, + vec![view_base_pack], + vec![ + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + view_base_visibility.encode()?, + ), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + filtered_catalog.encode_delta()?, + ), + ], + )?; + crab_write::capsule_protocol::publish( + &view_router, + view_root, + &view_base_transaction, + &view_base_capsule, + ) + .await?; + + if checkpointed { + for layout in [ctx.router(), &view_router] { + let outcome = crab_remote::checkpoint::publish_capsule_checkpoint( + layout, + 0, + 2 * 1024 * 1024 * 1024, + &CancellationToken::new(), + ) + .await + .map_err(std::io::Error::other)?; + assert!(outcome.published); + } + } + let lfs_content = b"protected view LFS content\n"; let lfs_oid = <[u8; 32]>::from(Sha256::digest(lfs_content)); let lfs_pointer = crab_git::lfs_pointer::LfsPointer { @@ -1540,6 +1794,10 @@ mod tests { ["add", "src/app.txt", "src/model.bin", "src/asset.lfs"], Some(&filtered), )?; + if borrowed_from.is_some() { + std::fs::write(filtered.join("src/leak.txt"), b"classified\n")?; + run_git(["add", "src/leak.txt"], Some(&filtered))?; + } run_git(["commit", "-m", "allowed update"], Some(&filtered))?; let view_new = run_git_capture(["-C", path_str(&filtered)?, "rev-parse", "HEAD"])? .trim() @@ -1568,15 +1826,23 @@ mod tests { }, ) .await?; + let candidate_revisions = if borrowed_from.is_some() { + // The candidate declares this source-only blob reachable but omits + // its body from the incoming pack. Source materialization must not + // satisfy that missing dependency from its broader private ODB. + format!("{view_new}\n^{view_old}\n^{hidden_oid}\n") + } else { + format!("{view_new}\n^{view_old}\n") + }; let candidate_pack_bytes = git_capture_with_input( ["pack-objects", "--stdout", "--revs"], &filtered, - format!("{view_new}\n^{view_old}\n").as_bytes(), + candidate_revisions.as_bytes(), )?; let candidate_pack = capsule_git_pack( &filtered.join(".git"), &candidate_pack_bytes, - git_object_count(&filtered, &format!("{view_new}\n^{view_old}\n"))?, + git_object_count(&filtered, &candidate_revisions)?, )?; let candidate_transaction = CapsuleTransaction::new( view.root().digest(), @@ -1594,6 +1860,10 @@ mod tests { &view_new, &format!("{view_new}\n"), )?; + let mut candidate_catalog = PointerCatalog::new(); + for (hash, entry) in xet.catalog.files() { + candidate_catalog.insert_file(hash.clone(), entry.clone())?; + } let candidate = Capsule::build( &candidate_transaction, vec![candidate_pack], @@ -1604,7 +1874,7 @@ mod tests { ), CapsuleSection::new( CapsuleSectionKind::CatalogDelta, - xet.catalog.encode_delta()?, + candidate_catalog.encode_delta()?, ), ], )?; @@ -1633,7 +1903,25 @@ mod tests { }; write_plan(&ctx, &plan).await?; - let verified = verify_receive(&ctx).await?; + counts.0.store(0, Ordering::SeqCst); + counts.1.store(0, Ordering::SeqCst); + let result = verify_receive(&ctx).await; + if borrowed_from.is_some() { + assert!( + result.is_err(), + "scoped candidate borrowed a hidden {borrowed_from:?} blob without supplying its body" + ); + let refs = crab_read::capsule_protocol::read_visible_refs(ctx.router()).await?; + assert_eq!(refs["refs/heads/main"], source_old); + return Ok(()); + } + let verified = result?; + assert!( + counts.0.load(Ordering::SeqCst) <= 64 && counts.1.load(Ordering::SeqCst) <= 512 * 1024, + "fixed protected verification exceeded read bounds: count={count}, checkpoint={checkpointed}, reads={}, bytes={}", + counts.0.load(Ordering::SeqCst), + counts.1.load(Ordering::SeqCst), + ); assert_eq!( verified.verified_changed_paths, vec!["src/app.txt", "src/asset.lfs", "src/model.bin"] @@ -1656,11 +1944,13 @@ mod tests { assert_ne!(final_oid, &view_new); let final_git = temp.path().join("final.git"); run_git(["init", "--bare", path_str(&final_git)?], None)?; - crab_read::capsule_protocol::install_git_packs_with_candidates( + crab_read::capsule_protocol::install_git_packs_from_store( &source_view, - &[], + ctx.router(), &final_git, u64::MAX, + None, + &CancellationToken::new(), ) .await?; assert_eq!( @@ -1681,6 +1971,13 @@ mod tests { ])?, "classified\n" ); + let hidden_pointer = run_git_capture([ + "--git-dir", + path_str(&final_git)?, + "show", + &format!("{final_oid}:secret/model.bin"), + ])?; + assert_eq!(Pointer::parse(hidden_pointer.as_bytes())?, hidden.pointer); let final_pointer = run_git_capture([ "--git-dir", path_str(&final_git)?, @@ -1722,17 +2019,26 @@ mod tests { ctx.store(), ctx.repo_prefix(), ); - let hydrated = temp.path().join("hydrated-model.bin"); - hydrator - .reconstruct_to_writer_with_cancel( - &xet.pointer, - std::fs::File::create(&hydrated)?, - Some(&lookup), - &CancellationToken::new(), - ) - .await?; + for (name, pointer, expected) in [ + ("hydrated-model.bin", &xet.pointer, xet_content.as_slice()), + ( + "hydrated-hidden.bin", + &hidden.pointer, + hidden_content.as_slice(), + ), + ] { + let hydrated = temp.path().join(name); + hydrator + .reconstruct_to_writer_with_cancel( + pointer, + std::fs::File::create(&hydrated)?, + Some(&lookup), + &CancellationToken::new(), + ) + .await?; + assert_eq!(std::fs::read(hydrated)?, expected); + } lookup.close().await?; - assert_eq!(std::fs::read(hydrated)?, xet_content); let (stored_lfs, _) = ctx .store() .get_with_etag_bounded(&ObjectPath::from(lfs_key), lfs_content.len() as u64) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 4955ae875..7a9cc39e8 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -72,7 +72,8 @@ capsule checkpoint formats; unshipped format 6 is rejected without a fallback. Captured run-v4 roots provide the same selected lookup over exact logical capsule occurrences; catalog bodies remain cold until their keys are selected. CLI and shared capsule-publication preparation use these captured queries. -Protected receive and hydration still require separate consumer integration. +Protected receive selects from separately authorized source/filtered captures; +hydration still requires consumer integration. External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 22f08e250..8a6ea11d4 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -66,6 +66,14 @@ metadata's shared file lookup. A lookup loads only its selected file dependency closure, preserving the captured publication order and immutable descriptor checks. It never substitutes a later catalog or mutable file-index session. +`install_git_packs_with_candidates_from_store` combines a captured view's +origin-backed packs with verified candidate capsules. Candidates participate +in the same pack intake limit and external-delta ordering, so protected receive +can use a paged view without materializing its catalog. The caller binds each +candidate to the captured root; installation does not grant visibility. +The existing candidate-free installer delegates to this implementation and +retains its cold-clone proof and cache behavior. + `ShardHydrator` provides memory, file, and half-open byte-range reconstruction. Before full-file reconstruction it checks that the pointer's shard terms are covered; after reconstruction it verifies the requested file hash. A shared diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index edad84b9d..f52bb40d4 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -2150,24 +2150,59 @@ pub async fn install_git_packs_from_store( cache: Option<&crab_cache_store::CachingStore>, cancel: &CancellationToken, ) -> Result { - if cancel.is_cancelled() { - return Err(ReadError::Cancelled); - } - if let Some(installed) = install_layered_cold_clone_packs_from_store( + install_git_packs_with_candidates_from_store( view, + &[], router, git_dir, max_input_bytes, cache, cancel, ) - .await? + .await +} + +/// Install a captured origin-backed view and verified candidate capsules together. +/// +/// Candidates must be bound to the captured root by the caller. Base and candidate +/// packs share intake limits and external-delta ordering; no catalog bodies are read. +pub async fn install_git_packs_with_candidates_from_store( + view: &CapsuleRepositoryView, + candidates: &[Capsule], + router: &StoreLayout, + git_dir: &Path, + max_input_bytes: u64, + cache: Option<&crab_cache_store::CachingStore>, + cancel: &CancellationToken, +) -> Result { + if cancel.is_cancelled() { + return Err(ReadError::Cancelled); + } + // The cold-clone proof covers only the captured view. Candidate packs must + // join the same intake and delta-base ordering before anything is installed. + if candidates.is_empty() + && let Some(installed) = install_layered_cold_clone_packs_from_store( + view, + router, + git_dir, + max_input_bytes, + cache, + cancel, + ) + .await? { return Ok(installed); } if view.layered_checkpoint().is_some() || (view.capsules.is_empty() && !view.capsule_run_sources.is_empty()) + || !candidates.is_empty() { + let capsules = view + .capsules + .iter() + .chain(candidates) + .cloned() + .collect::>(); // Metadata views expose authenticated source ranges before the first // checkpoint too. Installing only resident capsules would drop their packs. let sources = view @@ -2176,7 +2211,7 @@ pub async fn install_git_packs_from_store( let paths = install_layered_git_packs_from_store_selected_with_sources( sources, &view.capsule_run_sources, - &view.capsules, + &capsules, router, git_dir, max_input_bytes, diff --git a/crates/crab-remote/tests/checkpoint/external_delta.rs b/crates/crab-remote/tests/checkpoint/external_delta.rs index c50bc3681..63960bbca 100644 --- a/crates/crab-remote/tests/checkpoint/external_delta.rs +++ b/crates/crab-remote/tests/checkpoint/external_delta.rs @@ -10,6 +10,19 @@ async fn publish_dependent_blob( base: &[u8], target: &[u8], ) { + let (root, capsule) = dependent_blob_capsule(layout, name, base, target).await; + let transaction = capsule.transaction().unwrap(); + crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + .await + .unwrap(); +} + +async fn dependent_blob_capsule( + layout: &StoreLayout, + name: &str, + base: &[u8], + target: &[u8], +) -> (crab_metadata::capsule_protocol::RootSnapshot, Capsule) { let kind = gix_object::Kind::Blob; let base_oid = crab_remote::objects::object_id(kind, base).unwrap(); let target_oid = crab_remote::objects::object_id(kind, target).unwrap(); @@ -82,9 +95,111 @@ async fn publish_dependent_blob( )], ) .unwrap(); - crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + (root, capsule) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn candidate_install_shares_intake_and_resolves_captured_external_bases() { + for checkpointed in [false, true] { + let (layout, observations) = empty_fixture().await; + let base = (0..4096_u32) + .flat_map(|ordinal| *blake3::hash(&ordinal.to_le_bytes()).as_bytes()) + .collect::>(); + let mut target = base.clone(); + for byte in &mut target[..64 * 1024] { + *byte ^= 0xa5; + } + publish_blobs(&layout, &[("base", &base)]).await; + if checkpointed { + checkpoint(&layout).await; + } + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let captured = crab_read::capsule_protocol::open_git_view_from_root(&layout, root, LIMITS) + .await + .unwrap(); + let (root, candidate) = dependent_blob_capsule(&layout, "candidate", &base, &target).await; + assert_eq!(root.record().digest(), captured.root().digest()); + let candidate_size = candidate + .section_bytes(candidate.git_packs()[0].pack_section()) + .unwrap() + .len() as u64; + let base_sizes = captured + .layered_checkpoint() + .into_iter() + .flat_map(|checkpoint| checkpoint.sources()) + .chain(captured.capsule_run_sources()) + .flat_map(|source| source.members()) + .map(|member| (member.pack().blake3().to_owned(), member.pack().length())) + .collect::>(); + let base_size = base_sizes.values().sum::(); + let budget = 160 * 1024; + assert!( + base_size < budget && candidate_size < budget && base_size + candidate_size > budget + ); + let bounded = tempfile::tempdir().unwrap(); + crab_git::initialize_bare_git_dir(bounded.path()).unwrap(); + let result = crab_read::capsule_protocol::install_git_packs_with_candidates_from_store( + &captured, + std::slice::from_ref(&candidate), + &layout, + bounded.path(), + budget, + None, + &CancellationToken::new(), + ) + .await; + assert!(matches!( + result, + Err(crab_read::ReadError::CapsuleReadLimit { + resource: "Git pack intake", + .. + }) + )); + let directory = tempfile::tempdir().unwrap(); + crab_git::initialize_bare_git_dir(directory.path()).unwrap(); + let cancelled = CancellationToken::new(); + cancelled.cancel(); + observations.0.lock().unwrap().clear(); + let result = crab_read::capsule_protocol::install_git_packs_with_candidates_from_store( + &captured, + std::slice::from_ref(&candidate), + &layout, + directory.path(), + LIMIT, + None, + &cancelled, + ) + .await; + assert!(matches!(result, Err(crab_read::ReadError::Cancelled))); + assert!(observations.0.lock().unwrap().is_empty()); + let installed = crab_read::capsule_protocol::install_git_packs_with_candidates_from_store( + &captured, + std::slice::from_ref(&candidate), + &layout, + directory.path(), + LIMIT, + None, + &CancellationToken::new(), + ) .await .unwrap(); + assert!(!installed.complete_visibility); + let oid = crab_remote::objects::object_id(gix_object::Kind::Blob, &target).unwrap(); + let output = std::process::Command::new("git") + .arg("--git-dir") + .arg(directory.path()) + .args(["cat-file", "blob", &oid.to_string()]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, target); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 994c236c4b3d4e72f3cf6617af60456224a13aba Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 23:17:38 -0700 Subject: [PATCH 057/127] test(read): cover catalog lifetimes after push --- crates/crab-metadata/README.md | 5 +- .../src/file_index_lookup/catalog_tests.rs | 4 +- .../src/file_index_lookup/shared_tests.rs | 63 +++++++++++++++++++ .../tests/checkpoint/publication_catalog.rs | 28 ++++++++- 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 7a9cc39e8..7678cf595 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -72,8 +72,9 @@ capsule checkpoint formats; unshipped format 6 is rejected without a fallback. Captured run-v4 roots provide the same selected lookup over exact logical capsule occurrences; catalog bodies remain cold until their keys are selected. CLI and shared capsule-publication preparation use these captured queries. -Protected receive selects from separately authorized source/filtered captures; -hydration still requires consumer integration. +Protected receive selects from separately authorized source/filtered captures. +Captured revision and remote-mount hydration pass the same paged lookup to the +shared hydrator; current-state hydration still requires capture integration. External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs index a035cf9cb..458aacdb7 100644 --- a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -7,7 +7,7 @@ use bytes::Bytes; use object_store::memory::InMemory; use std::sync::atomic::AtomicUsize; -async fn capture( +pub(super) async fn capture( layout: &crab_storage::StoreLayout, catalog: PointerCatalog, ) -> (PointerCatalogLookup, object_store::path::Path) { @@ -48,7 +48,7 @@ async fn capture( ) } -fn catalog(count: usize) -> PointerCatalog { +pub(super) fn catalog(count: usize) -> PointerCatalog { let mut result = PointerCatalog::new(); for index in 0..count { let file = super::tests::hash_from_seed(index as u64); diff --git a/crates/crab-metadata/src/file_index_lookup/shared_tests.rs b/crates/crab-metadata/src/file_index_lookup/shared_tests.rs index c2db586e9..2a52edc02 100644 --- a/crates/crab-metadata/src/file_index_lookup/shared_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/shared_tests.rs @@ -345,3 +345,66 @@ async fn concurrent_close_waits_for_reader_checkpoint_cleanup() { "concurrent close returned before reader cleanup" ); } + +#[tokio::test(flavor = "multi_thread")] +async fn captured_catalog_close_waits_for_active_reads_and_abandonment_releases_them() { + for abandon in [false, true] { + let inner: Arc = Arc::new(InMemory::new()); + let storage = crab_storage::Store::new(inner.clone()); + let router = crab_storage::StoreLayout::new(storage, "pinned/close".to_owned()); + let (catalog, path) = + super::catalog_tests::capture(&router, super::catalog_tests::catalog(10_000)).await; + let store = Arc::new(PausedLookupStore { + inner, + shard: path, + paused: AtomicBool::new(true), + pause_write: AtomicBool::new(false), + entered: Notify::new(), + release: Semaphore::new(0), + }); + let router = crab_storage::StoreLayout::new( + crab_storage::Store::new(store.clone()), + "pinned/close".to_owned(), + ); + let lookup = SharedFileIndexLookup::for_pointer_catalog(router, catalog); + let file = hash_from_seed(0); + lookup.lookup(&file).await.unwrap().unwrap(); + store.paused.store(false, Ordering::Release); + let cold_file = hash_from_seed(9_999); + let task = tokio::spawn({ + let lookup = lookup.clone(); + async move { lookup.lookup_batch(&[cold_file]).await } + }); + timeout(Duration::from_secs(5), store.entered.notified()) + .await + .expect("query reached captured page"); + let warm = timeout(Duration::from_secs(2), lookup.lookup(&file)) + .await + .expect("a cold page read must not serialize a warm query") + .unwrap(); + assert_eq!(warm, Some(hash_from_seed(1_000_000))); + let close = lookup.clone().close(); + tokio::pin!(close); + assert!(futures_util::poll!(close.as_mut()).is_pending()); + assert!(matches!( + lookup.lookup(&file).await, + Err(MetadataError::Internal(_)) + )); + if abandon { + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + } else { + store.release.add_permits(1); + let result = timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(result, vec![Some(hash_from_seed(1_009_999))]); + } + timeout(Duration::from_secs(5), close) + .await + .unwrap() + .unwrap(); + } +} diff --git a/crates/crab-remote/tests/checkpoint/publication_catalog.rs b/crates/crab-remote/tests/checkpoint/publication_catalog.rs index 40ccb2116..b1493ff7c 100644 --- a/crates/crab-remote/tests/checkpoint/publication_catalog.rs +++ b/crates/crab-remote/tests/checkpoint/publication_catalog.rs @@ -137,13 +137,17 @@ fn dependency_limits() -> crab_read::dependency_proof::DependencyProofLimits { } #[tokio::test(flavor = "multi_thread")] -async fn fixed_pointer_publication_keeps_unrelated_catalog_history_cold() { +async fn fixed_pointer_publication_and_hydration_keep_unrelated_history_cold() { for count in [100, 10_000] { for checkpointed in [false, true] { for supplied in [false, true] { let (layout, observations) = empty_fixture().await; let directory = tempfile::tempdir().unwrap(); let (content, pointer, file_hash) = native_content(&layout, directory.path()).await; + let expected_shard = MerkleHash::from_hex( + content.pointer_catalog().files()[&file_hash].shard_hash(), + ) + .unwrap(); let mut catalog = PointerCatalog::new(); for index in 0..count { let shard = format!("{:064x}", index + 1_000_000); @@ -229,6 +233,9 @@ async fn fixed_pointer_publication_keeps_unrelated_catalog_history_cold() { ).await?; assert_eq!(prepared.plan().pointers().len(), 1); if supplied { prepared = prepared.attach_content(content)?; } + let captured_lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( + layout.clone(), view.pointer_catalog_lookup(&layout, 256 * 1024)?, + ); observations.0.lock().unwrap().clear(); let artifacts = prepared.upload_capsule(&view, dependency_limits(), &cancel).await?; let requests = observations.0.lock().unwrap().clone(); @@ -239,7 +246,24 @@ async fn fixed_pointer_publication_keeps_unrelated_catalog_history_cold() { assert!(matches!(outcome, crab_remote::prepare::CapsuleCommitOutcome::Committed { .. })); let after = open_history_view(&layout).await; assert_eq!(after.refs()[&target], oid); - assert!(after.pointer_catalog_lookup(&layout, LIMIT)?.files(layout.store(), std::slice::from_ref(&file_hash)).await?.files().contains_key(&file_hash)); + let file = MerkleHash::from_hex(&file_hash).unwrap(); + let captured_result = captured_lookup.lookup(&file).await; + captured_lookup.close().await?; + assert_eq!(captured_result?, if supplied { None } else { Some(expected_shard) }, "captured hydration must not follow a later publication"); + observations.0.lock().unwrap().clear(); + let lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( + layout.clone(), after.pointer_catalog_lookup(&layout, 256 * 1024)?, + ); + assert!(observations.0.lock().unwrap().is_empty(), "hydration lookup construction read catalog bodies"); + let file = MerkleHash::from_hex(&file_hash).unwrap(); + let missing = MerkleHash::from_hex(&"f".repeat(64)).unwrap(); + let result = lookup.lookup_batch(&[file, missing, file]).await; + lookup.close().await?; + assert_eq!(result?, vec![Some(expected_shard), None, Some(expected_shard)]); + let requests = observations.0.lock().unwrap().clone(); + assert!(requests.len() <= 24 && requests.iter().map(|r| r.bytes_read).sum::() <= 256 * 1024, + "fixed hydration lookup exceeds bounds: count={count}, checkpoint={checkpointed}, supplied={supplied}"); + assert!(requests.iter().all(|r| r.operation == StorageOperation::Range), "captured hydration must not read mutable state or write checkpoints"); drop(prepared); Ok::<_, Box>(()) }, From 92915cc037b27833c5bde9738f3e49e297062ba4 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 00:12:29 -0700 Subject: [PATCH 058/127] fix(test): await capsule visibility proof --- crab/src/git/capsule_push.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index dbcad109e..5d6c9e128 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -1552,7 +1552,7 @@ mod tests { "{name}: remote tip" ); if succeeds { - let visibility = view.git_visibility_index().unwrap(); + let visibility = view.git_visibility_index().await.unwrap(); let refs = visibility.ref_closures(); let closure = &refs["refs/heads/main"]; assert!( From 2a90f8200c3226e77e8500b3a600906fb20f6de6 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 22:23:22 -0700 Subject: [PATCH 059/127] refactor(metadata): share ref capture authority --- crates/crab-metadata/README.md | 5 + .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/store.rs | 22 ++- .../src/capsule_protocol/store/tests.rs | 37 +++++ crates/crab-read/src/capsule_protocol.rs | 145 +----------------- 5 files changed, 66 insertions(+), 147 deletions(-) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 7678cf595..522ee677c 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -46,6 +46,11 @@ Catalog readers resolve only ref heads in that root's authority epoch, before loading multi-ref activation records. Restore can leave retired heads in storage; their checkpoint positions and dependencies must not enter the restored catalog. Current-epoch activation and checkpoint-chain errors still fail closed. +`capture_ref_heads` and `resolve_referenced_activations` own that stable capture +for metadata and read orchestration. Transactionless heads must match the +compacted root; selected-ref capture reuses the same activation validator while +keeping its targeted ETag probes. This extraction does not yet remove the eager +current-state file-lookup catalog loader. Run controls retain each capsule's own pack directory in capsule-relative coordinates. The run directory keeps physical transport coordinates separately. Logical capsule equality compares authenticated visibility section identities diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index a93cdb337..c435b426a 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -64,10 +64,10 @@ pub use run::{ pub use run::{CapsuleRunAdmissionReader, RunCatalogSource}; #[cfg(feature = "storage")] pub use store::{ - RootSnapshot, create_root, load_capsule_run, load_capsule_run_control, + RootSnapshot, capture_ref_heads, create_root, load_capsule_run, load_capsule_run_control, load_capsule_run_git_control, load_history_chain, load_history_segment, load_layered_checkpoint, load_layered_checkpoint_control, load_pointer_catalog, - load_pointer_catalog_from_root, load_root, + load_pointer_catalog_from_root, load_root, resolve_referenced_activations, }; pub use transaction::{CapsuleRefEdit, CapsuleTransaction}; pub use transaction_record::{ diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index c5c35d069..d3ed7d433 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -674,7 +674,11 @@ async fn load_visible_ref_capsules( Ok(ordered) } -async fn capture_ref_heads( +/// Capture one stable, epoch-filtered ref-head view and its committed activations. +/// +/// Rejects mismatched object identities, activation records and uncommitted base +/// heads. Concurrent changes retry within the existing bounded capture policy. +pub async fn capture_ref_heads( router: &StoreLayout, root: &super::RepositoryRoot, ) -> Result<(Vec, BTreeSet)> { @@ -717,6 +721,16 @@ async fn capture_ref_heads( if before != after { continue; } + for head in &heads { + let visible = head.visible(&active); + let base_oid = root.refs().get(head.ref_name()).map(String::as_str); + if visible.transaction_id().is_none() && visible.oid() != base_oid { + return Err(corrupt( + &router.capsule_ref_head_path(&capsule_ref_name_key(head.ref_name())), + "capsule ref head without a transaction differs from the compacted root", + )); + } + } return Ok((heads, active)); } Err(contract_error( @@ -745,7 +759,11 @@ fn listed_version_matches(object: &ObjectMeta, etag: &ETag) -> bool { .is_none_or(|listed| etag.version.as_ref() == Some(listed)) } -async fn resolve_referenced_activations( +/// Validate referenced activation records and select those committed for these heads. +/// +/// Callers filter retired epochs before resolving activations so stale heads +/// cannot introduce authority or demand retired dependencies. +pub async fn resolve_referenced_activations( router: &StoreLayout, heads: &[CapsuleRefHead], ) -> Result> { diff --git a/crates/crab-metadata/src/capsule_protocol/store/tests.rs b/crates/crab-metadata/src/capsule_protocol/store/tests.rs index 70560bb41..e407e0061 100644 --- a/crates/crab-metadata/src/capsule_protocol/store/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/store/tests.rs @@ -534,3 +534,40 @@ async fn visibility_lookup_keeps_oid_pages_cold_until_requested() { .is_err() ); } + +#[tokio::test] +async fn ref_capture_rejects_uncommitted_base_changes_only_in_the_active_epoch() { + for retired in [false, true] { + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "repositories/ref-capture".to_owned(), + ); + let root = RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(); + let name = "refs/tags/uncommitted"; + let head = CapsuleRefHead::from_root( + name, + if retired { + "2".repeat(64) + } else { + root.ref_epoch().to_owned() + }, + Some("3".repeat(40)), + None, + ) + .unwrap(); + layout + .store() + .create_strict( + &layout.capsule_ref_head_path(&capsule_ref_name_key(name)), + head.encode().unwrap(), + ) + .await + .unwrap(); + let result = capture_ref_heads(&layout, &root).await; + if retired { + assert!(result.unwrap().0.is_empty()); + } else { + assert!(matches!(result, Err(MetadataError::CorruptObject { .. }))); + } + } +} diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index f52bb40d4..8fb1df7cb 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -4,13 +4,13 @@ use bytes::Bytes; use crab_metadata::capsule_protocol::{ Capsule, CapsuleControl, CapsulePointer, CapsuleRun, CapsuleRunControl, CheckpointPointer, LayeredCheckpoint, PackMemberDescriptor, PackRange, PackSourceDescriptor, PackSourceKind, - PointerCatalog, RootRecord, load_root, visibility_object_set_digest, + PointerCatalog, RootRecord, capture_ref_heads, load_root, resolve_referenced_activations, + visibility_object_set_digest, }; use crab_storage::{Store, StoreLayout}; use futures_util::future::try_join_all; use futures_util::{StreamExt, TryStreamExt}; use gix_hash::ObjectId; -use object_store::ObjectMeta; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -5231,73 +5231,6 @@ fn materialize_visible_ref_heads( }) } -async fn load_ref_heads( - router: &StoreLayout, - objects: &[ObjectMeta], -) -> Result>> { - let loaded = futures_util::stream::iter(objects.iter().cloned().map(|object| async move { - let (body, etag) = router.store().get_with_etag(&object.location).await?; - let head = crab_metadata::capsule_protocol::CapsuleRefHead::decode(&body)?; - let expected = router.capsule_ref_head_path( - &crab_metadata::capsule_protocol::capsule_ref_name_key(head.ref_name()), - ); - if expected != object.location { - return Err(corrupt( - &object.location, - "capsule ref-head key does not match its ref name", - )); - } - Ok::<_, ReadError>((head, listed_version_matches(&object, &etag))) - })) - .buffer_unordered(32) - .try_collect::>() - .await?; - if loaded.iter().any(|(_, matched)| !matched) { - return Ok(None); - } - let mut heads = loaded.into_iter().map(|(head, _)| head).collect::>(); - heads.sort_unstable_by(|left, right| left.ref_name().cmp(right.ref_name())); - Ok(Some(heads)) -} - -async fn capture_ref_heads( - router: &StoreLayout, - root: &crab_metadata::capsule_protocol::RepositoryRoot, -) -> Result<( - Vec, - BTreeSet, -)> { - for _ in 0..8 { - let before = list_ref_head_objects(router).await?; - let Some(heads) = load_ref_heads(router, &before).await? else { - continue; - }; - let heads = heads - .into_iter() - .filter(|head| head.ref_epoch() == root.ref_epoch()) - .collect::>(); - let active = resolve_referenced_activations(router, &heads).await?; - let after = list_ref_head_objects(router).await?; - if before != after { - continue; - } - for head in &heads { - let visible = head.visible(&active); - let base_oid = root.refs().get(head.ref_name()).map(String::as_str); - if visible.transaction_id().is_none() && visible.oid() != base_oid { - return Err(corrupt_path( - "capsule-protocol ref heads", - "capsule ref head without a transaction differs from the compacted root", - )); - } - } - return Ok((heads, active)); - } - Err(ReadError::internal( - "capsule ref snapshot changed during every bounded capture attempt", - )) -} - async fn capture_selected_ref_heads( router: &StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, @@ -5370,80 +5303,6 @@ async fn load_selected_ref_heads( .await } -async fn list_ref_head_objects(router: &StoreLayout) -> Result> { - let mut objects = router - .store() - .list_prefix_bounded( - &router.capsule_ref_heads_prefix(), - crab_metadata::capsule_protocol::MAX_CAPSULE_REF_HEADS, - ) - .await? - .ok_or_else(|| ReadError::internal("capsule ref-head limit exceeded"))?; - objects.sort_unstable_by(|left, right| left.location.cmp(&right.location)); - Ok(objects) -} - -fn listed_version_matches(object: &ObjectMeta, etag: &crab_storage::ETag) -> bool { - object - .e_tag - .as_ref() - .is_none_or(|listed| etag.e_tag.as_ref() == Some(listed)) - && object - .version - .as_ref() - .is_none_or(|listed| etag.version.as_ref() == Some(listed)) -} - -async fn resolve_referenced_activations( - router: &StoreLayout, - heads: &[crab_metadata::capsule_protocol::CapsuleRefHead], -) -> Result> { - let referenced = heads - .iter() - .filter_map(|head| head.prepared_activation_id()) - .map(str::to_owned) - .collect::>(); - futures_util::stream::iter(referenced.into_iter().map(|activation_id| async move { - let path = router.capsule_transaction_path(&activation_id); - let (body, _) = router - .store() - .get_with_etag_bounded( - &path, - crab_metadata::capsule_protocol::MAX_CAPSULE_TRANSACTION_RECORD_BYTES, - ) - .await?; - let record = crab_metadata::capsule_protocol::CapsuleTransactionRecord::decode(&body)?; - if record.activation_id() != activation_id { - return Err(corrupt( - &path, - "capsule transaction record does not match its activation key", - )); - } - if heads.iter().any(|head| { - head.prepared_activation_id() == Some(activation_id.as_str()) - && head - .visible(&BTreeSet::from([activation_id.clone()])) - .transaction_id() - != Some(record.transaction_id()) - }) { - return Err(corrupt( - &path, - "capsule transaction record does not match its prepared ref heads", - )); - } - Ok::<_, ReadError>(( - activation_id, - record.status() == crab_metadata::capsule_protocol::CapsuleTransactionStatus::Committed, - )) - })) - .buffer_unordered(32) - .try_filter_map( - |(activation_id, committed)| async move { Ok(committed.then_some(activation_id)) }, - ) - .try_collect() - .await -} - fn order_ref_capsules( base_refs: &BTreeMap, base_peeled: &BTreeMap, From 5b31b1897c126634ccdcf2f4eb5ff9024d9ea11f Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 23:15:23 -0700 Subject: [PATCH 060/127] perf(metadata): page current catalog lookups Share captured ref ordering and visibility with Git readers. Validate full catalog dependencies after assembling the visible cross-ref set. --- crates/crab-metadata/README.md | 34 +- .../capsule_protocol/captured_visibility.rs} | 502 ++++++----------- .../captured_visibility/tests.rs | 195 +++++++ .../src/capsule_protocol/catalog_capture.rs | 175 ++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 21 +- .../src/capsule_protocol/pointer.rs | 44 +- .../src/capsule_protocol/pointer/paged.rs | 2 +- .../src/capsule_protocol/ref_view.rs | 330 +++++++++++ .../src/capsule_protocol/store.rs | 195 +------ .../src/capsule_protocol/store/tests.rs | 138 ++++- crates/crab-metadata/src/file_index_lookup.rs | 47 +- .../src/file_index_lookup/catalog_tests.rs | 162 +++++- crates/crab-read/README.md | 6 + crates/crab-read/src/capsule_protocol.rs | 511 ++---------------- .../src/capsule_protocol/visibility_tests.rs | 106 ++++ 15 files changed, 1416 insertions(+), 1052 deletions(-) rename crates/{crab-read/src/capsule_protocol/visibility.rs => crab-metadata/src/capsule_protocol/captured_visibility.rs} (55%) create mode 100644 crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/catalog_capture.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/ref_view.rs create mode 100644 crates/crab-read/src/capsule_protocol/visibility_tests.rs diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 522ee677c..79984ed78 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -49,8 +49,9 @@ Current-epoch activation and checkpoint-chain errors still fail closed. `capture_ref_heads` and `resolve_referenced_activations` own that stable capture for metadata and read orchestration. Transactionless heads must match the compacted root; selected-ref capture reuses the same activation validator while -keeping its targeted ETag probes. This extraction does not yet remove the eager -current-state file-lookup catalog loader. +keeping its targeted ETag probes. Current-state catalog lookups reuse that owner together with the shared +control-selection and visibility machinery; full maintenance materialization +remains explicit. Run controls retain each capsule's own pack directory in capsule-relative coordinates. The run directory keeps physical transport coordinates separately. Logical capsule equality compares authenticated visibility section identities @@ -71,15 +72,18 @@ accepts captured lookup roots and resolves requested files with their authentica shard/xorb closure. Point and batch calls share the page cache and retain the session read guard through I/O, so close waits for active reads. Missing catalog pages remain errors rather than reopening the mutable file-index database. -The current-state opener still materializes a complete v2 catalog; callers with -an authenticated view should pass its captured roots. All 19 audited local release tags omit these +The current-state opener also captures authenticated catalog roots and leaves +catalog bodies cold. Explicitly pinned callers pass their existing capture, while +current-state sessions bind once to the first root they open. Catalog and Git +views share stable ref capture, ref-frontier selection, visibility reconciliation +and transaction ordering; neither path invents a separate precedence policy. All 19 audited local release tags omit these capsule checkpoint formats; unshipped format 6 is rejected without a fallback. -Captured run-v4 roots provide the same selected lookup over exact logical -capsule occurrences; catalog bodies remain cold until their keys are selected. +Checkpoint and selected run catalog roots share one lookup budget and page cache. CLI and shared capsule-publication preparation use these captured queries. Protected receive selects from separately authorized source/filtered captures. -Captured revision and remote-mount hydration pass the same paged lookup to the -shared hydrator; current-state hydration still requires capture integration. +Captured revision, remote-mount and current-state hydration use the shared paged +lookup. Control capture and selected closure queries are measured separately; +complete opening and history-amplification qualification remain required. External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and @@ -88,12 +92,16 @@ ordering, including the previous lowest-xorb/first-occurrence placement rule. Complete inspection verifies the body hash, complete dependency closure and canonical page layout. Delta inspection defers base-owned closure to `apply`, which retains later file bindings at equal size and rejects immutable shard/xorb -conflicts. Encoded catalogs are limited to 2 GiB and descriptors to 4 KiB. +conflicts. +Full inspection of a captured visible set uses `PointerCatalogAssembly`: binding +order need not be dependency order across independent refs. Rows and immutable +conflicts are checked while collecting deltas, then the complete closure is +checked before returning the catalog. Publication-time `PointerCatalog::apply` +still requires each delta's dependencies to exist in its base. Encoded catalogs are limited to 2 GiB and descriptors to 4 KiB. The retired monolithic JSON encoding was absent from all 19 audited release tags -and is rejected. Foreground consumers must use captured selected lookups rather -than infer bounded work from the page encoding alone. Maintenance retains explicit full -inspection; shared/protected hydration and lifecycle qualification remain -separate gates. +and is rejected. Shared file lookup uses paged capture for pinned and current-state sessions. +Complete catalog inspection remains explicit; consumer, historical amplification +and lifecycle qualification must not be inferred from the page representation. `v2/browse-indexes` is a bounded, mutable derived record, not ref authority. Its version 3 record binds a durable immutable commit graph, optional complete diff --git a/crates/crab-read/src/capsule_protocol/visibility.rs b/crates/crab-metadata/src/capsule_protocol/captured_visibility.rs similarity index 55% rename from crates/crab-read/src/capsule_protocol/visibility.rs rename to crates/crab-metadata/src/capsule_protocol/captured_visibility.rs index 4682e2fa2..60dd7d5cb 100644 --- a/crates/crab-read/src/capsule_protocol/visibility.rs +++ b/crates/crab-metadata/src/capsule_protocol/captured_visibility.rs @@ -1,30 +1,46 @@ //! Captured visibility closure layers; physical presence never grants membership. -use super::*; -use crab_metadata::capsule_protocol::{ - CapsuleVisibilityReader, LayeredVisibilityReader, VisibilityChange, VisibilityEditHeader, +use crate::capsule_protocol::{ + CapsuleControl, CapsuleTransaction, CapsuleVisibilityDelta, CapsuleVisibilityReader, + LayeredVisibilityReader, VisibilityChange, VisibilityEditHeader, }; -use crab_metadata::git_visibility::{GitVisibilityIndex, MAX_GIT_VISIBILITY_OBJECTS}; +use crate::error::{MetadataError, Result}; +use crate::git_visibility::{GitVisibilityIndex, MAX_GIT_VISIBILITY_OBJECTS, decode_oid}; +use crab_storage::Store; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; -pub(super) enum Reconciliation { +/// Reconcile captured controls against complete or paged visibility evidence. +pub enum VisibilityReconciliation { Complete(GitVisibilityIndex), - Paged(Box), + Paged(Box), } -impl Reconciliation { - pub(super) async fn ready( +impl VisibilityReconciliation { + /// Check expected ref values and captured prior-closure availability. + pub async fn ready( &self, control: &CapsuleControl, refs: &BTreeMap, ) -> Result { match self { - Self::Complete(index) => control_is_ready(control, refs, index), + Self::Complete(index) => visibility_is_ready( + control.transaction(), + control.visibility_delta()?, + refs, + index, + ), Self::Paged(index) => index.ready(control, refs).await, } } - pub(super) async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { + /// Apply one authenticated control without widening to mutable ref state. + pub async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { match self { Self::Complete(index) if control.has_visibility() => { - apply_capsule_control_visibility_index(control, index) + apply_transaction_visibility_index( + control.transaction(), + control.visibility_delta()?, + index, + ) } Self::Complete(_) => Ok(()), Self::Paged(index) => index.apply(control).await, @@ -77,7 +93,8 @@ impl Initial { } } -pub(super) struct Visibility { +/// Retain immutable prior ref closures while applying authenticated visibility edits. +pub struct CapturedVisibility { store: Store, initial: Initial, layers: Vec, @@ -87,16 +104,10 @@ pub(super) struct Visibility { dictionary_count: u64, } -fn oid(value: &str) -> Result<[u8; 20]> { - ObjectId::from_hex(value.as_bytes()) - .map_err(|_| corrupt_path("capsule Git visibility", "invalid object identity"))? - .as_bytes() - .try_into() - .map_err(|_| corrupt_path("capsule Git visibility", "object format is unsupported")) -} - -impl Visibility { - pub(super) fn new( +impl CapturedVisibility { + /// Seed captured closures from complete authenticated visibility evidence. + #[must_use] + pub fn new( store: Store, initial: GitVisibilityIndex, names: impl Iterator, @@ -107,7 +118,8 @@ impl Visibility { Self::seed(store, Initial::Complete(initial), names) } - pub(super) async fn from_checkpoint( + /// Seed captured closures from the checkpoint's authenticated visibility pages. + pub async fn from_checkpoint( store: Store, reader: Arc, names: impl Iterator, @@ -285,7 +297,8 @@ impl Visibility { Ok(Some((reader.clone(), headers))) } - pub(super) async fn ready( + /// Check expected ref values and captured prior-closure availability. + pub async fn ready( &self, control: &CapsuleControl, refs: &BTreeMap, @@ -311,7 +324,7 @@ impl Visibility { if edit.expected_old().is_none() && !header.replaces() && let Some(old) = header.old_oid() - && self.first_ref(oid(old)?).await?.is_none() + && self.first_ref(decode_oid(old)?).await?.is_none() { return Ok(false); } @@ -319,7 +332,8 @@ impl Visibility { Ok(true) } - pub(super) async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { + /// Apply one authenticated control without widening to mutable ref state. + pub async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { let Some((reader, headers)) = self.headers(control).await? else { return Ok(()); }; @@ -333,17 +347,17 @@ impl Visibility { && !header.replaces() && let Some(old) = header.old_oid() { - parent = self.first_ref(oid(old)?).await?; + parent = self.first_ref(decode_oid(old)?).await?; } if let Some(old) = header.old_oid() - && !self.contains(parent, &[oid(old)?]).await?[0] + && !self.contains(parent, &[decode_oid(old)?]).await?[0] { return Err(corrupt_path( "capsule Git visibility", "visibility delta prior closure does not contain its old ref tip", )); } - let new = oid(header.new_oid())?; + let new = decode_oid(header.new_oid())?; let change = reader .lookup(&self.store, edit.ref_name(), &[new]) .await? @@ -420,315 +434,145 @@ impl Visibility { } } -#[cfg(test)] -mod tests { - use super::*; - use crab_metadata::capsule_protocol::{ - CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, - CapsuleVisibilityDelta, - }; - use crab_metadata::git_visibility::GitVisibilityEdit; - use object_store::memory::InMemory; - - fn hex(value: u8) -> String { - format!("{value:040x}") - } - fn object(value: u8) -> [u8; 20] { - let mut oid = [0; 20]; - oid[19] = value; - oid +/// Validate publication evidence and check its prior ref dependencies. +pub fn visibility_is_ready( + transaction: &CapsuleTransaction, + delta: Option<&CapsuleVisibilityDelta>, + refs: &BTreeMap, + visibility_index: &crate::git_visibility::GitVisibilityIndex, +) -> Result { + if transaction + .edits() + .iter() + .any(|edit| refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old()) + { + return Ok(false); } - async fn control( - layout: &StoreLayout, - name: &str, - expected: Option, - edit: GitVisibilityEdit, - ) -> CapsuleControl { - let transaction = CapsuleTransaction::new( - &"1".repeat(64), - vec![CapsuleRefEdit::new( - name, - expected.map(hex), - Some(edit.new_oid.clone()), - None, - )], - ) - .unwrap(); - let delta = CapsuleVisibilityDelta::new(BTreeMap::from([(name.to_owned(), edit)])).unwrap(); - let capsule = Capsule::build( - &transaction, - Vec::new(), - vec![CapsuleSection::new( - CapsuleSectionKind::VisibilityDelta, - delta.encode().unwrap(), - )], - ) - .unwrap(); - let run = CapsuleRun::leaf(capsule).unwrap(); - layout - .store() - .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) - .await - .unwrap(); - let pointer = CapsulePointer::new( - run.hash(), - run.bytes().len() as u64, - run.control_offset(), - run.control_size(), - run.footer_hash(), - run.level(), - run.transaction_ids(), - run.newest_base_root_digest(), - ) - .unwrap(); - let (_, mut controls) = - crab_metadata::capsule_protocol::load_capsule_run_git_control(layout, &pointer) - .await - .unwrap(); - controls.remove(0) + let Some(delta) = delta else { + return Ok(true); + }; + // Transactions have unique refs. Count matched rows without copying OID lists; + // unmatched evidence still fails before it can authorize a publication. + let evidence = delta.edits(); + let mut matched = 0; + for edit in transaction.edits() { + let Some(new_oid) = edit.new_oid() else { + if evidence.get(edit.ref_name()).is_some() { + return Err(corrupt_path( + "capsule Git visibility", + "deleted ref has visibility evidence", + )); + } + continue; + }; + let Some(visibility) = evidence.get(edit.ref_name()) else { + return Err(corrupt_path( + "capsule Git visibility", + "live ref edit has no visibility evidence", + )); + }; + matched += 1; + visibility.validate()?; + if visibility.new_oid != new_oid { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence does not match its ref edit", + )); + } + match edit.expected_old() { + Some(expected_old) => { + if visibility.old_oid.as_deref() != Some(expected_old) { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence does not match the expected old ref", + )); + } + if !visibility_index.contains_ref(edit.ref_name()) { + return Ok(false); + } + } + None if visibility.replaces => {} + None => { + let Some(old_oid) = visibility.old_oid.as_deref() else { + continue; + }; + if !visibility_index.contains_hex_in_any_ref(old_oid) { + return Ok(false); + } + } + } } - fn initial() -> GitVisibilityIndex { - GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new()).unwrap() + if evidence.len() != matched { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence contains an uncommitted ref", + )); } + Ok(true) +} - #[tokio::test] - async fn initial_replacement_opens_without_materializing_its_oid_set() { - use crab_metadata::capsule_protocol::{RepositoryRoot, RootRecord}; - use std::sync::atomic::{AtomicU64, Ordering}; - let bytes = Arc::new(AtomicU64::new(0)); - let observed = bytes.clone(); - let store = - Store::new(Arc::new(InMemory::new())).with_read_byte_observer(Arc::new(move |size| { - observed.fetch_add(size, Ordering::Relaxed); - })); - let layout = StoreLayout::new(store, "repositories/large-root".into()); - let initial = RootRecord::encode( - RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(), - ) - .unwrap(); - crab_metadata::capsule_protocol::create_root(&layout, initial.clone()) - .await - .unwrap(); - let objects = (0u32..10000) - .map(|index| format!("{index:040x}")) - .collect::>(); - let tip = objects[9000].clone(); - let transaction = CapsuleTransaction::new( - initial.digest(), - vec![CapsuleRefEdit::new( - "refs/heads/main", - None, - Some(tip.clone()), - None, - )], - ) - .unwrap(); - let delta = CapsuleVisibilityDelta::new(BTreeMap::from([( - "refs/heads/main".into(), - GitVisibilityEdit::from_replacement_objects(None, tip.clone(), objects), - )])) - .unwrap(); - let capsule = Capsule::build( - &transaction, - Vec::new(), - vec![CapsuleSection::new( - CapsuleSectionKind::VisibilityDelta, - delta.encode().unwrap(), - )], - ) - .unwrap(); - let run = CapsuleRun::leaf(capsule).unwrap(); - layout - .store() - .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) - .await - .unwrap(); - let pointer = CapsulePointer::new( - run.hash(), - run.bytes().len() as u64, - run.control_offset(), - run.control_size(), - run.footer_hash(), - run.level(), - run.transaction_ids(), - run.newest_base_root_digest(), - ) - .unwrap(); - let next = RootRecord::encode( - initial - .root() - .advance( - initial.digest(), - BTreeMap::from([("refs/heads/main".into(), tip)]), - BTreeMap::new(), - vec![pointer], - transaction.id().unwrap().as_str(), +/// Apply transaction-bound visibility evidence to a complete captured index. +pub fn apply_transaction_visibility_index( + transaction: &CapsuleTransaction, + delta: Option<&CapsuleVisibilityDelta>, + index: &mut crate::git_visibility::GitVisibilityIndex, +) -> Result<()> { + let evidence = delta.map(|delta| delta.edits()); + let mut matched = 0; + for edit in transaction.edits() { + let Some(new_oid) = edit.new_oid() else { + if evidence + .and_then(|rows| rows.get(edit.ref_name())) + .is_some() + { + return Err(corrupt_path( + "capsule Git visibility", + "deleted ref has visibility evidence", + )); + } + index.remove_ref(edit.ref_name()); + continue; + }; + let visibility = evidence + .and_then(|rows| rows.get(edit.ref_name())) + .ok_or_else(|| { + corrupt_path( + "capsule Git visibility", + "live ref edit has no visibility evidence", ) - .unwrap(), - ) - .unwrap(); - layout - .store() - .put_overwrite(&layout.capsule_root_path(), next.bytes().clone()) - .await - .unwrap(); - let root = crab_metadata::capsule_protocol::load_root(&layout) - .await - .unwrap(); - bytes.store(0, Ordering::Relaxed); - let view = open_git_view_from_root( - &layout, - root, - CapsuleReadLimits { - max_capsule_bytes: 1024 * 1024, - max_frontier_bytes: 1024 * 1024, - }, - ) - .await - .unwrap(); - assert!( - bytes.load(Ordering::Relaxed) < 64 * 1024 - && view.capsule_controls[0].visibility_delta().is_err() - ); - } - - #[tokio::test] - async fn layered_closures_match_complete_edits_and_preserve_captured_bases() { - let layout = StoreLayout::new( - Store::new(Arc::new(InMemory::new())), - "repositories/layers".into(), - ); - let mut full = initial(); - let mut lazy = Visibility::new(layout.store().clone(), full.clone(), std::iter::empty()); - let edits = [ - ( - "refs/heads/main", - None, - GitVisibilityEdit::from_replacement_objects( - None, - hex(1), - vec![hex(1), hex(2), hex(3)], - ), - ), - ( - "refs/heads/feature", - None, - GitVisibilityEdit::from_delta_objects( - Some(hex(1)), - hex(2), - vec![hex(4)], - vec![hex(3)], - ), - ), - ( - "refs/heads/main", - Some(1), - GitVisibilityEdit::from_delta_objects( - Some(hex(1)), - hex(5), - vec![hex(5)], - vec![hex(3)], - ), - ), - ]; - for (name, expected, edit) in edits { - let control = control(&layout, name, expected, edit.clone()).await; - full.apply_ref_edit(name.to_owned(), &edit).unwrap(); - lazy.apply(&control).await.unwrap(); + })?; + matched += 1; + if visibility.new_oid != new_oid { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence does not match its ref edit", + )); } - let requested = (1..7).map(object).collect::>(); - let mut actual = BTreeMap::new(); - for (name, index) in &lazy.refs { - actual.insert( - name.clone(), - lazy.contains(Some(*index), &requested).await.unwrap(), - ); + if let Some(expected_old) = edit.expected_old() + && visibility.old_oid.as_deref() != Some(expected_old) + { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence does not match the expected old ref", + )); } - let expected = lazy - .refs - .keys() - .map(|name| { - ( - name.clone(), - requested - .iter() - .map(|oid| full.contains_in_ref(name, oid)) - .collect::>(), - ) - }) - .collect::>(); - assert_eq!( - (actual, lazy.dictionary_count), - (expected, full.objects().len() as u64) - ); + index.apply_ref_edit(edit.ref_name().to_owned(), visibility)?; } - - #[tokio::test] - async fn removed_invisible_objects_do_not_supply_new_ref_bases() { - let layout = StoreLayout::new( - Store::new(Arc::new(InMemory::new())), - "repositories/invisible".into(), - ); - let mut lazy = Visibility::new(layout.store().clone(), initial(), std::iter::empty()); - let create = control( - &layout, - "refs/heads/main", - None, - GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1), hex(3)]), - ) - .await; - lazy.apply(&create).await.unwrap(); - let remove = control( - &layout, - "refs/heads/main", - Some(1), - GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(2)], vec![hex(3)]), - ) - .await; - lazy.apply(&remove).await.unwrap(); - let branch = control( - &layout, - "refs/heads/feature", - None, - GitVisibilityEdit::from_delta_objects(Some(hex(3)), hex(4), vec![hex(4)], vec![]), - ) - .await; - assert_eq!( - ( - lazy.dictionary_contains(&[object(3)]).await.unwrap(), - lazy.ready( - &branch, - &BTreeMap::from([("refs/heads/main".into(), hex(2))]) - ) - .await - .unwrap() - ), - (vec![true], false) - ); + if evidence.is_some_and(|rows| rows.len() != matched) { + return Err(corrupt_path( + "capsule Git visibility", + "visibility evidence contains an uncommitted ref", + )); } + Ok(()) +} - #[tokio::test] - async fn removal_outside_captured_prior_closure_is_rejected() { - let layout = StoreLayout::new( - Store::new(Arc::new(InMemory::new())), - "repositories/bad-removal".into(), - ); - let mut lazy = Visibility::new(layout.store().clone(), initial(), std::iter::empty()); - let create = control( - &layout, - "refs/heads/main", - None, - GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1)]), - ) - .await; - lazy.apply(&create).await.unwrap(); - let bad = control( - &layout, - "refs/heads/main", - Some(1), - GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(2)], vec![hex(9)]), - ) - .await; - assert!(lazy.apply(&bad).await.is_err()); +fn corrupt_path(path: impl Into, reason: impl Into) -> MetadataError { + MetadataError::CorruptObject { + path: path.into(), + reason: reason.into(), } } + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs b/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs new file mode 100644 index 000000000..e7cd4593b --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs @@ -0,0 +1,195 @@ +use super::*; +use crate::capsule_protocol::{ + Capsule, CapsulePointer, CapsuleRefEdit, CapsuleRun, CapsuleSection, CapsuleSectionKind, + CapsuleTransaction, CapsuleVisibilityDelta, +}; +use crate::git_visibility::GitVisibilityEdit; +use crab_storage::StoreLayout; +use object_store::memory::InMemory; + +fn hex(value: u8) -> String { + format!("{value:040x}") +} +fn object(value: u8) -> [u8; 20] { + let mut oid = [0; 20]; + oid[19] = value; + oid +} +async fn control( + layout: &StoreLayout, + name: &str, + expected: Option, + edit: GitVisibilityEdit, +) -> CapsuleControl { + let transaction = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + name, + expected.map(hex), + Some(edit.new_oid.clone()), + None, + )], + ) + .unwrap(); + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([(name.to_owned(), edit)])).unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let (_, mut controls) = crate::capsule_protocol::load_capsule_run_git_control(layout, &pointer) + .await + .unwrap(); + controls.remove(0) +} +fn initial() -> GitVisibilityIndex { + GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new()).unwrap() +} + +#[tokio::test] +async fn layered_closures_match_complete_edits_and_preserve_captured_bases() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/layers".into(), + ); + let mut full = initial(); + let mut lazy = + CapturedVisibility::new(layout.store().clone(), full.clone(), std::iter::empty()); + let edits = [ + ( + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1), hex(2), hex(3)]), + ), + ( + "refs/heads/feature", + None, + GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(4)], vec![hex(3)]), + ), + ( + "refs/heads/main", + Some(1), + GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(5), vec![hex(5)], vec![hex(3)]), + ), + ]; + for (name, expected, edit) in edits { + let control = control(&layout, name, expected, edit.clone()).await; + full.apply_ref_edit(name.to_owned(), &edit).unwrap(); + lazy.apply(&control).await.unwrap(); + } + let requested = (1..7).map(object).collect::>(); + let mut actual = BTreeMap::new(); + for (name, index) in &lazy.refs { + actual.insert( + name.clone(), + lazy.contains(Some(*index), &requested).await.unwrap(), + ); + } + let expected = lazy + .refs + .keys() + .map(|name| { + ( + name.clone(), + requested + .iter() + .map(|oid| full.contains_in_ref(name, oid)) + .collect::>(), + ) + }) + .collect::>(); + assert_eq!( + (actual, lazy.dictionary_count), + (expected, full.objects().len() as u64) + ); +} + +#[tokio::test] +async fn removed_invisible_objects_do_not_supply_new_ref_bases() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/invisible".into(), + ); + let mut lazy = CapturedVisibility::new(layout.store().clone(), initial(), std::iter::empty()); + let create = control( + &layout, + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1), hex(3)]), + ) + .await; + lazy.apply(&create).await.unwrap(); + let remove = control( + &layout, + "refs/heads/main", + Some(1), + GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(2)], vec![hex(3)]), + ) + .await; + lazy.apply(&remove).await.unwrap(); + let branch = control( + &layout, + "refs/heads/feature", + None, + GitVisibilityEdit::from_delta_objects(Some(hex(3)), hex(4), vec![hex(4)], vec![]), + ) + .await; + assert_eq!( + ( + lazy.dictionary_contains(&[object(3)]).await.unwrap(), + lazy.ready( + &branch, + &BTreeMap::from([("refs/heads/main".into(), hex(2))]) + ) + .await + .unwrap() + ), + (vec![true], false) + ); +} + +#[tokio::test] +async fn removal_outside_captured_prior_closure_is_rejected() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/bad-removal".into(), + ); + let mut lazy = CapturedVisibility::new(layout.store().clone(), initial(), std::iter::empty()); + let create = control( + &layout, + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1)]), + ) + .await; + lazy.apply(&create).await.unwrap(); + let bad = control( + &layout, + "refs/heads/main", + Some(1), + GitVisibilityEdit::from_delta_objects(Some(hex(1)), hex(2), vec![hex(2)], vec![hex(9)]), + ) + .await; + assert!(lazy.apply(&bad).await.is_err()); +} diff --git a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs new file mode 100644 index 000000000..78c162da2 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs @@ -0,0 +1,175 @@ +//! One shared authenticated control capture for current-state catalog queries. +use super::pointer::paged::MAX_CATALOG_BYTES; +use super::{ + CapsuleControl, CapturedVisibility, LayeredCheckpoint, PointerCatalog, PointerCatalogAssembly, + PointerCatalogLookup, RootSnapshot, RunCatalogSource, VisibilityReconciliation, + apply_control_refs, capture_ref_heads, load_capsule_run_git_control, load_layered_checkpoint, + load_layered_checkpoint_control, materialize_visible_ref_heads, order_ref_controls, + select_ref_control_frontiers, select_ref_journal_controls, +}; +use crate::error::{MetadataError, Result}; +use crate::git_visibility::GitVisibilityIndex; +use crab_storage::{Store, StoreLayout}; +use futures_util::{StreamExt, TryStreamExt}; +use std::collections::BTreeMap; +use std::sync::Arc; + +/// Capture current-state catalog roots without decoding catalog or Git pack bodies. +/// +/// The caller retains the captured roots against GC. Missing dependencies remain +/// errors. Control intake and each selected catalog query use the format byte bound. +pub async fn load_pointer_catalog_lookup_from_root( + layout: &StoreLayout, + root: &RootSnapshot, +) -> Result { + capture(layout, root).await?.lookup(layout) +} + +pub(super) async fn materialize_from_root( + layout: &StoreLayout, + root: &RootSnapshot, +) -> Result { + let captured = capture(layout, root).await?; + let base_catalog = match root.record().root().checkpoint() { + Some(pointer) => load_layered_checkpoint(layout, pointer) + .await? + .pointer_catalog()?, + None => PointerCatalog::new(), + }; + // Concurrent ref histories have a canonical binding order, not a global + // payload dependency order. Validate completeness after collecting the set. + let mut catalog = PointerCatalogAssembly::new(base_catalog); + for control in captured.controls { + if let Some(delta) = control.materialize_catalog(layout.store()).await? { + catalog.apply(&delta)?; + } + } + catalog.finish() +} + +struct Capture { + checkpoint: Option, + controls: Vec, + sources: Vec, +} + +impl Capture { + fn lookup(self, layout: &StoreLayout) -> Result { + let lookup = self + .checkpoint + .as_ref() + .map(|checkpoint| checkpoint.pointer_catalog_lookup(layout, MAX_CATALOG_BYTES)) + .transpose()? + .unwrap_or_else(|| PointerCatalogLookup::empty(MAX_CATALOG_BYTES)); + let order = self + .controls + .iter() + .map(|control| control.hash().to_owned()) + .collect::>(); + lookup.with_run_sources(self.sources, &order) + } +} + +async fn capture(layout: &StoreLayout, snapshot: &RootSnapshot) -> Result { + let root = snapshot.record().root(); + let (heads, active) = capture_ref_heads(layout, root).await?; + let visible = materialize_visible_ref_heads(root, &heads, &active)?; + let mut intake = root + .checkpoint() + .map_or(0, |pointer| pointer.control_size()); + for pointer in &visible.pointers { + intake = intake + .checked_add(pointer.control_size()) + .ok_or_else(|| corrupt("catalog control intake overflows"))?; + } + if intake > MAX_CATALOG_BYTES { + return Err(corrupt( + "catalog control intake exceeds the format byte bound", + )); + } + let checkpoint = match root.checkpoint() { + Some(pointer) => Some(load_layered_checkpoint_control(layout, pointer).await?), + None => None, + }; + let run_layout = layout.clone(); + let runs = futures_util::stream::iter(visible.pointers.into_iter().map(move |pointer| { + let layout = run_layout.clone(); + async move { + let (control, capsules) = load_capsule_run_git_control(&layout, &pointer).await?; + if !control.git_packs().is_empty() { + control.source_descriptor()?; + } + Ok::<_, MetadataError>((control.hash().to_owned(), (control, capsules))) + } + })) + .buffer_unordered(32) + .try_collect::>() + .await?; + let selected = select_ref_control_frontiers(root, &visible.frontiers, &runs)?; + let pending = select_ref_journal_controls(root, &selected.required, &runs)?; + let initial = match checkpoint + .as_ref() + .map(|checkpoint| checkpoint.visibility_reader(layout, MAX_CATALOG_BYTES)) + .transpose()? + .flatten() + { + Some(reader) => { + CapturedVisibility::from_checkpoint( + layout.store().clone(), + Arc::new(reader), + root.refs().keys().cloned(), + ) + .await? + } + None => CapturedVisibility::new( + layout.store().clone(), + GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new())?, + root.refs().keys().cloned(), + ), + }; + let mut visibility = VisibilityReconciliation::Paged(Box::new(initial)); + let mut controls = selected.root_controls; + for control in &controls { + visibility.apply(control).await?; + } + let ordered = order_ref_controls( + root.refs(), + root.peeled_refs(), + visibility, + &selected.predecessors, + pending, + ) + .await?; + let mut refs = root.refs().clone(); + let mut peeled = root.peeled_refs().clone(); + // Root-owned controls are already reflected in the root's ref values. + // Only journal controls advance them during this captured reconciliation. + for control in &ordered { + apply_control_refs(control, &mut refs, &mut peeled)?; + } + controls.extend(ordered); + if refs != visible.refs || peeled != visible.peeled_refs { + return Err(corrupt( + "materialized controls differ from the captured ref-head state", + )); + } + let sources = runs + .values() + .map(|(run, _)| run.catalog_source(layout)) + .collect::>>()? + .into_iter() + .flatten() + .collect(); + Ok(Capture { + checkpoint, + controls, + sources, + }) +} + +fn corrupt(reason: impl Into) -> MetadataError { + MetadataError::CorruptObject { + path: "capsule-protocol catalog capture".to_owned(), + reason: reason.into(), + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index c435b426a..0da15ce80 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -4,12 +4,18 @@ use bstr::ByteSlice; mod browse; mod capsule; +#[cfg(feature = "storage")] +mod captured_visibility; +#[cfg(feature = "storage")] +mod catalog_capture; mod history; mod layered; #[cfg(feature = "storage")] mod plan; mod pointer; mod ref_head; +#[cfg(feature = "storage")] +mod ref_view; mod root; mod run; #[cfg(feature = "storage")] @@ -25,6 +31,13 @@ pub use capsule::{ Capsule, CapsuleGitPack, CapsuleGitPackDescriptor, CapsuleSection, CapsuleSectionKind, CapsuleSectionLocation, }; +#[cfg(feature = "storage")] +pub use captured_visibility::{ + CapturedVisibility, VisibilityReconciliation, apply_transaction_visibility_index, + visibility_is_ready, +}; +#[cfg(feature = "storage")] +pub use catalog_capture::load_pointer_catalog_lookup_from_root; pub use history::{ HistorySegment, HistorySegmentPointer, HistorySegmentState, MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, @@ -45,13 +58,19 @@ pub use plan::{ #[cfg(feature = "storage")] pub use pointer::PointerCatalogLookup; pub use pointer::{ - FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + FileCatalogEntry, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, XorbCatalogEntry, + XorbChunkEntry, }; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, MAX_CAPSULE_REF_FRONTIER, MAX_CAPSULE_REF_HEADS, capsule_ref_name_from_key, capsule_ref_name_key, }; +#[cfg(feature = "storage")] +pub use ref_view::{ + RefControlFrontiers, VisibleRefHeads, apply_control_refs, materialize_visible_ref_heads, + order_ref_controls, select_ref_control_frontiers, select_ref_journal_controls, +}; pub use root::{ CapsulePointer, CheckpointPointer, GcFence, MAX_CAPSULE_FRONTIER, MAX_ROOT_BYTES, RepositoryRoot, RootRecord, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 2e9363aed..2b359154a 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -158,6 +158,36 @@ impl Default for PointerCatalog { } } +/// Assemble a captured visible catalog without assuming dependency order across refs. +/// +/// File bindings retain capture order; immutable conflicts fail immediately. +/// Dependency completeness is required before the catalog can leave the assembly. +#[must_use] +pub struct PointerCatalogAssembly { + catalog: PointerCatalog, +} + +impl PointerCatalogAssembly { + /// Start with the captured checkpoint catalog, or an empty catalog before checkpointing. + pub fn new(catalog: PointerCatalog) -> Self { + Self { catalog } + } + + /// Add one ordered visible delta, validating its rows and immutable conflicts. + /// + /// Discard the assembly if a delta fails. Dependencies may come from another + /// visible ref; callers must finish before using the resulting catalog. + pub fn apply(&mut self, delta: &PointerCatalog) -> Result<()> { + self.catalog.merge_delta(delta) + } + + /// Return the captured catalog only if its complete dependency closure is valid. + pub fn finish(self) -> Result { + self.catalog.validate(true, false)?; + Ok(self.catalog) + } +} + impl PointerCatalog { #[must_use] pub fn new() -> Self { @@ -190,6 +220,11 @@ impl PointerCatalog { /// Apply one publication delta, rejecting immutable identity conflicts. pub fn apply(&mut self, delta: &Self) -> Result<()> { + self.merge_delta(delta)?; + self.validate(true, false) + } + + fn merge_delta(&mut self, delta: &Self) -> Result<()> { delta.validate(false, false)?; for (hash, entry) in &delta.xorbs { self.insert_xorb(hash.clone(), entry.clone())?; @@ -209,7 +244,7 @@ impl PointerCatalog { } } } - self.validate(true, false) + Ok(()) } /// Encode this complete catalog as bounded authenticated page trees. @@ -446,6 +481,13 @@ mod tests { assert!(current.apply(&delta_with_base_xorb()).is_err()); } + #[test] + fn visible_assembly_rejects_unresolved_dependencies_at_finish() { + let mut assembly = PointerCatalogAssembly::new(PointerCatalog::new()); + assembly.apply(&delta_with_base_xorb()).unwrap(); + assert!(assembly.finish().is_err()); + } + #[test] fn apply_allows_new_file_mapping_but_not_new_size() { let mut current = catalog(); diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index 61346bb59..b7e3e9518 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -10,7 +10,7 @@ pub use reader::PointerCatalogLookup; const MAGIC: &[u8; 8] = b"CRBPCP01"; const TRAILER_BYTES: usize = 12; const MAX_DESCRIPTOR_BYTES: usize = 4096; -const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; +pub(in crate::capsule_protocol) const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/crab-metadata/src/capsule_protocol/ref_view.rs b/crates/crab-metadata/src/capsule_protocol/ref_view.rs new file mode 100644 index 000000000..4230bc951 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/ref_view.rs @@ -0,0 +1,330 @@ +//! Shared captured ref values and transaction ordering for Git and catalog readers. +use crate::capsule_protocol::{ + CapsuleControl, CapsulePointer, CapsuleRunControl, RepositoryRoot, VisibilityReconciliation, +}; +use crate::error::{MetadataError, Result}; +use std::collections::{BTreeMap, BTreeSet}; + +/// Visible ref values, publication positions and immutable frontiers for one capture. +pub struct VisibleRefHeads { + pub refs: BTreeMap, + pub peeled_refs: BTreeMap, + pub transactions: BTreeMap, + /// First uncompacted transaction offset and its ordered physical runs per ref. + pub frontiers: BTreeMap)>, + pub pointers: Vec, +} + +/// Materialize only ref-head metadata from one epoch-filtered stable capture. +pub fn materialize_visible_ref_heads( + root: &crate::capsule_protocol::RepositoryRoot, + heads: &[crate::capsule_protocol::CapsuleRefHead], + active: &BTreeSet, +) -> Result { + let mut refs = root.refs().clone(); + let mut peeled_refs = root.peeled_refs().clone(); + let mut transactions = BTreeMap::new(); + let mut frontiers = BTreeMap::new(); + let mut pointers = root.capsule_frontier().to_vec(); + for head in heads { + let state = head.visible(active); + if let Some(transaction_id) = state.transaction_id() { + transactions.insert(head.ref_name().to_owned(), transaction_id.to_owned()); + } + if state.transaction_id() + == root + .compacted_ref_transactions() + .get(head.ref_name()) + .map(String::as_str) + { + continue; + } + match state.oid() { + Some(oid) => { + refs.insert(head.ref_name().to_owned(), oid.to_owned()); + match state.peeled_oid() { + Some(peeled) => { + peeled_refs.insert(head.ref_name().to_owned(), peeled.to_owned()); + } + None => { + peeled_refs.remove(head.ref_name()); + } + } + } + None => { + refs.remove(head.ref_name()); + peeled_refs.remove(head.ref_name()); + } + } + for pointer in state.frontier() { + match pointers + .iter() + .find(|candidate| candidate.hash() == pointer.hash()) + { + Some(candidate) if candidate != pointer => { + return Err(corrupt_path( + "capsule-protocol ref heads", + "capsule run identity has conflicting authenticated metadata", + )); + } + Some(_) => {} + None => pointers.push(pointer.clone()), + } + } + // Pointers commit to exact ordered transaction IDs; both run loaders + // verify that binding. Resolve the checkpoint offset once for all readers. + let ids = state + .frontier() + .iter() + .flat_map(|pointer| pointer.transaction_ids()) + .collect::>(); + let start = match root.compacted_ref_transactions().get(head.ref_name()) { + Some(compacted) => match ids.iter().position(|id| *id == compacted) { + Some(index) => index + 1, + None if state.checkpoint_transaction_id() == Some(compacted.as_str()) => 0, + None => { + return Err(frontier_contract(format!( + "ref {} does not extend its compacted transaction", + head.ref_name() + ))); + } + }, + None if state.checkpoint_transaction_id().is_none() => 0, + None => { + return Err(frontier_contract(format!( + "ref {} names a checkpoint absent from the repository root", + head.ref_name() + ))); + } + }; + frontiers.insert( + head.ref_name().to_owned(), + (start, state.frontier().to_vec()), + ); + } + Ok(VisibleRefHeads { + refs, + peeled_refs, + transactions, + frontiers, + pointers, + }) +} + +/// Order selected controls by captured ref predecessors and visibility readiness. +pub async fn order_ref_controls( + base_refs: &BTreeMap, + base_peeled: &BTreeMap, + mut visibility: VisibilityReconciliation, + predecessors: &BTreeMap>, + mut pending: BTreeMap, +) -> Result> { + let mut refs = base_refs.clone(); + let mut peeled = base_peeled.clone(); + let tracked = pending.keys().cloned().collect::>(); + let mut applied = BTreeSet::new(); + let mut ordered = Vec::with_capacity(pending.len()); + while !pending.is_empty() { + let mut ready = None; + for (id, capsule) in &pending { + if predecessors + .get(capsule.transaction_id()) + .is_some_and(|required| { + required + .iter() + .any(|id| tracked.contains(id) && !applied.contains(id)) + }) + { + continue; + } + if visibility.ready(capsule, &refs).await? { + ready = Some(id.clone()); + break; + } + } + let id = ready.ok_or_else(|| { + corrupt_path( + "capsule-protocol ref heads", + "capsule ref history is cyclic or does not extend the compacted root", + ) + })?; + let capsule = pending + .remove(&id) + .ok_or_else(|| internal("ready capsule control disappeared"))?; + apply_control_refs(&capsule, &mut refs, &mut peeled)?; + visibility.apply(&capsule).await?; + applied.insert(capsule.transaction_id().to_owned()); + ordered.push(capsule); + } + Ok(ordered) +} + +/// Apply expected-old-checked ref edits to captured ref and peeled-ref maps. +pub fn apply_control_refs( + capsule: &CapsuleControl, + refs: &mut BTreeMap, + peeled_refs: &mut BTreeMap, +) -> Result<()> { + for edit in capsule.transaction().edits() { + if refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old() { + return Err(corrupt_path( + "capsule-protocol ref heads", + "capsule expected-old ref does not match its parent state", + )); + } + match edit.new_oid() { + Some(oid) => { + refs.insert(edit.ref_name().to_owned(), oid.to_owned()); + match edit.peeled_oid() { + Some(peeled) => { + peeled_refs.insert(edit.ref_name().to_owned(), peeled.to_owned()); + } + None => { + peeled_refs.remove(edit.ref_name()); + } + } + } + None => { + refs.remove(edit.ref_name()); + peeled_refs.remove(edit.ref_name()); + } + } + } + Ok(()) +} + +fn internal(reason: impl Into) -> MetadataError { + MetadataError::Internal(reason.into()) +} +fn corrupt_path(path: impl Into, reason: impl Into) -> MetadataError { + MetadataError::CorruptObject { + path: path.into(), + reason: reason.into(), + } +} + +/// Selected root controls and ref-frontier dependencies before visibility reconciliation. +pub struct RefControlFrontiers { + pub root_controls: Vec, + pub required: BTreeSet, + pub predecessors: BTreeMap>, + pub counts: BTreeMap, +} + +/// Resolve captured frontier offsets into verified physical run occurrences. +pub fn select_ref_control_frontiers( + root: &RepositoryRoot, + ref_frontiers: &BTreeMap)>, + runs: &BTreeMap)>, +) -> Result { + let mut required_transactions = BTreeSet::new(); + let mut transaction_predecessors = BTreeMap::>::new(); + let mut ref_capsule_counts = BTreeMap::new(); + for (ref_name, (start, frontier)) in ref_frontiers { + let transaction_ids = frontier + .iter() + .map(|pointer| { + runs.get(pointer.hash()) + .ok_or_else(|| internal("loaded capsule run control disappeared")) + }) + .collect::>>()? + .into_iter() + .flat_map(|(_, capsules)| capsules.iter().map(|capsule| capsule.transaction_id())) + .collect::>(); + let required = transaction_ids.get(*start..).ok_or_else(|| { + corrupt_path( + "capsule-protocol ref heads", + "captured frontier offset exceeds its validated transaction count", + ) + })?; + ref_capsule_counts.insert( + ref_name.clone(), + u32::try_from(required.len()).map_err(|_| internal("ref capsule count overflowed"))?, + ); + required_transactions.extend( + required + .iter() + .map(|transaction_id| (*transaction_id).to_owned()), + ); + for pair in required.windows(2) { + transaction_predecessors + .entry(pair[1].to_owned()) + .or_default() + .insert(pair[0].to_owned()); + } + } + let mut root_capsules = Vec::new(); + let mut root_capsule_identities = BTreeMap::new(); + for pointer in root.capsule_frontier() { + let (_, capsules) = runs + .get(pointer.hash()) + .ok_or_else(|| internal("loaded root capsule run control disappeared"))?; + for capsule in capsules { + match root_capsule_identities.get(capsule.transaction_id()) { + Some(existing) if existing != capsule => { + return Err(corrupt_path( + "capsule-protocol root frontier", + "transaction identity names conflicting capsules", + )); + } + Some(_) => {} + None => { + root_capsule_identities + .insert(capsule.transaction_id().to_owned(), capsule.clone()); + root_capsules.push(capsule.clone()); + } + } + } + } + + Ok(RefControlFrontiers { + root_controls: root_capsules, + required: required_transactions, + predecessors: transaction_predecessors, + counts: ref_capsule_counts, + }) +} + +/// Deduplicate only the journal transactions required by the captured ref frontiers. +pub fn select_ref_journal_controls( + root: &RepositoryRoot, + required: &BTreeSet, + runs: &BTreeMap)>, +) -> Result> { + let root_transactions = root + .capsule_frontier() + .iter() + .flat_map(|pointer| pointer.transaction_ids().iter().cloned()) + .collect::>(); + let mut journal_capsules = BTreeMap::new(); + for (_, capsules) in runs.values() { + for capsule in capsules { + if root_transactions.contains(capsule.transaction_id()) + || !required.contains(capsule.transaction_id()) + { + continue; + } + match journal_capsules.get(capsule.transaction_id()) { + Some(existing) if existing != capsule => { + return Err(corrupt_path( + "capsule-protocol ref heads", + "transaction identity names conflicting capsules", + )); + } + Some(_) => {} + None => { + journal_capsules.insert(capsule.transaction_id().to_owned(), capsule.clone()); + } + } + } + } + + Ok(journal_capsules) +} + +fn frontier_contract(reason: impl Into) -> MetadataError { + MetadataError::CapsuleContract { + record: "stored root", + reason: reason.into(), + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index d3ed7d433..8cc7fe82f 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -5,7 +5,7 @@ use futures_util::{StreamExt, TryStreamExt}; use object_store::ObjectMeta; use crate::capsule_protocol::{ - Capsule, CapsuleControl, CapsuleRefHead, CapsuleRun, CapsuleRunControl, CapsuleSectionKind, + CapsuleControl, CapsuleRefHead, CapsuleRun, CapsuleRunControl, CapsuleSectionKind, HistorySegment, HistorySegmentPointer, LayeredCheckpoint, MAX_CAPSULE_REF_HEADS, MAX_HISTORY_SEGMENT_BYTES, MAX_ROOT_BYTES, PointerCatalog, RootRecord, capsule_ref_name_key, }; @@ -480,198 +480,7 @@ pub async fn load_pointer_catalog_from_root( router: &StoreLayout, snapshot: &RootSnapshot, ) -> Result { - let root = snapshot.record().root().clone(); - let mut catalog = if let Some(pointer) = root.checkpoint() { - load_layered_checkpoint(router, pointer) - .await? - .pointer_catalog()? - } else { - PointerCatalog::new() - }; - for pointer in root.capsule_frontier() { - let run = load_run(router, pointer).await?; - for capsule in run.capsules() { - if let Some(delta) = capsule.pointer_catalog_delta()? { - catalog.apply(&delta)?; - } - } - } - for capsule in load_visible_ref_capsules(router, &root, &catalog).await? { - if let Some(delta) = capsule.pointer_catalog_delta()? { - catalog.apply(&delta)?; - } - } - Ok(catalog) -} - -async fn load_visible_ref_capsules( - router: &StoreLayout, - root: &super::RepositoryRoot, - base_catalog: &PointerCatalog, -) -> Result> { - let (heads, active) = capture_ref_heads(router, root).await?; - - let mut pointers = BTreeMap::new(); - let mut sequences = BTreeMap::new(); - let mut expected_refs = root.refs().clone(); - for head in heads { - let state = head.visible(&active); - if state.transaction_id() - == root - .compacted_ref_transactions() - .get(head.ref_name()) - .map(String::as_str) - { - continue; - } - match state.oid() { - Some(oid) => { - expected_refs.insert(head.ref_name().to_owned(), oid.to_owned()); - } - None => { - expected_refs.remove(head.ref_name()); - } - } - for pointer in state.frontier() { - match pointers.get(pointer.hash()) { - Some(existing) if existing != pointer => { - return Err(contract_error( - "capsule run identity has conflicting authenticated metadata", - )); - } - Some(_) => {} - None => { - pointers.insert(pointer.hash().to_owned(), pointer.clone()); - } - } - } - sequences.insert( - head.ref_name().to_owned(), - ( - state.checkpoint_transaction_id().map(str::to_owned), - state.frontier().to_vec(), - ), - ); - } - - let run_router = router.clone(); - let runs = futures_util::stream::iter(pointers.into_values().map(move |pointer| { - let router = run_router.clone(); - async move { - load_run(&router, &pointer) - .await - .map(|run| (run.hash().to_owned(), run)) - } - })) - .buffer_unordered(32) - .try_collect::>() - .await?; - let mut required = BTreeSet::new(); - for (ref_name, (checkpoint_transaction_id, frontier)) in sequences { - let transaction_ids = frontier - .iter() - .flat_map(|pointer| { - runs.get(pointer.hash()) - .into_iter() - .flat_map(|run| run.capsules().iter().map(Capsule::transaction_id)) - }) - .collect::>(); - let start = match root.compacted_ref_transactions().get(&ref_name) { - Some(compacted) => match transaction_ids - .iter() - .position(|transaction_id| *transaction_id == compacted) - { - Some(index) => index + 1, - None if checkpoint_transaction_id.as_deref() == Some(compacted.as_str()) => 0, - None => { - return Err(contract_error(format!( - "ref {ref_name} does not extend its compacted transaction" - ))); - } - }, - None if checkpoint_transaction_id.is_none() => 0, - None => { - return Err(contract_error(format!( - "ref {ref_name} names a checkpoint absent from the repository root" - ))); - } - }; - required.extend( - transaction_ids[start..] - .iter() - .map(|transaction_id| (*transaction_id).to_owned()), - ); - } - let mut pending = BTreeMap::new(); - for capsule in runs - .values() - .flat_map(|run| run.capsules()) - .filter(|capsule| required.contains(capsule.transaction_id())) - { - match pending.get(capsule.transaction_id()) { - Some(existing) if existing != capsule => { - return Err(contract_error( - "transaction identity names conflicting capsules", - )); - } - Some(_) => {} - None => { - pending.insert(capsule.transaction_id().to_owned(), capsule.clone()); - } - } - } - let mut refs = root.refs().clone(); - let mut catalog = base_catalog.clone(); - let mut ordered = Vec::with_capacity(pending.len()); - while !pending.is_empty() { - let mut ready = None; - for (id, capsule) in &pending { - let transaction = capsule.transaction()?; - if transaction - .edits() - .iter() - .any(|edit| refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old()) - { - continue; - } - let mut candidate = catalog.clone(); - if let Some(delta) = capsule.pointer_catalog_delta()? - && candidate.apply(&delta).is_err() - { - continue; - } - ready = Some(id.clone()); - break; - } - let Some(ready) = ready else { - return Err(contract_error( - "capsule ref history is cyclic or has an unsatisfied catalog dependency", - )); - }; - let capsule = pending - .remove(&ready) - .ok_or_else(|| contract_error("ready capsule disappeared"))?; - for edit in capsule.transaction()?.edits() { - match edit.new_oid() { - Some(oid) => { - refs.insert(edit.ref_name().to_owned(), oid.to_owned()); - } - None => { - refs.remove(edit.ref_name()); - } - } - } - if let Some(delta) = capsule.pointer_catalog_delta()? { - catalog.apply(&delta)?; - } - ordered.push(capsule); - } - if refs != expected_refs { - return Err(contract_error( - "materialized capsules do not match visible ref-head state", - )); - } - Ok(ordered) + super::catalog_capture::materialize_from_root(router, snapshot).await } /// Capture one stable, epoch-filtered ref-head view and its committed activations. diff --git a/crates/crab-metadata/src/capsule_protocol/store/tests.rs b/crates/crab-metadata/src/capsule_protocol/store/tests.rs index e407e0061..45a3729dd 100644 --- a/crates/crab-metadata/src/capsule_protocol/store/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/store/tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::capsule_protocol::{ - CapsuleRefEdit, CapsuleRefState, CapsuleTransaction, RepositoryRoot, + Capsule, CapsuleRefEdit, CapsuleRefState, CapsuleTransaction, RepositoryRoot, }; use std::sync::Arc; @@ -571,3 +571,139 @@ async fn ref_capture_rejects_uncommitted_base_changes_only_in_the_active_epoch() } } } + +#[tokio::test] +async fn full_catalog_matches_selected_closure_across_independent_ref_dependencies() { + use crate::capsule_protocol::{ + CapsulePointer, CapsuleSection, FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, + XorbChunkEntry, load_pointer_catalog_lookup_from_root, + }; + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "repositories/catalog-order".to_owned(), + ); + let root = create_root( + &layout, + RootRecord::encode(RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap()) + .unwrap(), + ) + .await + .unwrap(); + let descriptors = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new( + "refs/tags/source", + None, + Some("1".repeat(40)), + None, + )], + ) + .unwrap(); + let descriptor_id = descriptors.id().unwrap(); + let mut binding = None; + for index in 0..256 { + let candidate = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new( + format!("refs/tags/consumer-{index}"), + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + if candidate.id().unwrap() < descriptor_id { + binding = Some(candidate); + break; + } + } + let binding = binding.expect("fixture must put its dependent binding first in hash order"); + let file = "3".repeat(64); + let shard = "4".repeat(64); + let xorb = "5".repeat(64); + let mut complete = PointerCatalog::new(); + complete + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 99, + "6".repeat(64), + vec![XorbChunkEntry::new("7".repeat(64), 16)], + ), + ) + .unwrap(); + complete + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + let mut file_delta = PointerCatalog::new(); + file_delta + .insert_file(file.clone(), FileCatalogEntry::new(16, shard)) + .unwrap(); + for (transaction, delta) in [(&descriptors, &complete), (&binding, &file_delta)] { + let capsule = Capsule::build( + transaction, + vec![], + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + delta.encode_delta().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let edit = &transaction.edits()[0]; + let head = CapsuleRefHead::from_root( + edit.ref_name(), + root.record().root().ref_epoch().to_owned(), + None, + None, + ) + .unwrap(); + let state = head + .successor_state( + &BTreeSet::new(), + edit.new_oid().map(str::to_owned), + None, + transaction.id().unwrap(), + vec![pointer], + ) + .unwrap(); + layout + .store() + .create_strict( + &layout.capsule_ref_head_path(&capsule_ref_name_key(edit.ref_name())), + head.commit(state).unwrap().encode().unwrap(), + ) + .await + .unwrap(); + } + complete.apply(&file_delta).unwrap(); + let selected = load_pointer_catalog_lookup_from_root(&layout, &root) + .await + .unwrap() + .files(layout.store(), &[file]) + .await + .unwrap(); + assert_eq!(selected, complete); + assert_eq!( + load_pointer_catalog_from_root(&layout, &root) + .await + .unwrap(), + selected + ); +} diff --git a/crates/crab-metadata/src/file_index_lookup.rs b/crates/crab-metadata/src/file_index_lookup.rs index 872418e4b..c8f3f4c9b 100644 --- a/crates/crab-metadata/src/file_index_lookup.rs +++ b/crates/crab-metadata/src/file_index_lookup.rs @@ -1,4 +1,4 @@ -//! Read-only file-index lookup over a repository's SlateDB metadata. +//! Snapshot-bound file/shard lookup over authenticated repository metadata. //! //! This Module owns the read Interface for `file_hash -> shard_hash` point //! lookups. It does not own metadata writes, CLI error presentation, metrics, @@ -225,7 +225,6 @@ async fn lookup_committed_record( /// closes the underlying SlateDB reader when consumed. pub struct FileIndexLookupSession { catalog: Option>, - static_entries: Option>>, reader: Option>, anchor: Option, storage: crab_storage::Store, @@ -246,7 +245,6 @@ impl FileIndexLookupSession { ) -> Result { Ok(Self { catalog: None, - static_entries: None, reader: None, anchor: CommittedShardAnchor::from_snapshot(snapshot)?, storage: router.store().clone(), @@ -310,7 +308,6 @@ impl FileIndexLookupSession { let anchor = CommittedShardAnchor::from_snapshot(snapshot)?; Ok(Self { catalog: None, - static_entries: None, reader: None, anchor, storage: router.store().clone(), @@ -371,7 +368,6 @@ impl FileIndexLookupSession { }; Ok(Self { catalog: None, - static_entries: None, reader: None, anchor, storage: router.store().clone(), @@ -393,33 +389,10 @@ impl FileIndexLookupSession { // Only an absent root permits v1 lookup. Missing dependencies // under a verified v2 root must not revive older v1 authority. let catalog = - crate::capsule_protocol::load_pointer_catalog_from_root(&router, &root).await?; - let entries = catalog - .files() - .iter() - .map(|(file_hash, entry)| { - Ok(( - MerkleHash::from_hex(file_hash).map_err(|error| { - MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!("invalid file hash {file_hash}: {error}"), - } - })?, - MerkleHash::from_hex(entry.shard_hash()).map_err(|error| { - MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!( - "invalid shard hash {}: {error}", - entry.shard_hash() - ), - } - })?, - )) - }) - .collect::>>()?; + crate::capsule_protocol::load_pointer_catalog_lookup_from_root(&router, &root) + .await?; return Ok(Self { - catalog: None, - static_entries: Some(Arc::new(entries)), + catalog: Some(Arc::new(catalog)), reader: None, anchor: None, storage, @@ -444,7 +417,6 @@ impl FileIndexLookupSession { }; let mut session = Self { catalog: None, - static_entries: None, reader: None, anchor, storage, @@ -491,9 +463,6 @@ impl FileIndexLookupSession { .next() .flatten()); } - if let Some(entries) = &self.static_entries { - return Ok(entries.get(file_hash).copied()); - } if let Some(reader) = self.reader.as_ref() && let Some(record) = lookup_committed_record(reader, *file_hash, self.anchor.as_ref()).await? @@ -543,13 +512,6 @@ impl FileIndexLookupSession { }) .collect(); } - if let Some(entries) = &self.static_entries { - return Ok(file_hashes - .iter() - .map(|file_hash| entries.get(file_hash).copied()) - .collect()); - } - let records = self.lookup_committed_records_batch(file_hashes).await?; let mut out = records .into_iter() @@ -799,7 +761,6 @@ impl LookupSource { match self { Self::Catalog { router, lookup } => Ok(FileIndexLookupSession { catalog: Some(Arc::clone(lookup)), - static_entries: None, reader: None, anchor: None, storage: router.store().clone(), diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs index 458aacdb7..f656df3f9 100644 --- a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -1,16 +1,18 @@ use super::*; use crate::capsule_protocol::{ - CapsuleGitPack, FileCatalogEntry, LayeredCheckpoint, PackLayer, PointerCatalog, - PointerCatalogLookup, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + CapsuleGitPack, CheckpointPointer, FileCatalogEntry, LayeredCheckpoint, PackLayer, + PointerCatalog, PointerCatalogLookup, RepositoryRoot, RootRecord, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }; use bytes::Bytes; use object_store::memory::InMemory; +use std::collections::BTreeMap; use std::sync::atomic::AtomicUsize; pub(super) async fn capture( layout: &crab_storage::StoreLayout, catalog: PointerCatalog, -) -> (PointerCatalogLookup, object_store::path::Path) { +) -> (PointerCatalogLookup, object_store::path::Path, RootRecord) { let pack = CapsuleGitPack::new( Bytes::from_static(b"pack"), Bytes::from_static(b"index"), @@ -24,8 +26,103 @@ pub(super) async fn capture( .unwrap() .source_descriptor() .unwrap(); - let checkpoint = - LayeredCheckpoint::build(1, &"3".repeat(64), vec![source], catalog, None).unwrap(); + let initial = + RootRecord::encode(RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap()) + .unwrap(); + let transaction = crate::capsule_protocol::CapsuleTransaction::new( + initial.digest(), + vec![crate::capsule_protocol::CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("1".repeat(40)), + None, + )], + ) + .unwrap(); + let run = crate::capsule_protocol::CapsuleRun::leaf( + crate::capsule_protocol::Capsule::build(&transaction, vec![], vec![]).unwrap(), + ) + .unwrap(); + let run_pointer = crate::capsule_protocol::CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + layout + .store() + .put_overwrite(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let base = RootRecord::encode( + initial + .root() + .advance( + initial.digest(), + BTreeMap::from([("refs/heads/main".to_owned(), "1".repeat(40))]), + BTreeMap::new(), + vec![run_pointer], + transaction.id().unwrap().as_str(), + ) + .unwrap(), + ) + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + base.root().generation(), + base.digest(), + vec![source], + catalog, + None, + ) + .unwrap(); + let pointer = CheckpointPointer::new_layered( + checkpoint.hash(), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.control_size(), + checkpoint.footer_hash(), + checkpoint.covered_generation(), + checkpoint.covered_root_digest(), + checkpoint.pack_count().unwrap(), + checkpoint.object_count().unwrap(), + ) + .unwrap(); + let history = crate::capsule_protocol::HistorySegment::build( + pointer.clone(), + None, + crate::capsule_protocol::HistorySegmentState::new( + base.root().refs().clone(), + base.root().peeled_refs().clone(), + base.root().head().to_owned(), + base.root().compacted_ref_transactions().clone(), + base.root().capsule_frontier().to_vec(), + ), + ) + .unwrap(); + layout + .store() + .put_overwrite( + &layout.capsule_history_segment_path(history.hash()), + history.bytes().clone(), + ) + .await + .unwrap(); + let root = RootRecord::encode( + base.root() + .install_checkpoint(base.digest(), pointer, Some(history.pointer().unwrap())) + .unwrap(), + ) + .unwrap(); + layout + .store() + .put_overwrite(&layout.capsule_root_path(), root.bytes().clone()) + .await + .unwrap(); let path = layout.capsule_checkpoint_path(checkpoint.hash()); layout .store() @@ -45,6 +142,7 @@ pub(super) async fn capture( ( control.pointer_catalog_lookup(layout, 512 * 1024).unwrap(), path, + root, ) } @@ -87,7 +185,7 @@ async fn pinned_batches_preserve_order_misses_and_page_bounds_as_history_grows() observed.fetch_add(1, Ordering::SeqCst); })); let layout = crab_storage::StoreLayout::new(storage, "pinned-lookup".to_owned()); - let (captured, _) = capture(&layout, catalog(count)).await; + let (captured, _, _) = capture(&layout, catalog(count)).await; let lookup = SharedFileIndexLookup::for_pointer_catalog(layout.clone(), captured); assert_eq!( reads.load(Ordering::SeqCst), @@ -126,7 +224,7 @@ async fn missing_captured_pages_are_errors_even_when_a_new_catalog_exists() { crab_storage::Store::new(Arc::new(InMemory::new())), "missing-page".to_owned(), ); - let (captured, path) = capture(&layout, catalog(100)).await; + let (captured, path, _) = capture(&layout, catalog(100)).await; capture(&layout, catalog(101)).await; layout.store().delete(&path).await.unwrap(); let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); @@ -138,3 +236,53 @@ async fn missing_captured_pages_are_errors_even_when_a_new_catalog_exists() { ); lookup.close().await.unwrap(); } + +#[tokio::test] +async fn current_state_opener_keeps_catalog_pages_selected_and_pins_its_first_root() { + use std::sync::atomic::AtomicU64; + for count in [100, 20_000] { + let reads = Arc::new(AtomicUsize::new(0)); + let bytes = Arc::new(AtomicU64::new(0)); + let observed_reads = reads.clone(); + let observed_bytes = bytes.clone(); + let storage = crab_storage::Store::new(Arc::new(InMemory::new())) + .with_read_request_observer(Arc::new(move |_| { + observed_reads.fetch_add(1, Ordering::SeqCst); + })) + .with_read_byte_observer(Arc::new(move |size| { + observed_bytes.fetch_add(size, Ordering::SeqCst); + })); + let layout = crab_storage::StoreLayout::new(storage.clone(), "current-catalog".to_owned()); + let (_, _, root) = capture(&layout, catalog(count)).await; + assert_eq!(root.root().generation(), 1); + let lookup = SharedFileIndexLookup::new_for_storage(&storage, layout.repo_prefix()); + assert_eq!(reads.load(Ordering::SeqCst), 0); + let file = super::tests::hash_from_seed(42); + let shard = super::tests::hash_from_seed(1_000_042); + assert_eq!( + lookup.lookup_batch(&[file, file]).await.unwrap(), + vec![Some(shard), Some(shard)] + ); + assert!( + reads.load(Ordering::SeqCst) <= 32 && bytes.load(Ordering::SeqCst) <= 512 * 1024, + "current-state fixed query materialized unrelated catalog history: count={count}, reads={}, bytes={}", + reads.load(Ordering::SeqCst), + bytes.load(Ordering::SeqCst) + ); + let (_, _, later_root) = capture(&layout, catalog(count + 1)).await; + layout + .store() + .put_overwrite(&layout.capsule_root_path(), later_root.bytes().clone()) + .await + .unwrap(); + let later_file = super::tests::hash_from_seed(count as u64); + assert_eq!(lookup.lookup(&later_file).await.unwrap(), None); + let latest = SharedFileIndexLookup::new_for_storage(&storage, layout.repo_prefix()); + assert_eq!( + latest.lookup(&later_file).await.unwrap(), + Some(super::tests::hash_from_seed(1_000_000 + count as u64)) + ); + lookup.close().await.unwrap(); + latest.close().await.unwrap(); + } +} diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 8a6ea11d4..eb53e0b23 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -61,6 +61,12 @@ work. `git_visibility_index`, `candidate_git_visibility` and `tip_bound_transitions` asynchronously materialize captured controls only when bulk inspection needs them. They do not recapture refs or switch origins. +Metadata owns the stable ref capture, control selection, visibility reconciliation +and transaction ordering used by Git views and current-state catalog lookup. +Current-state file sessions capture their first root once and query selected +file/shard/xorb pages; they do not materialize the complete catalog or open +SlateDB under a v2 root. Full maintenance inspection remains explicit. + Pinned hydration and remote-mount setup pass captured paged catalog roots into metadata's shared file lookup. A lookup loads only its selected file dependency closure, preserving the captured publication order and immutable descriptor diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 8fb1df7cb..17941f462 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -2,10 +2,15 @@ use bytes::Bytes; use crab_metadata::capsule_protocol::{ - Capsule, CapsuleControl, CapsulePointer, CapsuleRun, CapsuleRunControl, CheckpointPointer, - LayeredCheckpoint, PackMemberDescriptor, PackRange, PackSourceDescriptor, PackSourceKind, - PointerCatalog, RootRecord, capture_ref_heads, load_root, resolve_referenced_activations, - visibility_object_set_digest, + Capsule, CapsuleControl, CapsulePointer, CapsuleRun, CapsuleRunControl, CapturedVisibility, + CheckpointPointer, LayeredCheckpoint, PackMemberDescriptor, PackRange, PackSourceDescriptor, + PackSourceKind, PointerCatalog, PointerCatalogAssembly, RootRecord, VisibilityReconciliation, + apply_transaction_visibility_index, capture_ref_heads, load_root, + resolve_referenced_activations, visibility_is_ready, visibility_object_set_digest, +}; +use crab_metadata::capsule_protocol::{ + apply_control_refs, materialize_visible_ref_heads, order_ref_controls, + select_ref_control_frontiers, select_ref_journal_controls, }; use crab_storage::{Store, StoreLayout}; use futures_util::future::try_join_all; @@ -18,7 +23,8 @@ use tokio_util::sync::CancellationToken; use crate::{ReadError, Result}; -mod visibility; +#[cfg(test)] +mod visibility_tests; const LAYERED_SIDECAR_COALESCE_GAP_BYTES: u64 = 64 * 1024; const LAYERED_SIDECAR_MAX_EXTRA_BYTES: u64 = 4 * 1024 * 1024; @@ -783,7 +789,7 @@ impl CapsuleRepositoryView { self.catalog_sources .as_ref() .ok_or(ReadError::IncompleteCatalogView)?; - let mut catalog = + let base_catalog = match self.checkpoint.as_ref() { Some(checkpoint) if checkpoint.is_control_only() => { let layout = self.metadata_layout.as_ref().ok_or_else(|| { @@ -800,6 +806,7 @@ impl CapsuleRepositoryView { Some(checkpoint) => checkpoint.pointer_catalog()?, None => PointerCatalog::new(), }; + let mut catalog = PointerCatalogAssembly::new(base_catalog); for capsule in &self.capsules { if let Some(delta) = capsule.pointer_catalog_delta()? { catalog.apply(&delta)?; @@ -813,7 +820,7 @@ impl CapsuleRepositoryView { catalog.apply(&delta)?; } } - Ok(catalog) + Ok(catalog.finish()?) } /// Open the authenticated embedded Git packs as a filesystem-free repository. @@ -4553,7 +4560,7 @@ async fn assemble_view( let mut required_transactions = BTreeSet::new(); let mut transaction_predecessors = BTreeMap::>::new(); let mut ref_capsule_counts = BTreeMap::new(); - for (ref_name, (checkpoint_transaction_id, frontier)) in &ref_frontiers { + for (ref_name, (start, frontier)) in &ref_frontiers { let transaction_ids = frontier .iter() .map(|pointer| { @@ -4564,34 +4571,12 @@ async fn assemble_view( .into_iter() .flat_map(|run| run.capsules().iter().map(Capsule::transaction_id)) .collect::>(); - let start = match snapshot - .record() - .root() - .compacted_ref_transactions() - .get(ref_name) - { - Some(compacted) => match transaction_ids - .iter() - .position(|transaction_id| *transaction_id == compacted) - { - Some(index) => index + 1, - None if checkpoint_transaction_id.as_deref() == Some(compacted.as_str()) => 0, - None => { - return Err(corrupt_path( - "capsule-protocol ref heads", - format!("ref {ref_name} does not extend its compacted transaction"), - )); - } - }, - None if checkpoint_transaction_id.is_none() => 0, - None => { - return Err(corrupt_path( - "capsule-protocol ref heads", - format!("ref {ref_name} names a checkpoint absent from the repository root"), - )); - } - }; - let required = &transaction_ids[start..]; + let required = transaction_ids.get(*start..).ok_or_else(|| { + corrupt_path( + "capsule-protocol ref heads", + "captured frontier offset exceeds its validated transaction count", + ) + })?; ref_capsule_counts.insert( ref_name.clone(), u32::try_from(required.len()) @@ -4851,95 +4836,9 @@ async fn assemble_layered_control_view( capsule_run_sources.push(source); } } - let root_transactions = snapshot - .record() - .root() - .capsule_frontier() - .iter() - .flat_map(|pointer| pointer.transaction_ids().iter().cloned()) - .collect::>(); - let mut required_transactions = BTreeSet::new(); - let mut transaction_predecessors = BTreeMap::>::new(); - let mut ref_capsule_counts = BTreeMap::new(); - for (ref_name, (checkpoint_transaction_id, frontier)) in &ref_frontiers { - let transaction_ids = frontier - .iter() - .map(|pointer| { - runs.get(pointer.hash()) - .ok_or_else(|| ReadError::internal("loaded capsule run control disappeared")) - }) - .collect::>>()? - .into_iter() - .flat_map(|(_, capsules)| capsules.iter().map(|capsule| capsule.transaction_id())) - .collect::>(); - let start = match snapshot - .record() - .root() - .compacted_ref_transactions() - .get(ref_name) - { - Some(compacted) => match transaction_ids - .iter() - .position(|transaction_id| *transaction_id == compacted) - { - Some(index) => index + 1, - None if checkpoint_transaction_id.as_deref() == Some(compacted.as_str()) => 0, - None => { - return Err(corrupt_path( - "capsule-protocol ref heads", - format!("ref {ref_name} does not extend its compacted transaction"), - )); - } - }, - None if checkpoint_transaction_id.is_none() => 0, - None => { - return Err(corrupt_path( - "capsule-protocol ref heads", - format!("ref {ref_name} names a checkpoint absent from the repository root"), - )); - } - }; - let required = &transaction_ids[start..]; - ref_capsule_counts.insert( - ref_name.clone(), - u32::try_from(required.len()) - .map_err(|_| ReadError::internal("ref capsule count overflowed"))?, - ); - required_transactions.extend( - required - .iter() - .map(|transaction_id| (*transaction_id).to_owned()), - ); - for pair in required.windows(2) { - transaction_predecessors - .entry(pair[1].to_owned()) - .or_default() - .insert(pair[0].to_owned()); - } - } - let mut root_capsules = Vec::new(); - let mut root_capsule_identities = BTreeMap::new(); - for pointer in snapshot.record().root().capsule_frontier() { - let (_, capsules) = runs - .get(pointer.hash()) - .ok_or_else(|| ReadError::internal("loaded root capsule run control disappeared"))?; - for capsule in capsules { - match root_capsule_identities.get(capsule.transaction_id()) { - Some(existing) if existing != capsule => { - return Err(corrupt_path( - "capsule-protocol root frontier", - "transaction identity names conflicting capsules", - )); - } - Some(_) => {} - None => { - root_capsule_identities - .insert(capsule.transaction_id().to_owned(), capsule.clone()); - root_capsules.push(capsule.clone()); - } - } - } - } + let selected = select_ref_control_frontiers(snapshot.record().root(), &ref_frontiers, &runs)?; + let mut root_capsules = selected.root_controls; + let ref_capsule_counts = selected.counts; if footer_only { return Ok(CapsuleRepositoryView { root: snapshot, @@ -4964,28 +4863,9 @@ async fn assemble_layered_control_view( frontier_object_admission, }); } - let mut journal_capsules = BTreeMap::new(); - for (_, capsules) in runs.values() { - for capsule in capsules { - if root_transactions.contains(capsule.transaction_id()) - || !required_transactions.contains(capsule.transaction_id()) - { - continue; - } - match journal_capsules.get(capsule.transaction_id()) { - Some(existing) if existing != capsule => { - return Err(corrupt_path( - "capsule-protocol ref heads", - "transaction identity names conflicting capsules", - )); - } - Some(_) => {} - None => { - journal_capsules.insert(capsule.transaction_id().to_owned(), capsule.clone()); - } - } - } - } + let journal_capsules = + select_ref_journal_controls(snapshot.record().root(), &selected.required, &runs)?; + let transaction_predecessors = selected.predecessors; let checkpoint_visibility = if matches!(read, ControlRead::Git) { checkpoint .as_ref() @@ -5000,22 +4880,18 @@ async fn assemble_layered_control_view( let names = snapshot.record().root().refs().keys().cloned(); let initial = match &checkpoint_visibility { Some(reader) => { - visibility::Visibility::from_checkpoint( - router.store().clone(), - reader.clone(), - names, - ) - .await? + CapturedVisibility::from_checkpoint(router.store().clone(), reader.clone(), names) + .await? } - None => visibility::Visibility::new( + None => CapturedVisibility::new( router.store().clone(), visibility_index_for_checkpoint(None)?, names, ), }; - visibility::Reconciliation::Paged(Box::new(initial)) + VisibilityReconciliation::Paged(Box::new(initial)) } else { - visibility::Reconciliation::Complete(visibility_index_for_checkpoint(checkpoint.as_ref())?) + VisibilityReconciliation::Complete(visibility_index_for_checkpoint(checkpoint.as_ref())?) }; for capsule in &root_capsules { visibility.apply(capsule).await?; @@ -5139,14 +5015,6 @@ fn extend_frontier_object_admission( } } -struct VisibleRefHeads { - refs: BTreeMap, - peeled_refs: BTreeMap, - transactions: BTreeMap, - frontiers: BTreeMap, Vec)>, - pointers: Vec, -} - fn state_digest(root: &RootRecord, transactions: &BTreeMap) -> String { let mut hasher = blake3::Hasher::new(); hasher.update(b"crab capsule repository view v2\0"); @@ -5159,78 +5027,6 @@ fn state_digest(root: &RootRecord, transactions: &BTreeMap) -> S hasher.finalize().to_hex().to_string() } -fn materialize_visible_ref_heads( - root: &crab_metadata::capsule_protocol::RepositoryRoot, - heads: &[crab_metadata::capsule_protocol::CapsuleRefHead], - active: &BTreeSet, -) -> Result { - let mut refs = root.refs().clone(); - let mut peeled_refs = root.peeled_refs().clone(); - let mut transactions = BTreeMap::new(); - let mut frontiers = BTreeMap::new(); - let mut pointers = root.capsule_frontier().to_vec(); - for head in heads { - let state = head.visible(active); - if let Some(transaction_id) = state.transaction_id() { - transactions.insert(head.ref_name().to_owned(), transaction_id.to_owned()); - } - if state.transaction_id() - == root - .compacted_ref_transactions() - .get(head.ref_name()) - .map(String::as_str) - { - continue; - } - match state.oid() { - Some(oid) => { - refs.insert(head.ref_name().to_owned(), oid.to_owned()); - match state.peeled_oid() { - Some(peeled) => { - peeled_refs.insert(head.ref_name().to_owned(), peeled.to_owned()); - } - None => { - peeled_refs.remove(head.ref_name()); - } - } - } - None => { - refs.remove(head.ref_name()); - peeled_refs.remove(head.ref_name()); - } - } - for pointer in state.frontier() { - match pointers - .iter() - .find(|candidate| candidate.hash() == pointer.hash()) - { - Some(candidate) if candidate != pointer => { - return Err(corrupt_path( - "capsule-protocol ref heads", - "capsule run identity has conflicting authenticated metadata", - )); - } - Some(_) => {} - None => pointers.push(pointer.clone()), - } - } - frontiers.insert( - head.ref_name().to_owned(), - ( - state.checkpoint_transaction_id().map(str::to_owned), - state.frontier().to_vec(), - ), - ); - } - Ok(VisibleRefHeads { - refs, - peeled_refs, - transactions, - frontiers, - pointers, - }) -} - async fn capture_selected_ref_heads( router: &StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, @@ -5358,175 +5154,6 @@ fn order_ref_capsules( Ok(ordered) } -async fn order_ref_controls( - base_refs: &BTreeMap, - base_peeled: &BTreeMap, - mut visibility: visibility::Reconciliation, - predecessors: &BTreeMap>, - mut pending: BTreeMap, -) -> Result> { - let mut refs = base_refs.clone(); - let mut peeled = base_peeled.clone(); - let tracked = pending.keys().cloned().collect::>(); - let mut applied = BTreeSet::new(); - let mut ordered = Vec::with_capacity(pending.len()); - while !pending.is_empty() { - let mut ready = None; - for (id, capsule) in &pending { - if predecessors - .get(capsule.transaction_id()) - .is_some_and(|required| { - required - .iter() - .any(|id| tracked.contains(id) && !applied.contains(id)) - }) - { - continue; - } - if visibility.ready(capsule, &refs).await? { - ready = Some(id.clone()); - break; - } - } - let id = ready.ok_or_else(|| { - corrupt_path( - "capsule-protocol ref heads", - "capsule ref history is cyclic or does not extend the compacted root", - ) - })?; - let capsule = pending - .remove(&id) - .ok_or_else(|| ReadError::internal("ready capsule control disappeared"))?; - apply_control_refs(&capsule, &mut refs, &mut peeled)?; - visibility.apply(&capsule).await?; - applied.insert(capsule.transaction_id().to_owned()); - ordered.push(capsule); - } - Ok(ordered) -} - -fn control_is_ready( - capsule: &CapsuleControl, - refs: &BTreeMap, - visibility_index: &crab_metadata::git_visibility::GitVisibilityIndex, -) -> Result { - visibility_is_ready( - capsule.transaction(), - capsule.visibility_delta()?, - refs, - visibility_index, - ) -} - -fn visibility_is_ready( - transaction: &crab_metadata::capsule_protocol::CapsuleTransaction, - delta: Option<&crab_metadata::capsule_protocol::CapsuleVisibilityDelta>, - refs: &BTreeMap, - visibility_index: &crab_metadata::git_visibility::GitVisibilityIndex, -) -> Result { - if transaction - .edits() - .iter() - .any(|edit| refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old()) - { - return Ok(false); - } - let Some(delta) = delta else { - return Ok(true); - }; - // Transactions have unique refs. Count matched rows without copying OID lists; - // unmatched evidence still fails before it can authorize a publication. - let evidence = delta.edits(); - let mut matched = 0; - for edit in transaction.edits() { - let Some(new_oid) = edit.new_oid() else { - if evidence.get(edit.ref_name()).is_some() { - return Err(corrupt_path( - "capsule Git visibility", - "deleted ref has visibility evidence", - )); - } - continue; - }; - let Some(visibility) = evidence.get(edit.ref_name()) else { - return Err(corrupt_path( - "capsule Git visibility", - "live ref edit has no visibility evidence", - )); - }; - matched += 1; - visibility.validate()?; - if visibility.new_oid != new_oid { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match its ref edit", - )); - } - match edit.expected_old() { - Some(expected_old) => { - if visibility.old_oid.as_deref() != Some(expected_old) { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match the expected old ref", - )); - } - if !visibility_index.contains_ref(edit.ref_name()) { - return Ok(false); - } - } - None if visibility.replaces => {} - None => { - let Some(old_oid) = visibility.old_oid.as_deref() else { - continue; - }; - if !visibility_index.contains_hex_in_any_ref(old_oid) { - return Ok(false); - } - } - } - } - if evidence.len() != matched { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence contains an uncommitted ref", - )); - } - Ok(true) -} - -fn apply_control_refs( - capsule: &CapsuleControl, - refs: &mut BTreeMap, - peeled_refs: &mut BTreeMap, -) -> Result<()> { - for edit in capsule.transaction().edits() { - if refs.get(edit.ref_name()).map(String::as_str) != edit.expected_old() { - return Err(corrupt_path( - "capsule-protocol ref heads", - "capsule expected-old ref does not match its parent state", - )); - } - match edit.new_oid() { - Some(oid) => { - refs.insert(edit.ref_name().to_owned(), oid.to_owned()); - match edit.peeled_oid() { - Some(peeled) => { - peeled_refs.insert(edit.ref_name().to_owned(), peeled.to_owned()); - } - None => { - peeled_refs.remove(edit.ref_name()); - } - } - } - None => { - refs.remove(edit.ref_name()); - peeled_refs.remove(edit.ref_name()); - } - } - } - Ok(()) -} - fn capsule_is_ready( capsule: &Capsule, refs: &BTreeMap, @@ -5534,68 +5161,19 @@ fn capsule_is_ready( ) -> Result { let transaction = capsule.transaction()?; let delta = capsule.visibility_delta()?; - visibility_is_ready(&transaction, delta.as_ref(), refs, visibility_index) + Ok(visibility_is_ready( + &transaction, + delta.as_ref(), + refs, + visibility_index, + )?) } fn apply_capsule_control_visibility_index( capsule: &CapsuleControl, index: &mut crab_metadata::git_visibility::GitVisibilityIndex, ) -> Result<()> { - apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta()?, index) -} - -fn apply_transaction_visibility_index( - transaction: &crab_metadata::capsule_protocol::CapsuleTransaction, - delta: Option<&crab_metadata::capsule_protocol::CapsuleVisibilityDelta>, - index: &mut crab_metadata::git_visibility::GitVisibilityIndex, -) -> Result<()> { - let evidence = delta.map(|delta| delta.edits()); - let mut matched = 0; - for edit in transaction.edits() { - let Some(new_oid) = edit.new_oid() else { - if evidence - .and_then(|rows| rows.get(edit.ref_name())) - .is_some() - { - return Err(corrupt_path( - "capsule Git visibility", - "deleted ref has visibility evidence", - )); - } - index.remove_ref(edit.ref_name()); - continue; - }; - let visibility = evidence - .and_then(|rows| rows.get(edit.ref_name())) - .ok_or_else(|| { - corrupt_path( - "capsule Git visibility", - "live ref edit has no visibility evidence", - ) - })?; - matched += 1; - if visibility.new_oid != new_oid { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match its ref edit", - )); - } - if let Some(expected_old) = edit.expected_old() - && visibility.old_oid.as_deref() != Some(expected_old) - { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence does not match the expected old ref", - )); - } - index.apply_ref_edit(edit.ref_name().to_owned(), visibility)?; - } - if evidence.is_some_and(|rows| rows.len() != matched) { - return Err(corrupt_path( - "capsule Git visibility", - "visibility evidence contains an uncommitted ref", - )); - } + apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta()?, index)?; Ok(()) } @@ -5605,7 +5183,8 @@ fn apply_capsule_visibility_index( ) -> Result<()> { let transaction = capsule.transaction()?; let delta = capsule.visibility_delta()?; - apply_transaction_visibility_index(&transaction, delta.as_ref(), index) + apply_transaction_visibility_index(&transaction, delta.as_ref(), index)?; + Ok(()) } fn apply_capsule_refs( @@ -7670,7 +7249,13 @@ mod tests { assert_eq!( ( capsule_is_ready(&capsule, &refs, &index).is_err(), - control_is_ready(&controls[0], &refs, &index).is_err(), + visibility_is_ready( + controls[0].transaction(), + controls[0].visibility_delta().unwrap(), + &refs, + &index + ) + .is_err(), apply_capsule_visibility_index(&capsule, &mut index.clone()).is_err(), apply_capsule_control_visibility_index(&controls[0], &mut index.clone()) .is_err(), diff --git a/crates/crab-read/src/capsule_protocol/visibility_tests.rs b/crates/crab-read/src/capsule_protocol/visibility_tests.rs new file mode 100644 index 000000000..9b11ca598 --- /dev/null +++ b/crates/crab-read/src/capsule_protocol/visibility_tests.rs @@ -0,0 +1,106 @@ +use super::*; +use crab_metadata::capsule_protocol::{ + CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, CapsuleVisibilityDelta, +}; +use crab_metadata::git_visibility::GitVisibilityEdit; +use object_store::memory::InMemory; + +#[tokio::test] +async fn initial_replacement_opens_without_materializing_its_oid_set() { + use crab_metadata::capsule_protocol::{RepositoryRoot, RootRecord}; + use std::sync::atomic::{AtomicU64, Ordering}; + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = + Store::new(Arc::new(InMemory::new())).with_read_byte_observer(Arc::new(move |size| { + observed.fetch_add(size, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store, "repositories/large-root".into()); + let initial = + RootRecord::encode(RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap()) + .unwrap(); + crab_metadata::capsule_protocol::create_root(&layout, initial.clone()) + .await + .unwrap(); + let objects = (0u32..10000) + .map(|index| format!("{index:040x}")) + .collect::>(); + let tip = objects[9000].clone(); + let transaction = CapsuleTransaction::new( + initial.digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(tip.clone()), + None, + )], + ) + .unwrap(); + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/main".into(), + GitVisibilityEdit::from_replacement_objects(None, tip.clone(), objects), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let next = RootRecord::encode( + initial + .root() + .advance( + initial.digest(), + BTreeMap::from([("refs/heads/main".into(), tip)]), + BTreeMap::new(), + vec![pointer], + transaction.id().unwrap().as_str(), + ) + .unwrap(), + ) + .unwrap(); + layout + .store() + .put_overwrite(&layout.capsule_root_path(), next.bytes().clone()) + .await + .unwrap(); + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + bytes.store(0, Ordering::Relaxed); + let view = open_git_view_from_root( + &layout, + root, + CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 1024 * 1024, + }, + ) + .await + .unwrap(); + assert!( + bytes.load(Ordering::Relaxed) < 64 * 1024 + && view.capsule_controls[0].visibility_delta().is_err() + ); +} From 52a2f71a9ef5b4da08e7886acf43653d5e0e4612 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 23:28:50 -0700 Subject: [PATCH 061/127] fix(read): retain control corruption errors Preserve read-side corruption classification when shared metadata code detects conflicting captured controls. Keep transport errors as metadata sources. --- crates/crab-read/src/capsule_protocol.rs | 88 ++++++++++++++---------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 17941f462..1272dd352 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -4077,7 +4077,8 @@ pub async fn open_ref_view_from_root( snapshot: crab_metadata::capsule_protocol::RootSnapshot, ) -> Result { let (heads, active) = capture_ref_heads(router, snapshot.record().root()).await?; - let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; + let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active) + .map_err(metadata_control_error)?; Ok(CapsuleRefView { root: snapshot, refs: visible.refs, @@ -4096,7 +4097,8 @@ pub async fn read_activity_from_root( snapshot: &crab_metadata::capsule_protocol::RootSnapshot, ) -> Result { let (heads, active) = capture_ref_heads(router, snapshot.record().root()).await?; - let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; + let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active) + .map_err(metadata_control_error)?; let capsule_count = visible.pointers.iter().try_fold(0_u64, |total, pointer| { total .checked_add(u64::from(pointer.capsule_count())) @@ -4137,7 +4139,8 @@ pub async fn open_ref_view_from_root_for_refs( ) -> Result { let (heads, active) = capture_selected_ref_heads(router, snapshot.record().root(), ref_names).await?; - let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; + let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active) + .map_err(metadata_control_error)?; Ok(CapsuleRefView { root: snapshot, refs: visible @@ -4191,7 +4194,8 @@ pub async fn open_ref_view_from_root_for_push( )); } } - let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; + let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active) + .map_err(metadata_control_error)?; let push_ref_head_bases = ref_names .iter() .cloned() @@ -4251,7 +4255,9 @@ pub async fn open_view_from_root_for_repack( ) -> Result { let root = snapshot.record().root(); let (heads, active) = capture_ref_heads(router, root).await?; - let pointers = materialize_visible_ref_heads(root, &heads, &active)?.pointers; + let pointers = materialize_visible_ref_heads(root, &heads, &active) + .map_err(metadata_control_error)? + .pointers; let frontier_bytes = pointers.iter().try_fold(0_u64, |total, pointer| { total .checked_add(pointer.size()) @@ -4486,7 +4492,8 @@ async fn assemble_view( } let mut refs = snapshot.record().root().refs().clone(); let mut peeled_refs = snapshot.record().root().peeled_refs().clone(); - let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; + let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active) + .map_err(metadata_control_error)?; let expected_refs = visible.refs; let expected_peeled = visible.peeled_refs; let visible_ref_transactions = visible.transactions; @@ -4699,7 +4706,8 @@ async fn assemble_layered_control_view( let footer_only = matches!(read, ControlRead::Fetch); let mut refs = snapshot.record().root().refs().clone(); let mut peeled_refs = snapshot.record().root().peeled_refs().clone(); - let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active)?; + let visible = materialize_visible_ref_heads(snapshot.record().root(), &heads, &active) + .map_err(metadata_control_error)?; let expected_refs = visible.refs; let expected_peeled = visible.peeled_refs; let visible_ref_transactions = visible.transactions; @@ -4836,7 +4844,8 @@ async fn assemble_layered_control_view( capsule_run_sources.push(source); } } - let selected = select_ref_control_frontiers(snapshot.record().root(), &ref_frontiers, &runs)?; + let selected = select_ref_control_frontiers(snapshot.record().root(), &ref_frontiers, &runs) + .map_err(metadata_control_error)?; let mut root_capsules = selected.root_controls; let ref_capsule_counts = selected.counts; if footer_only { @@ -4864,7 +4873,8 @@ async fn assemble_layered_control_view( }); } let journal_capsules = - select_ref_journal_controls(snapshot.record().root(), &selected.required, &runs)?; + select_ref_journal_controls(snapshot.record().root(), &selected.required, &runs) + .map_err(metadata_control_error)?; let transaction_predecessors = selected.predecessors; let checkpoint_visibility = if matches!(read, ControlRead::Git) { checkpoint @@ -4881,7 +4891,8 @@ async fn assemble_layered_control_view( let initial = match &checkpoint_visibility { Some(reader) => { CapturedVisibility::from_checkpoint(router.store().clone(), reader.clone(), names) - .await? + .await + .map_err(metadata_control_error)? } None => CapturedVisibility::new( router.store().clone(), @@ -4894,7 +4905,10 @@ async fn assemble_layered_control_view( VisibilityReconciliation::Complete(visibility_index_for_checkpoint(checkpoint.as_ref())?) }; for capsule in &root_capsules { - visibility.apply(capsule).await?; + visibility + .apply(capsule) + .await + .map_err(metadata_control_error)?; } let ordered = order_ref_controls( snapshot.record().root().refs(), @@ -4903,9 +4917,10 @@ async fn assemble_layered_control_view( &transaction_predecessors, journal_capsules, ) - .await?; + .await + .map_err(metadata_control_error)?; for capsule in &ordered { - apply_control_refs(capsule, &mut refs, &mut peeled_refs)?; + apply_control_refs(capsule, &mut refs, &mut peeled_refs).map_err(metadata_control_error)?; } if refs != expected_refs || peeled_refs != expected_peeled { return Err(corrupt_path( @@ -5161,20 +5176,16 @@ fn capsule_is_ready( ) -> Result { let transaction = capsule.transaction()?; let delta = capsule.visibility_delta()?; - Ok(visibility_is_ready( - &transaction, - delta.as_ref(), - refs, - visibility_index, - )?) + visibility_is_ready(&transaction, delta.as_ref(), refs, visibility_index) + .map_err(metadata_control_error) } fn apply_capsule_control_visibility_index( capsule: &CapsuleControl, index: &mut crab_metadata::git_visibility::GitVisibilityIndex, ) -> Result<()> { - apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta()?, index)?; - Ok(()) + apply_transaction_visibility_index(capsule.transaction(), capsule.visibility_delta()?, index) + .map_err(metadata_control_error) } fn apply_capsule_visibility_index( @@ -5183,8 +5194,8 @@ fn apply_capsule_visibility_index( ) -> Result<()> { let transaction = capsule.transaction()?; let delta = capsule.visibility_delta()?; - apply_transaction_visibility_index(&transaction, delta.as_ref(), index)?; - Ok(()) + apply_transaction_visibility_index(&transaction, delta.as_ref(), index) + .map_err(metadata_control_error) } fn apply_capsule_refs( @@ -5282,28 +5293,31 @@ fn admit_frontier(pointers: &[CapsulePointer], limits: CapsuleReadLimits) -> Res Ok(()) } -async fn load_run(router: &StoreLayout, pointer: &CapsulePointer) -> Result { - match crab_metadata::capsule_protocol::load_capsule_run(router, pointer).await { - Ok(run) => Ok(run), - // Read callers distinguish malformed committed data from transport errors. - Err(crab_metadata::error::MetadataError::CorruptObject { path, reason }) => { - Err(ReadError::CorruptObject { path, reason }) +// Read consumers classify corrupt captured state separately from metadata I/O. +// Keep that boundary when shared metadata code detects the same invalid control. +fn metadata_control_error(error: crab_metadata::error::MetadataError) -> ReadError { + match error { + crab_metadata::error::MetadataError::CorruptObject { path, reason } => { + ReadError::CorruptObject { path, reason } } - Err(error) => Err(error.into()), + crab_metadata::error::MetadataError::Internal(reason) => ReadError::internal(reason), + error => error.into(), } } +async fn load_run(router: &StoreLayout, pointer: &CapsulePointer) -> Result { + crab_metadata::capsule_protocol::load_capsule_run(router, pointer) + .await + .map_err(metadata_control_error) +} + async fn load_run_control( router: &StoreLayout, pointer: &CapsulePointer, ) -> Result<(CapsuleRunControl, Vec)> { - match crab_metadata::capsule_protocol::load_capsule_run_control(router, pointer).await { - Ok(value) => Ok(value), - Err(crab_metadata::error::MetadataError::CorruptObject { path, reason }) => { - Err(ReadError::CorruptObject { path, reason }) - } - Err(error) => Err(error.into()), - } + crab_metadata::capsule_protocol::load_capsule_run_control(router, pointer) + .await + .map_err(metadata_control_error) } fn corrupt(path: &object_store::path::Path, reason: impl Into) -> ReadError { From 801fc69c3e73ad14569ae6a582b2d824edd30683 Mon Sep 17 00:00:00 2001 From: forhappy Date: Wed, 14 Oct 2026 23:56:21 -0700 Subject: [PATCH 062/127] test(remote): measure current catalog sessions Measure opening before cold and warm selected queries. Compare duplicate results with full inspection only after closing the session. --- crates/crab-remote/examples/qualify_remote.rs | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 07142244b..56661eeb9 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -36,13 +36,14 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|ref-point|resolve-oid|blame-history|blame-history-uncached]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|catalog-current|ref-point|resolve-oid|blame-history|blame-history-uncached]", ) .into()), }; if selection != "all" && selection != "build-indexes" && selection != "catalog" + && selection != "catalog-current" && selection != "checkpoint" && selection != "graph-pages" && selection != "graph-ancestry" @@ -101,6 +102,53 @@ async fn main() -> Result<()> { "{}", json!({"kind":"result", "operation":"ref-point", "value":previous}) ); + } else if selection == "catalog-current" { + let hash = crab_xet::hash::MerkleHash::from_hex(std::str::from_utf8(path.as_bytes())?)?; + let session = counters + .measure("open", "cold", async { + Ok( + crab_metadata::file_index_lookup::FileIndexLookupSession::open_for_storage( + layout.store(), + layout.repo_prefix(), + ) + .await?, + ) + }) + .await?; + let result: Result<_> = async { + let mut previous = None; + for cache in ["cold", "warm"] { + let selected = counters + .measure("catalog-current", cache, async { + Ok(session.lookup_batch(&[hash, hash]).await?) + }) + .await?; + if previous.as_ref().is_some_and(|value| value != &selected) { + return Err(io::Error::other("cold/warm current catalog results differ").into()); + } + previous = Some(selected); + } + Ok(previous) + } + .await; + let closed = session.close().await; + let selected = result?; + closed?; + // Run the bulk maintenance oracle only after measured reads and close: + // it must not populate page caches before the cold selected query. + let full = crab_metadata::capsule_protocol::load_pointer_catalog(&layout).await?; + let expected = full + .files() + .get(&hash.hex()) + .ok_or_else(|| io::Error::other("requested catalog file is absent"))?; + let expected = crab_xet::hash::MerkleHash::from_hex(expected.shard_hash())?; + if selected.as_ref() != Some(&vec![Some(expected), Some(expected)]) { + return Err(io::Error::other("current catalog batch differs from full catalog").into()); + } + println!( + "{}", + json!({"kind":"result", "operation":"catalog-current", "files":2, "shard":expected.hex()}) + ); } else if selection == "catalog" { let view = counters .measure("open", "cold", async { From 20b499925d9cec639df4358648fcd96073d3c9fb Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 00:52:57 -0700 Subject: [PATCH 063/127] perf(metadata): cache verified catalog bindings --- crates/crab-metadata/README.md | 6 + .../capsule_protocol/pointer/paged/reader.rs | 35 ++++- crates/crab-metadata/src/file_index_lookup.rs | 107 +++++++++++----- .../src/file_index_lookup/catalog_tests.rs | 120 ++++++++++++++++-- .../src/file_index_lookup/shared_tests.rs | 8 +- 5 files changed, 234 insertions(+), 42 deletions(-) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 79984ed78..395091efe 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -72,6 +72,12 @@ accepts captured lookup roots and resolves requested files with their authentica shard/xorb closure. Point and batch calls share the page cache and retain the session read guard through I/O, so close waits for active reads. Missing catalog pages remain errors rather than reopening the mutable file-index database. +A session caches at most 4,096 verified catalog bindings or authenticated misses, +using the same result-cache owner as canonical shard lookup. Cold queries still +validate the full selected dependency closure before publishing cache entries; +errors are never cached as absence. Cached inputs retain the original admission +charge. Catalog cache locks cover memory work only, so unrelated warm lookups can finish +while a cold origin read is paused. Large cold batches do not retain their results. The current-state opener also captures authenticated catalog roots and leaves catalog bodies cold. Explicitly pinned callers pass their existing capture, while current-state sessions bind once to the first root they open. Catalog and Git diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index 14449fba8..f9e3bf451 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -10,6 +10,8 @@ use std::sync::Arc; #[cfg(test)] mod tests; +const QUERY_KEY_BYTES: u64 = 96; + /// Snapshot-bound catalog queries with one cache and aggregate byte budget. /// /// The owner retains the captured objects against GC. Missing keys return an @@ -236,6 +238,17 @@ impl PointerCatalogLookup { self.dependencies(store, hashes, &[], &[]).await } + #[cfg(feature = "file-index-reader")] + pub(crate) async fn files_for_lookup( + &self, + store: &Store, + hashes: &[String], + cached_files: usize, + ) -> Result { + self.dependencies_with_admission(store, hashes, &[], &[], cached_files) + .await + } + /// Select files and explicit shard/xorb identities with their complete closure. /// /// All keys and dependency pages share one query budget. Absent requested @@ -246,11 +259,31 @@ impl PointerCatalogLookup { files: &[String], shards: &[String], xorbs: &[String], + ) -> Result { + self.dependencies_with_admission(store, files, shards, xorbs, 0) + .await + } + + async fn dependencies_with_admission( + &self, + store: &Store, + files: &[String], + shards: &[String], + xorbs: &[String], + cached_files: usize, ) -> Result { let mut work = self.work(); + // Cached bindings skip page decoding, not the aggregate input budget. + admit( + &mut work, + (cached_files as u64).saturating_mul(QUERY_KEY_BYTES), + )?; let hashes = requested(files, &mut work)?; let mut shards = requested(shards, &mut work)?; let mut xorbs = requested(xorbs, &mut work)?; + if hashes.is_empty() && shards.is_empty() && xorbs.is_empty() { + return Ok(PointerCatalog::new()); + } let root = self .checkpoint .as_ref() @@ -425,7 +458,7 @@ fn admit(work: &mut PageWork, bytes: u64) -> Result<()> { } fn requested(hashes: &[String], work: &mut PageWork) -> Result> { - admit(work, (hashes.len() as u64).saturating_mul(96))?; + admit(work, (hashes.len() as u64).saturating_mul(QUERY_KEY_BYTES))?; for value in hashes { hash(value)?; } diff --git a/crates/crab-metadata/src/file_index_lookup.rs b/crates/crab-metadata/src/file_index_lookup.rs index c8f3f4c9b..a9a433b24 100644 --- a/crates/crab-metadata/src/file_index_lookup.rs +++ b/crates/crab-metadata/src/file_index_lookup.rs @@ -22,6 +22,7 @@ use crab_xet::xorb::format::MerkleHash; const DB_LABEL: &str = "file_index_db"; const GET_BATCH_CONCURRENCY: usize = 256; const SHARD_SEARCH_CONCURRENCY: usize = 4; +const MAX_VERIFIED_CATALOG_FILES: usize = 4096; static SHARD_SCANS: LazyLock> = LazyLock::new(|| Arc::new(tokio::sync::Semaphore::new(SHARD_SEARCH_CONCURRENCY))); @@ -145,7 +146,7 @@ impl CommittedShardAnchor { } #[derive(Debug, Default)] -struct ManifestFallbackCache { +struct FileLookupCache { found: HashMap, missing: HashSet, shard_visits: usize, @@ -230,7 +231,7 @@ pub struct FileIndexLookupSession { storage: crab_storage::Store, router: crab_storage::StoreLayout, parsers: tokio_util::task::TaskTracker, - manifest_fallback: tokio::sync::Mutex, + resolved_files: tokio::sync::Mutex, limits: FileIndexLookupLimits, } @@ -250,7 +251,7 @@ impl FileIndexLookupSession { storage: router.store().clone(), router, parsers: tokio_util::task::TaskTracker::new(), - manifest_fallback: tokio::sync::Mutex::new(ManifestFallbackCache::default()), + resolved_files: tokio::sync::Mutex::new(FileLookupCache::default()), limits: FileIndexLookupLimits::CURRENT_STATE, }) } @@ -313,7 +314,7 @@ impl FileIndexLookupSession { storage: router.store().clone(), router: router.clone(), parsers: tokio_util::task::TaskTracker::new(), - manifest_fallback: tokio::sync::Mutex::new(ManifestFallbackCache::default()), + resolved_files: tokio::sync::Mutex::new(FileLookupCache::default()), limits, }) } @@ -373,7 +374,7 @@ impl FileIndexLookupSession { storage: router.store().clone(), router: router.clone(), parsers: tokio_util::task::TaskTracker::new(), - manifest_fallback: tokio::sync::Mutex::new(ManifestFallbackCache::default()), + resolved_files: tokio::sync::Mutex::new(FileLookupCache::default()), limits, }) } @@ -398,7 +399,7 @@ impl FileIndexLookupSession { storage, router, parsers: tokio_util::task::TaskTracker::new(), - manifest_fallback: tokio::sync::Mutex::new(ManifestFallbackCache::default()), + resolved_files: tokio::sync::Mutex::new(FileLookupCache::default()), limits: FileIndexLookupLimits::CURRENT_STATE, }); } @@ -422,7 +423,7 @@ impl FileIndexLookupSession { storage, router, parsers: tokio_util::task::TaskTracker::new(), - manifest_fallback: tokio::sync::Mutex::new(ManifestFallbackCache::default()), + resolved_files: tokio::sync::Mutex::new(FileLookupCache::default()), limits: FileIndexLookupLimits::CURRENT_STATE, }; if use_acceleration { @@ -489,28 +490,7 @@ impl FileIndexLookupSession { return Ok(Vec::new()); } if let Some(lookup) = &self.catalog { - let hashes = file_hashes.iter().map(MerkleHash::hex).collect::>(); - let selected = lookup.files(&self.storage, &hashes).await?; - return hashes - .iter() - .map(|hash| { - selected - .files() - .get(hash) - .map(|entry| { - MerkleHash::from_hex(entry.shard_hash()).map_err(|source| { - MetadataError::CorruptObject { - path: "capsule-protocol pointer catalog".to_owned(), - reason: format!( - "invalid shard hash {}: {source}", - entry.shard_hash() - ), - } - }) - }) - .transpose() - }) - .collect(); + return self.catalog_lookup_batch(lookup, file_hashes).await; } let records = self.lookup_committed_records_batch(file_hashes).await?; let mut out = records @@ -533,6 +513,71 @@ impl FileIndexLookupSession { Ok(out) } + async fn catalog_lookup_batch( + &self, + lookup: &crate::capsule_protocol::PointerCatalogLookup, + hashes: &[MerkleHash], + ) -> Result>> { + let mut output = vec![None; hashes.len()]; + let missing = { + let cache = self.resolved_files.lock().await; + hashes + .iter() + .enumerate() + .filter_map(|(index, hash)| { + if let Some(shard) = cache.found.get(hash) { + output[index] = Some(*shard); + return None; + } + (!cache.missing.contains(hash)).then_some((index, *hash)) + }) + .collect::>() + }; + let keys = missing + .iter() + .map(|(_, hash)| hash.hex()) + .collect::>(); + let cached_files = hashes.len() - missing.len(); + // Verified bindings belong to this immutable capture. Keep only the + // small result, but admit every input and validate cold dependency closure. + let selected = lookup + .files_for_lookup(&self.storage, &keys, cached_files) + .await?; + for (index, hash) in &missing { + output[*index] = selected + .files() + .get(&hash.hex()) + .map(|entry| { + MerkleHash::from_hex(entry.shard_hash()).map_err(|source| { + MetadataError::CorruptObject { + path: "capsule-protocol pointer catalog".to_owned(), + reason: format!("invalid shard hash {}: {source}", entry.shard_hash()), + } + }) + }) + .transpose()?; + } + if !missing.is_empty() && missing.len() <= MAX_VERIFIED_CATALOG_FILES { + let mut cache = self.resolved_files.lock().await; + let cached = cache.found.len().saturating_add(cache.missing.len()); + if cached.saturating_add(missing.len()) > MAX_VERIFIED_CATALOG_FILES { + cache.found.clear(); + cache.missing.clear(); + } + for (index, hash) in missing { + match output[index] { + Some(shard) => { + cache.found.insert(hash, shard); + } + None => { + cache.missing.insert(hash); + } + } + } + } + Ok(output) + } + /// Return only generation-pinned acceleration hits, without invoking the /// manifest-scoped correctness fallback. pub async fn lookup_committed_records_batch( @@ -581,7 +626,7 @@ impl FileIndexLookupSession { let Some(anchor) = self.anchor.as_ref() else { return Ok(vec![None; file_hashes.len()]); }; - let mut cache = self.manifest_fallback.lock().await; + let mut cache = self.resolved_files.lock().await; let unresolved = file_hashes .iter() .filter(|file_hash| { @@ -766,7 +811,7 @@ impl LookupSource { storage: router.store().clone(), router: router.clone(), parsers: tokio_util::task::TaskTracker::new(), - manifest_fallback: tokio::sync::Mutex::new(ManifestFallbackCache::default()), + resolved_files: tokio::sync::Mutex::new(FileLookupCache::default()), limits: FileIndexLookupLimits::CURRENT_STATE, }), Self::Current { diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs index f656df3f9..95b847f4f 100644 --- a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -12,6 +12,7 @@ use std::sync::atomic::AtomicUsize; pub(super) async fn capture( layout: &crab_storage::StoreLayout, catalog: PointerCatalog, + maximum: u64, ) -> (PointerCatalogLookup, object_store::path::Path, RootRecord) { let pack = CapsuleGitPack::new( Bytes::from_static(b"pack"), @@ -140,7 +141,7 @@ pub(super) async fn capture( ) .unwrap(); ( - control.pointer_catalog_lookup(layout, 512 * 1024).unwrap(), + control.pointer_catalog_lookup(layout, maximum).unwrap(), path, root, ) @@ -185,7 +186,7 @@ async fn pinned_batches_preserve_order_misses_and_page_bounds_as_history_grows() observed.fetch_add(1, Ordering::SeqCst); })); let layout = crab_storage::StoreLayout::new(storage, "pinned-lookup".to_owned()); - let (captured, _, _) = capture(&layout, catalog(count)).await; + let (captured, _, _) = capture(&layout, catalog(count), 512 * 1024).await; let lookup = SharedFileIndexLookup::for_pointer_catalog(layout.clone(), captured); assert_eq!( reads.load(Ordering::SeqCst), @@ -193,7 +194,7 @@ async fn pinned_batches_preserve_order_misses_and_page_bounds_as_history_grows() "construction must leave pages cold" ); // A later authenticated catalog must not change an existing capture. - capture(&layout, catalog(count + 1)).await; + capture(&layout, catalog(count + 1), 512 * 1024).await; let file = super::tests::hash_from_seed(42); let later = super::tests::hash_from_seed(count as u64); let shard = super::tests::hash_from_seed(1_000_042); @@ -224,16 +225,30 @@ async fn missing_captured_pages_are_errors_even_when_a_new_catalog_exists() { crab_storage::Store::new(Arc::new(InMemory::new())), "missing-page".to_owned(), ); - let (captured, path, _) = capture(&layout, catalog(100)).await; - capture(&layout, catalog(101)).await; + let (captured, path, _) = capture(&layout, catalog(100), 512 * 1024).await; + let (original, _) = layout + .store() + .get_with_etag_bounded(&path, 512 * 1024) + .await + .unwrap(); + capture(&layout, catalog(101), 512 * 1024).await; layout.store().delete(&path).await.unwrap(); - let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); + let lookup = SharedFileIndexLookup::for_pointer_catalog(layout.clone(), captured); assert!( lookup .lookup(&super::tests::hash_from_seed(42)) .await .is_err() ); + layout.store().put_overwrite(&path, original).await.unwrap(); + assert_eq!( + lookup + .lookup(&super::tests::hash_from_seed(42)) + .await + .unwrap(), + Some(super::tests::hash_from_seed(1_000_042)), + "failed closure reads must not cache absence" + ); lookup.close().await.unwrap(); } @@ -253,7 +268,7 @@ async fn current_state_opener_keeps_catalog_pages_selected_and_pins_its_first_ro observed_bytes.fetch_add(size, Ordering::SeqCst); })); let layout = crab_storage::StoreLayout::new(storage.clone(), "current-catalog".to_owned()); - let (_, _, root) = capture(&layout, catalog(count)).await; + let (_, _, root) = capture(&layout, catalog(count), 512 * 1024).await; assert_eq!(root.root().generation(), 1); let lookup = SharedFileIndexLookup::new_for_storage(&storage, layout.repo_prefix()); assert_eq!(reads.load(Ordering::SeqCst), 0); @@ -269,7 +284,7 @@ async fn current_state_opener_keeps_catalog_pages_selected_and_pins_its_first_ro reads.load(Ordering::SeqCst), bytes.load(Ordering::SeqCst) ); - let (_, _, later_root) = capture(&layout, catalog(count + 1)).await; + let (_, _, later_root) = capture(&layout, catalog(count + 1), 512 * 1024).await; layout .store() .put_overwrite(&layout.capsule_root_path(), later_root.bytes().clone()) @@ -286,3 +301,92 @@ async fn current_state_opener_keeps_catalog_pages_selected_and_pins_its_first_ro latest.close().await.unwrap(); } } + +#[tokio::test] +async fn repeated_large_recipe_binding_queries_reuse_verified_results() { + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let storage = crab_storage::Store::new(Arc::new(InMemory::new())).with_read_request_observer( + Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + }), + ); + let layout = crab_storage::StoreLayout::new(storage, "large-recipe-binding".to_owned()); + let file = super::tests::hash_from_seed(42); + let shard = super::tests::hash_from_seed(1_000_042); + let mut catalog = PointerCatalog::new(); + let mut xorbs = Vec::new(); + for index in 0..160_u64 { + let xorb = super::tests::hash_from_seed(2_000_000 + index).hex(); + let chunks = (0..1024) + .map(|offset| { + XorbChunkEntry::new( + super::tests::hash_from_seed(3_000_000 + index * 1024 + offset).hex(), + 64 * 1024, + ) + }) + .collect(); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new(64 * 1024 * 1024, "e".repeat(64), chunks), + ) + .unwrap(); + xorbs.push(xorb); + } + catalog + .insert_shard(shard.hex(), ShardCatalogEntry::new(80, xorbs)) + .unwrap(); + // This is metadata for a 10 GiB recipe, not a payload reconstruction fixture. + // Its verified closure outgrows the ordinary decoded-page cache. + catalog + .insert_file( + file.hex(), + FileCatalogEntry::new(10 * 1024 * 1024 * 1024, shard.hex()), + ) + .unwrap(); + let (captured, _, _) = capture(&layout, catalog, 16 * 1024 * 1024).await; + let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); + let cold = lookup.lookup(&file).await; + let cold_reads = reads.load(Ordering::SeqCst); + let warm = lookup.lookup_batch(&[file, file]).await; + let warm_reads = reads.load(Ordering::SeqCst); + lookup.close().await.unwrap(); + assert_eq!( + (cold.unwrap(), warm.unwrap()), + (Some(shard), vec![Some(shard), Some(shard)]) + ); + assert!(cold_reads > 0); + assert_eq!( + warm_reads, cold_reads, + "verified immutable bindings must not re-read the large recipe" + ); +} + +#[tokio::test] +async fn cached_binding_batches_keep_the_original_input_admission() { + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let storage = crab_storage::Store::new(Arc::new(InMemory::new())).with_read_request_observer( + Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + }), + ); + let layout = crab_storage::StoreLayout::new(storage, "cached-admission".to_owned()); + let file = super::tests::hash_from_seed(42); + let (captured, _, _) = capture(&layout, catalog(100), 512 * 1024).await; + let queries = vec![file; 10_000]; + let keys = queries.iter().map(MerkleHash::hex).collect::>(); + assert!(captured.files(layout.store(), &keys).await.is_err()); + let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); + lookup.lookup(&file).await.unwrap().unwrap(); + let cold_reads = reads.load(Ordering::SeqCst); + let result = lookup.lookup_batch(&queries).await; + lookup.close().await.unwrap(); + assert!(matches!(result, Err(MetadataError::CorruptObject { .. }))); + assert_eq!( + reads.load(Ordering::SeqCst), + cold_reads, + "cached input rejection must precede I/O" + ); +} diff --git a/crates/crab-metadata/src/file_index_lookup/shared_tests.rs b/crates/crab-metadata/src/file_index_lookup/shared_tests.rs index 2a52edc02..2f7c5a1b2 100644 --- a/crates/crab-metadata/src/file_index_lookup/shared_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/shared_tests.rs @@ -352,8 +352,12 @@ async fn captured_catalog_close_waits_for_active_reads_and_abandonment_releases_ let inner: Arc = Arc::new(InMemory::new()); let storage = crab_storage::Store::new(inner.clone()); let router = crab_storage::StoreLayout::new(storage, "pinned/close".to_owned()); - let (catalog, path) = - super::catalog_tests::capture(&router, super::catalog_tests::catalog(10_000)).await; + let (catalog, path, _) = super::catalog_tests::capture( + &router, + super::catalog_tests::catalog(10_000), + 512 * 1024, + ) + .await; let store = Arc::new(PausedLookupStore { inner, shard: path, From 978720bd32d768fd308671c09298e8f816019ad6 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 01:20:28 -0700 Subject: [PATCH 064/127] fix(metadata): retain duplicate batch bindings --- crates/crab-metadata/README.md | 3 +- crates/crab-metadata/src/file_index_lookup.rs | 15 ++++++---- .../src/file_index_lookup/catalog_tests.rs | 29 +++++++++++++++++-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 395091efe..676c023e5 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -77,7 +77,8 @@ using the same result-cache owner as canonical shard lookup. Cold queries still validate the full selected dependency closure before publishing cache entries; errors are never cached as absence. Cached inputs retain the original admission charge. Catalog cache locks cover memory work only, so unrelated warm lookups can finish -while a cold origin read is paused. Large cold batches do not retain their results. +while a cold origin read is paused. Retention counts distinct file identities; +duplicate occurrences do not consume additional entries or prevent caching. The current-state opener also captures authenticated catalog roots and leaves catalog bodies cold. Explicitly pinned callers pass their existing capture, while current-state sessions bind once to the first root they open. Catalog and Git diff --git a/crates/crab-metadata/src/file_index_lookup.rs b/crates/crab-metadata/src/file_index_lookup.rs index a9a433b24..dbf710cc9 100644 --- a/crates/crab-metadata/src/file_index_lookup.rs +++ b/crates/crab-metadata/src/file_index_lookup.rs @@ -557,14 +557,17 @@ impl FileIndexLookupSession { }) .transpose()?; } - if !missing.is_empty() && missing.len() <= MAX_VERIFIED_CATALOG_FILES { + if !missing.is_empty() { let mut cache = self.resolved_files.lock().await; - let cached = cache.found.len().saturating_add(cache.missing.len()); - if cached.saturating_add(missing.len()) > MAX_VERIFIED_CATALOG_FILES { - cache.found.clear(); - cache.missing.clear(); - } for (index, hash) in missing { + if cache.found.contains_key(&hash) || cache.missing.contains(&hash) { + continue; + } + // Duplicate occurrences consume input budget, but retain one fact. + if cache.found.len() + cache.missing.len() == MAX_VERIFIED_CATALOG_FILES { + cache.found.clear(); + cache.missing.clear(); + } match output[index] { Some(shard) => { cache.found.insert(hash, shard); diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs index 95b847f4f..3c4efd60e 100644 --- a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -347,14 +347,18 @@ async fn repeated_large_recipe_binding_queries_reuse_verified_results() { .unwrap(); let (captured, _, _) = capture(&layout, catalog, 16 * 1024 * 1024).await; let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); - let cold = lookup.lookup(&file).await; + let duplicates = vec![file; super::MAX_VERIFIED_CATALOG_FILES + 1]; + let cold = lookup.lookup_batch(&duplicates).await; let cold_reads = reads.load(Ordering::SeqCst); let warm = lookup.lookup_batch(&[file, file]).await; let warm_reads = reads.load(Ordering::SeqCst); lookup.close().await.unwrap(); assert_eq!( (cold.unwrap(), warm.unwrap()), - (Some(shard), vec![Some(shard), Some(shard)]) + ( + vec![Some(shard); duplicates.len()], + vec![Some(shard), Some(shard)] + ) ); assert!(cold_reads > 0); assert_eq!( @@ -363,6 +367,27 @@ async fn repeated_large_recipe_binding_queries_reuse_verified_results() { ); } +#[tokio::test] +async fn binding_cache_stays_bounded_for_distinct_misses() { + let layout = crab_storage::StoreLayout::new( + crab_storage::Store::new(Arc::new(InMemory::new())), + "bounded-binding-cache".to_owned(), + ); + let (captured, _, _) = capture(&layout, PointerCatalog::new(), 16 * 1024 * 1024).await; + let lookup = SharedFileIndexLookup::for_pointer_catalog(layout, captured); + let files = (0..super::MAX_VERIFIED_CATALOG_FILES * 2) + .map(|index| super::tests::hash_from_seed(index as u64)) + .collect::>(); + lookup.lookup_batch(&files).await.unwrap(); + let retained = { + let session = lookup.session().await.unwrap(); + let cache = session.resolved_files.lock().await; + cache.found.len() + cache.missing.len() + }; + lookup.close().await.unwrap(); + assert!(retained <= super::MAX_VERIFIED_CATALOG_FILES); +} + #[tokio::test] async fn cached_binding_batches_keep_the_original_input_admission() { let reads = Arc::new(AtomicUsize::new(0)); From de3869df4b5b6e11f314d5a7163c537ecfea1a9c Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 02:43:36 -0700 Subject: [PATCH 065/127] fix(gc): retain published browse-index pages Keep published readiness dependencies after refs move. Resolve physical index retention from its own authenticated descriptor without treating old inventory as current authority. Retain regressions for ref movement and readiness replacement. Replacement lifetime remains an explicit draft release gate. --- crab/src/cmd/gc/capsule_cleanup_tests.rs | 2 - crab/src/cmd/gc/captured_index_tests.rs | 307 ++++++++++++++++++ crab/src/cmd/gc/mod.rs | 52 +-- crates/crab-metadata/README.md | 5 +- .../src/git_object_locator/paged/reader.rs | 57 +++- .../src/git_object_locator/paged/tests.rs | 46 +++ 6 files changed, 413 insertions(+), 56 deletions(-) create mode 100644 crab/src/cmd/gc/captured_index_tests.rs diff --git a/crab/src/cmd/gc/capsule_cleanup_tests.rs b/crab/src/cmd/gc/capsule_cleanup_tests.rs index 7d49556c4..a501550aa 100644 --- a/crab/src/cmd/gc/capsule_cleanup_tests.rs +++ b/crab/src/cmd/gc/capsule_cleanup_tests.rs @@ -226,7 +226,6 @@ async fn capsule_gc_does_not_report_revalidated_retained_objects_as_deleted() { .unwrap(); let mut manifest = crab_metadata::manifests::Manifest::default_for_repo("refs/heads/main"); manifest.seal_git_validation(); - let state_digest = "a".repeat(64); let outcome = sweep_capsule_objects( &GcArgs { force, @@ -236,7 +235,6 @@ async fn capsule_gc_does_not_report_revalidated_retained_objects_as_deleted() { &layout, &root, &[], - &state_digest, &manifest, &HashSet::new(), &CancellationToken::new(), diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs new file mode 100644 index 000000000..4fd56e2a1 --- /dev/null +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -0,0 +1,307 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +use bytes::Bytes; +use crab_metadata::capsule_protocol::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, + CapsuleTransaction, CapsuleVisibilityDelta, +}; +use crab_metadata::git_object_locator::{GitPackInventoryEntry, PagedGitObjectIndex}; +use crab_metadata::git_visibility::GitVisibilityEdit; +use crab_read::capsule_protocol::{CapsuleReadLimits, CapsuleRepositoryView}; +use crab_remote_git::{RemoteGitRuntime, RepositoryIdentity, RepositoryOptions}; +use crab_storage::StoreLayout; +use crab_xet::hash::MerkleHash; +use futures_util::{TryStreamExt, stream::BoxStream}; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, memory::InMemory, path::Path, +}; +use tokio_util::sync::CancellationToken; + +use super::{GcArgs, sweep_capsule_objects}; +use crate::storage::store::Store; + +const LIMIT: u64 = 8 * 1024 * 1024; + +// Immutable bodies can predate their first reader by years. Alter only their +// provider timestamps; native pack validation and conditional publication stay real. +#[derive(Debug)] +struct AgedObjects { + inner: InMemory, + paths: Mutex>, + modified: SystemTime, +} + +impl std::fmt::Display for AgedObjects { + fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + output.write_str("aged index fixture") + } +} + +#[async_trait::async_trait] +impl ObjectStore for AgedObjects { + async fn put_opts( + &self, + path: &Path, + payload: PutPayload, + options: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + + async fn put_multipart_opts( + &self, + path: &Path, + options: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + + async fn get_opts(&self, path: &Path, options: GetOptions) -> object_store::Result { + let mut result = self.inner.get_opts(path, options).await?; + if self.paths.lock().unwrap().contains(path) { + result.meta.last_modified = self.modified.into(); + } + Ok(result) + } + + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + let paths = self.paths.lock().unwrap().clone(); + let modified = self.modified; + Box::pin(self.inner.list(prefix).map_ok(move |mut meta| { + if paths.contains(&meta.location) { + meta.last_modified = modified.into(); + } + meta + })) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + let mut result = self.inner.list_with_delimiter(prefix).await?; + let paths = self.paths.lock().unwrap(); + for meta in &mut result.objects { + if paths.contains(&meta.location) { + meta.last_modified = self.modified.into(); + } + } + Ok(result) + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +async fn publish_blob(layout: &StoreLayout, name: &str) -> String { + let payload = name.as_bytes(); + let oid = crab_remote::objects::object_id(gix_object::Kind::Blob, payload) + .unwrap() + .to_string(); + let mut bytes = Vec::new(); + crab_git::pack_writer::write_pack( + &mut bytes, + std::iter::once(Ok((gix_object::Kind::Blob, payload.len() as u64, payload))), + LIMIT, + || false, + ) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("incoming.pack"); + std::fs::write(&path, &bytes).unwrap(); + let installed = crab_git::pack::install_pack_file_from_path( + &directory.path().join("indexed"), + &path, + blake3::hash(&bytes).to_hex().as_ref(), + LIMIT, + true, + ) + .unwrap(); + let checksum = gix_hash::ObjectId::from_hex(installed.git_sha1.as_bytes()).unwrap(); + let kinds = + crab_git::pack_locator::encode_pack_kind_metadata(checksum, &[gix_object::Kind::Blob]) + .unwrap(); + let pack = CapsuleGitPack::new( + bytes.into(), + std::fs::read(installed.idx_path).unwrap().into(), + std::fs::read(installed.rev_path).unwrap().into(), + Bytes::from(kinds), + installed.git_sha1, + 1, + ) + .unwrap(); + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + let name = format!("refs/tags/{name}"); + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new(&name, None, Some(oid.clone()), None)], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + name, + GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid.clone()]), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![pack], + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + )], + ) + .unwrap(); + crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + .await + .unwrap(); + oid +} + +async fn view(layout: &StoreLayout) -> CapsuleRepositoryView { + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + crab_read::capsule_protocol::open_git_view_from_root( + layout, + root, + CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap() +} + +async fn cold_capture_after_gc(replace_readiness: bool) -> crab_metadata::error::Result { + let backend = Arc::new(AgedObjects { + inner: InMemory::new(), + paths: Mutex::new(HashSet::new()), + modified: SystemTime::now() - Duration::from_secs(7200), + }); + let store = Store::new(backend.clone()); + let layout = StoreLayout::new(store.as_storage().clone(), "org/captured-gc".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let oid = publish_blob(&layout, "original").await; + let runtime = Arc::new(RemoteGitRuntime::default()); + let identity = RepositoryIdentity::new("local", "captured-gc", 1).unwrap(); + let cancel = CancellationToken::new(); + let indexes = crab_remote::browse_indexes::ensure( + &layout, + &identity, + runtime.clone(), + RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap() + .unwrap(); + let captured = view(&layout).await; + let snapshot = captured.git_snapshot().unwrap(); + let reader = PagedGitObjectIndex::load( + layout.store(), + &layout, + indexes.object_index_hash().unwrap(), + &captured.state_digest(), + &snapshot.manifest, + LIMIT, + ) + .await + .unwrap(); + // Age only this immutable body. The capture, ref change, readiness + // replacement and GC all happen now, within a fresh reader's lifetime. + backend + .paths + .lock() + .unwrap() + .insert(reader.body_path().clone()); + let inventory = snapshot + .journal + .packs + .iter() + .map(|pack| { + let id = MerkleHash::from_hex(&pack.pack_id).unwrap(); + ( + id, + GitPackInventoryEntry { + pack_id: id, + object_count: pack.object_count, + pack_size: pack.size, + }, + ) + }) + .collect::>(); + publish_blob(&layout, "later").await; + if replace_readiness { + let replacement = crab_remote::browse_indexes::ensure( + &layout, + &identity, + runtime.clone(), + RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap() + .unwrap(); + assert_ne!(replacement.object_index_hash(), indexes.object_index_hash()); + } + let current = view(&layout).await; + let current_snapshot = current.git_snapshot().unwrap(); + sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + current.root().root(), + current.capsule_run_pointers(), + ¤t_snapshot.manifest, + &HashSet::new(), + &cancel, + SystemTime::now(), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + let oid = gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap(); + let oid: [u8; 20] = oid.as_bytes().try_into().unwrap(); + let result = reader.lookup(layout.store(), &[oid], &inventory).await; + runtime.shutdown().await; + result.map(|entries| entries[0].is_some()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn gc_preserves_cold_captures_after_ref_movement() { + let result = cold_capture_after_gc(false).await; + assert!( + matches!(result, Ok(true)), + "cold capture lost after ref movement: {result:?}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn gc_preserves_cold_captures_after_readiness_replacement() { + let result = cold_capture_after_gc(true).await; + assert!( + matches!(result, Ok(true)), + "cold capture lost after readiness replacement: {result:?}" + ); +} diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index 00304d8d4..4077483ab 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -22,6 +22,8 @@ pub mod parallel_enum; #[cfg(test)] mod capsule_cleanup_tests; +#[cfg(test)] +mod captured_index_tests; use std::collections::HashSet; #[cfg(test)] @@ -3134,14 +3136,12 @@ async fn run_capsule_gc( ) .await?; let snapshot = view.git_snapshot()?; - let state_digest = view.state_digest().to_owned(); sweep_capsule_objects( args, store, &layout, fenced.record().root(), view.capsule_run_pointers(), - &state_digest, &snapshot.manifest, coordinator_protected_keys, cancel, @@ -3183,7 +3183,6 @@ async fn sweep_capsule_objects( layout: &crab_storage::StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, capsule_runs: &[crab_metadata::capsule_protocol::CapsulePointer], - state_digest: &str, manifest: &crab_metadata::manifests::Manifest, coordinator_protected_keys: &HashSet, cancel: &CancellationToken, @@ -3235,14 +3234,7 @@ async fn sweep_capsule_objects( } } reachable.extend(coordinator_protected_keys.iter().cloned()); - mark_derived_index_objects( - store.as_storage(), - layout, - state_digest, - manifest, - &mut reachable, - ) - .await?; + mark_derived_index_objects(store.as_storage(), layout, manifest, &mut reachable).await?; let capsule_prefix = layout.repo_path("v2/capsules/"); let checkpoint_prefix = layout.repo_path("v2/checkpoints/"); @@ -3384,13 +3376,12 @@ async fn sweep_capsule_objects( async fn mark_derived_index_objects( store: &crab_storage::Store, layout: &crab_storage::StoreLayout, - state_digest: &str, manifest: &crab_metadata::manifests::Manifest, reachable: &mut HashSet, ) -> Result<()> { - if let Some(indexes) = crab_metadata::capsule_protocol::load_browse_indexes(layout).await? - && indexes.state_digest() == state_digest - { + if let Some(indexes) = crab_metadata::capsule_protocol::load_browse_indexes(layout).await? { + // Ref movement makes readiness stale for new queries, but existing + // readers still own that published root and may not have loaded pages. let graph_hash = indexes.commit_graph_hash(); let graph_path = layout.bulk_manifest_path("commit-graph", graph_hash); let graph = crab_metadata::split_commit_graph::load_split_commit_graph_descriptor( @@ -3409,23 +3400,15 @@ async fn mark_derived_index_objects( ); if let Some(hash) = indexes.object_index_hash() { - let index = crab_metadata::git_object_locator::PagedGitObjectIndex::load( + let paths = crab_metadata::git_object_locator::PagedGitObjectIndex::retained_paths( store, layout, hash, - state_digest, - manifest, + indexes.state_digest(), crab_metadata::split_commit_graph::DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES, ) .await?; - // Every authenticated physical page lives in this immutable body. - // Retain it with readiness so pruning cannot strand future delta reads. - reachable.insert( - layout - .bulk_manifest_path("git-object-index", hash) - .to_string(), - ); - reachable.insert(index.body_path().to_string()); + reachable.extend(paths.into_iter().map(|path| path.to_string())); } if let Some(path_state_hash) = indexes.path_state_hash() { @@ -5517,7 +5500,6 @@ mod tests { .await .unwrap(); let snapshot = current_view.git_snapshot().unwrap(); - let state_digest = current_view.state_digest().to_owned(); assert_eq!( root.record().root().compacted_ref_transactions(), captured.visible_ref_transactions() @@ -5558,7 +5540,6 @@ mod tests { &layout, root.record().root(), &[], - &state_digest, &snapshot.manifest, &protected, &CancellationToken::new(), @@ -5586,7 +5567,6 @@ mod tests { &layout, root.record().root(), &[], - &state_digest, &snapshot.manifest, &protected, &CancellationToken::new(), @@ -5821,15 +5801,9 @@ mod tests { .unwrap(), }); let mut retained = HashSet::new(); - mark_derived_index_objects( - store.as_storage(), - &layout, - &state_digest, - &manifest, - &mut retained, - ) - .await - .unwrap(); + mark_derived_index_objects(store.as_storage(), &layout, &manifest, &mut retained) + .await + .unwrap(); assert!( retained.contains( &layout @@ -5957,7 +5931,6 @@ mod tests { &layout, root.record().root(), &[], - &state_digest, &manifest, &HashSet::new(), &CancellationToken::new(), @@ -5975,7 +5948,6 @@ mod tests { &layout, root.record().root(), &[], - &state_digest, &manifest, &HashSet::new(), &CancellationToken::new(), diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 676c023e5..4c835987e 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -118,7 +118,10 @@ requires complete verified pack coverage before publication; path completion retains that binding. Graph publication does not wait for path construction. The prior versions were never reachable from a release tag; maintenance rebuilds this derived record in place, without a legacy reader. Readers ignore stale -records; absent indexes remain a readiness state. Rebuilding an immutable index +records; absent indexes remain a readiness state. GC retains dependencies of the currently +published readiness record even when newer refs make it unusable for new reads. +Storage retention does not grant inventory membership or ref visibility. +Rebuilding an immutable index verifies the generated content identity and repairs corrupt stored bytes only with a conditional update against the observed version, followed by readback. This applies to graph/path-state/physical-index descriptors and bodies, not Git/Xet data. diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index c1f39af8d..f26164933 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -12,18 +12,7 @@ impl PagedGitObjectIndex { manifest: &Manifest, maximum: u64, ) -> Result { - validate_content_hash(hash, "object index descriptor", "Git object index")?; - let expected = hash_bytes(hash)?; - let bytes = store - .verify_bounded( - &layout.bulk_manifest_path("git-object-index", hash), - &expected, - maximum.min(MAX_DESCRIPTOR_BYTES), - ) - .await?; - let descriptor: Descriptor = serde_json::from_slice(&bytes) - .map_err(|source| crate::error::MetadataError::GitObjectIndexRecord { source })?; - descriptor.validate()?; + let (descriptor, descriptor_bytes) = Self::descriptor(store, layout, hash, maximum).await?; if descriptor.state_digest != state_digest || descriptor.generation != manifest.generation || descriptor.pack_index_hash != manifest.pack_index_hash @@ -34,7 +23,7 @@ impl PagedGitObjectIndex { let path = body_path(layout, &descriptor.body_hash); Ok(Self { descriptor, - descriptor_bytes: bytes.len() as u64, + descriptor_bytes, path, origin: Arc::clone(layout.store().inner()), global_prefix: layout.global_prefix().to_owned(), @@ -43,6 +32,48 @@ impl PagedGitObjectIndex { }) } + /// Resolve retained readiness to its authenticated descriptor and complete page body. + /// + /// The state digest must come from the retained root. This proves storage + /// dependencies for GC, not current inventory membership or ref visibility. + pub async fn retained_paths( + store: &Store, + layout: &StoreLayout, + hash: &str, + state_digest: &str, + maximum: u64, + ) -> Result<[object_store::path::Path; 2]> { + let (descriptor, _) = Self::descriptor(store, layout, hash, maximum).await?; + if descriptor.state_digest != state_digest { + return bad("retained object index does not match captured state"); + } + Ok([ + layout.bulk_manifest_path("git-object-index", hash), + body_path(layout, &descriptor.body_hash), + ]) + } + + async fn descriptor( + store: &Store, + layout: &StoreLayout, + hash: &str, + maximum: u64, + ) -> Result<(Descriptor, u64)> { + validate_content_hash(hash, "object index descriptor", "Git object index")?; + let expected = hash_bytes(hash)?; + let bytes = store + .verify_bounded( + &layout.bulk_manifest_path("git-object-index", hash), + &expected, + maximum.min(MAX_DESCRIPTOR_BYTES), + ) + .await?; + let descriptor: Descriptor = serde_json::from_slice(&bytes) + .map_err(|source| crate::error::MetadataError::GitObjectIndexRecord { source })?; + descriptor.validate()?; + Ok((descriptor, bytes.len() as u64)) + } + /// Check captured state and in-process origin before attaching lookup to a repository. #[must_use] pub fn matches_binding( diff --git a/crates/crab-metadata/src/git_object_locator/paged/tests.rs b/crates/crab-metadata/src/git_object_locator/paged/tests.rs index bef5084ef..a7bd74db0 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/tests.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/tests.rs @@ -246,6 +246,52 @@ async fn descriptors_cannot_cross_captured_git_states() { ); } +#[tokio::test] +async fn retained_paths_cover_the_complete_body_without_loading_pages() { + let (manifest, inventory, entries) = inputs(10_000); + let state = "c".repeat(64); + let write = PagedGitObjectIndexWrite::build(&state, &manifest, &inventory, &entries).unwrap(); + let reads = Arc::new(AtomicU64::new(0)); + let observed = reads.clone(); + let store = Store::new(Arc::new(InMemory::new())).with_read_byte_observer(Arc::new(move |n| { + observed.fetch_add(n, Ordering::Relaxed); + })); + let layout = StoreLayout::new(store.clone(), "retained-object-index".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + reads.store(0, Ordering::Relaxed); + let paths = PagedGitObjectIndex::retained_paths(&store, &layout, &hash, &state, 256 * 1024) + .await + .unwrap(); + assert_eq!( + (paths, reads.load(Ordering::Relaxed) <= MAX_DESCRIPTOR_BYTES), + ( + [ + layout.bulk_manifest_path("git-object-index", &hash), + body_path(&layout, &write.descriptor.body_hash) + ], + true + ) + ); +} + +#[tokio::test] +async fn retained_paths_require_the_captured_state_and_descriptor_budget() { + let (manifest, inventory, entries) = inputs(10); + let state = "c".repeat(64); + let write = PagedGitObjectIndexWrite::build(&state, &manifest, &inventory, &entries).unwrap(); + let store = Store::new(Arc::new(InMemory::new())); + let layout = StoreLayout::new(store.clone(), "retained-object-binding".into()); + let hash = write.upload(&store, &layout).await.unwrap(); + let foreign = "d".repeat(64); + for (state, maximum) in [(&foreign, 256 * 1024), (&state, 1)] { + assert!( + PagedGitObjectIndex::retained_paths(&store, &layout, &hash, state, maximum) + .await + .is_err() + ); + } +} + #[tokio::test] async fn object_aliases_prefer_full_encodings_and_keep_every_physical_offset() { let (mut manifest, mut inventory, mut entries) = inputs(1); From ffeb5839f27179ad8322acbc238e1b037305821a Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 03:22:58 -0700 Subject: [PATCH 066/127] fix(gc): retain replaced browse captures Publish a separate authenticated lifetime chain before readiness CAS. Retain original Git source ranges with cold index pages until explicit history pruning retires their generation. Prune lifetime roots only after history CAS commits, retry cleanup after partial success, and preserve current readiness. Keep foreground readiness reads unchanged. Refs #591 --- crab/src/cmd/gc/mod.rs | 41 ++- crab/src/cmd/history_recovery_v2.rs | 24 +- crates/crab-metadata/README.md | 17 + .../src/capsule_protocol/browse_retention.rs | 337 ++++++++++++++++++ .../browse_retention/tests.rs | 224 ++++++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 7 + crates/crab-remote/examples/qualify_remote.rs | 20 +- .../examples/support/captured_browse.rs | 67 ++++ crates/crab-remote/src/browse_indexes.rs | 16 +- crates/crab-storage/src/layout.rs | 12 + 10 files changed, 755 insertions(+), 10 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/browse_retention.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs create mode 100644 crates/crab-remote/examples/support/captured_browse.rs diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index 4077483ab..bcd60d9e0 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3243,6 +3243,7 @@ async fn sweep_capsule_objects( let path_state_prefix = layout.repo_path("metadata/path-state/"); let commit_graph_prefix = layout.repo_path("metadata/commit-graph/"); let object_index_prefix = layout.repo_path("metadata/git-object-index/"); + let browse_root_prefix = layout.repo_path("metadata/browse-roots/"); let manifests_prefix = layout.repo_path("manifests/"); let path_state_descriptor_prefix = layout.repo_path("manifests/path-state-"); let commit_graph_descriptor_prefix = layout.repo_path("manifests/commit-graph-"); @@ -3255,6 +3256,7 @@ async fn sweep_capsule_objects( path_state_objects, commit_graph_objects, object_index_objects, + browse_root_objects, manifests, ) = tokio::try_join!( store.list_prefix(&capsule_prefix), @@ -3264,12 +3266,14 @@ async fn sweep_capsule_objects( store.list_prefix(&path_state_prefix), store.list_prefix(&commit_graph_prefix), store.list_prefix(&object_index_prefix), + store.list_prefix(&browse_root_prefix), store.list_prefix(&manifests_prefix), )?; let derived_index_objects = path_state_objects .into_iter() .chain(commit_graph_objects) .chain(object_index_objects) + .chain(browse_root_objects) .chain(manifests.into_iter().filter(|object| { let key = object.location.as_ref(); key.starts_with(path_state_descriptor_prefix.as_ref()) @@ -3365,8 +3369,8 @@ async fn sweep_capsule_objects( } } Ok(GcOutcome { - list_requests: 7, - list_parallelism: 7, + list_requests: 9, + list_parallelism: 9, list_wall_seconds: started.elapsed().as_secs_f64(), dry_run: args.dry_run, ..accounting @@ -3379,9 +3383,27 @@ async fn mark_derived_index_objects( manifest: &crab_metadata::manifests::Manifest, reachable: &mut HashSet, ) -> Result<()> { - if let Some(indexes) = crab_metadata::capsule_protocol::load_browse_indexes(layout).await? { - // Ref movement makes readiness stale for new queries, but existing - // readers still own that published root and may not have loaded pages. + let current = crab_metadata::capsule_protocol::load_browse_indexes(layout).await?; + let roots = crab_metadata::capsule_protocol::load_retained_browse_indexes(layout).await?; + if current + .as_ref() + .is_some_and(|current| !roots.iter().any(|root| root.indexes() == current)) + { + return Err(CrabError::CorruptObject { + path: layout.capsule_browse_indexes_path().to_string(), + reason: "published browse indexes have no retained capture; rebuild browse indexes before GC".to_owned(), + }); + } + // Captured readers own original source ranges as well as cold index + // pages. Repacking cannot substitute a new source behind those ranges. + for root in roots { + reachable.insert(layout.browse_retained_root_path(root.hash()).to_string()); + reachable.extend( + root.source_paths(layout) + .into_iter() + .map(|path| path.to_string()), + ); + let indexes = root.indexes(); let graph_hash = indexes.commit_graph_hash(); let graph_path = layout.bulk_manifest_path("commit-graph", graph_hash); let graph = crab_metadata::split_commit_graph::load_split_commit_graph_descriptor( @@ -5791,6 +5813,15 @@ mod tests { .unwrap() .with_object_index(physical_hash.clone()) .unwrap(); + crab_metadata::capsule_protocol::retain_browse_indexes( + &layout, + &indexes, + manifest.generation, + None, + &[], + ) + .await + .unwrap(); let index_path = layout.capsule_browse_indexes_path(); let bytes = indexes.encode().unwrap(); index_etag = Some(match index_etag { diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index f46a75818..c951f80f6 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -459,7 +459,18 @@ async fn prune_history( let base = crab_write::capsule_protocol::open_root(layout).await?; let chain = history_chain(layout, &base).await?; let mut payload = prune_payload(&chain, args.keep_last, true); + let minimum = chain + .iter() + .take(args.keep_last) + .last() + .map(|segment| segment.checkpoint().covered_generation()); if payload.roots_pruned == 0 { + // Retry capture cleanup after an earlier history CAS committed but + // its derived-retention update failed. Empty history has no cutoff. + if let Some(minimum) = minimum { + crab_metadata::capsule_protocol::prune_retained_browse_indexes(layout, minimum) + .await?; + } return Ok(payload); } // Finish fallible local preparation before acquiring the root fence; @@ -484,7 +495,18 @@ async fn prune_history( let replacement = crab_write::capsule_protocol::replace_history(layout, fenced.clone(), &rebuilt).await; let (primary, release_base) = match replacement { - Ok(root) => (None, root), + Ok(root) => { + let cleanup = match minimum { + Some(minimum) => { + crab_metadata::capsule_protocol::prune_retained_browse_indexes( + layout, minimum, + ) + .await + } + None => Ok(()), + }; + (cleanup.err().map(CrabError::from), root) + } Err(error) => (Some(CrabError::from(error)), fenced), }; let release = crab_write::capsule_protocol::end_gc(layout, release_base, &fence_id).await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 4c835987e..1f0dd789f 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -121,6 +121,23 @@ record in place, without a legacy reader. Readers ignore stale records; absent indexes remain a readiness state. GC retains dependencies of the currently published readiness record even when newer refs make it unusable for new reads. Storage retention does not grant inventory membership or ref visibility. +The separate `v2/browse-retention` CAS head owns a bounded chain of authenticated +`metadata/browse-roots/` captures. Maintenance records original checkpoint and +Git source identities before advertising replacement readiness. GC follows every +retained capture's graph, path-state, physical pages and original source bodies; +foreground reads still load only the current bounded readiness record. An +unchanged capture does not append a node. Missing/corrupt lifetime dependencies +stop GC before deletion. A valid readiness record without lifetime ownership +requires background browse maintenance before GC; there is no scan fallback. + +Explicit history pruning trims superseded captures below the oldest retained +checkpoint generation only after the authoritative history CAS succeeds. Current +readiness and the newest lifetime capture remain rooted. A failed cleanup leaves +extra retention and is retried even if the history chain already has the requested +length. Pruning is the boundary that can invalidate captures from discarded +generations; ordinary rebuilding, checkpointing and GC do not expire readers. +The new lifetime format is version 1; no release tag contains capsule browse +readiness, and its existing version 3 encoding remains unchanged. Rebuilding an immutable index verifies the generated content identity and repairs corrupt stored bytes only with a conditional update against the observed version, followed by readback. diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs new file mode 100644 index 000000000..3c476f26e --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -0,0 +1,337 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use bytes::Bytes; +use crab_storage::{ETag, Store, StoreLayout}; +use serde::{Deserialize, Serialize}; + +use super::{ + BrowseIndexes, LayeredCheckpoint, MAX_BROWSE_INDEXES_BYTES, MAX_HISTORY_CHAIN_BYTES, + MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, PackSourceDescriptor, PackSourceKind, +}; +use crate::error::{MetadataError, Result}; +use crate::validation::{corrupt_object, validate_content_hash}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Head { + version: u32, + hash: String, + roots: usize, + bytes: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Payload { + version: u32, + generation: u64, + indexes: BrowseIndexes, + previous: Option, + checkpoint: Option, + sources: BTreeMap, +} + +/// Immutable capture of browse pages and the original Git sources used by their readers. +#[derive(Debug)] +pub struct RetainedBrowseIndexes { + hash: String, + payload: Payload, +} + +impl RetainedBrowseIndexes { + /// Return this immutable retention record's identity. + #[must_use] + pub fn hash(&self) -> &str { + &self.hash + } + + /// Return the source generation used by explicit history pruning. + #[must_use] + pub fn generation(&self) -> u64 { + self.payload.generation + } + + /// Return captured index dependencies without granting current visibility. + #[must_use] + pub fn indexes(&self) -> &BrowseIndexes { + &self.payload.indexes + } + + /// Return original source bodies, including a captured checkpoint when present. + #[must_use] + pub fn source_paths(&self, layout: &StoreLayout) -> Vec { + self.payload + .checkpoint + .iter() + .map(|hash| layout.capsule_checkpoint_path(hash)) + .chain(self.payload.sources.iter().map(|(hash, kind)| match kind { + PackSourceKind::CapsuleRun => layout.capsule_path(hash), + PackSourceKind::PackLayer => layout.capsule_pack_layer_path(hash), + })) + .collect() + } +} + +impl Payload { + fn encode(&self) -> Result { + self.validate()?; + let bytes = serde_json::to_vec(self) + .map_err(|source| MetadataError::BrowseIndexRecord { source })?; + if bytes.len() as u64 > MAX_HISTORY_SEGMENT_BYTES { + return Err(corrupt("retained browse record exceeds byte limit")); + } + Ok(bytes.into()) + } + + fn validate(&self) -> Result<()> { + if self.version != 1 { + return Err(corrupt("unsupported retained browse version")); + } + self.indexes.encode()?; + for hash in self + .previous + .iter() + .chain(self.checkpoint.iter()) + .chain(self.sources.keys()) + { + validate_content_hash(hash, "retention dependency", "browse retention")?; + } + Ok(()) + } +} + +impl Head { + fn append(previous: Option<&Self>, bytes: &Bytes) -> Result { + let value = Self { + version: 1, + hash: blake3::hash(bytes).to_hex().to_string(), + roots: previous.map_or(0, |head| head.roots).saturating_add(1), + bytes: previous + .map_or(0, |head| head.bytes) + .saturating_add(bytes.len() as u64), + }; + value.validate()?; + Ok(value) + } + + fn validate(&self) -> Result<()> { + validate_content_hash(&self.hash, "retained root", "browse retention")?; + if self.version != 1 + || self.roots == 0 + || self.roots > MAX_HISTORY_CHAIN_SEGMENTS + || self.bytes == 0 + || self.bytes > MAX_HISTORY_CHAIN_BYTES + { + return Err(corrupt( + "retained browse head exceeds version or resource bounds", + )); + } + Ok(()) + } +} + +async fn load_head(layout: &StoreLayout) -> Result> { + let (bytes, etag) = match layout + .store() + .get_with_etag_bounded( + &layout.browse_retained_head_path(), + MAX_BROWSE_INDEXES_BYTES, + ) + .await + { + Ok(value) => value, + Err(crab_storage::StorageError::NotFound { .. }) => return Ok(None), + Err(error) => return Err(error.into()), + }; + let head: Head = serde_json::from_slice(&bytes) + .map_err(|source| MetadataError::BrowseIndexRecord { source })?; + head.validate()?; + Ok(Some((head, etag))) +} + +async fn load_payload( + layout: &StoreLayout, + hash: &str, + maximum: u64, +) -> Result<(Payload, u64)> { + validate_content_hash(hash, "retained root", "browse retention")?; + let expected = *blake3::Hash::from_hex(hash) + .map_err(|_| corrupt("invalid retained browse hash"))? + .as_bytes(); + let bytes = layout + .store() + .verify_bounded( + &layout.browse_retained_root_path(hash), + &expected, + maximum.min(MAX_HISTORY_SEGMENT_BYTES), + ) + .await?; + let payload: Payload = serde_json::from_slice(&bytes) + .map_err(|source| MetadataError::BrowseIndexRecord { source })?; + payload.validate()?; + Ok((payload, bytes.len() as u64)) +} + +async fn publish_head(layout: &StoreLayout, head: &Head, etag: Option) -> Result<()> { + let bytes = + serde_json::to_vec(head).map_err(|source| MetadataError::BrowseIndexRecord { source })?; + let path = layout.browse_retained_head_path(); + match etag { + Some(etag) => layout.store().update(&path, bytes.into(), etag).await?, + None => { + layout + .store() + .create_strict_with_etag(&path, bytes.into()) + .await? + } + }; + Ok(()) +} + +/// Retain original dependencies before publishing replacement browse readiness. +/// +/// The caller holds generation ownership and both GC writer fences. Lifetime +/// CAS precedes readiness CAS: a failed readiness publication may retain extra +/// data, but a crash can never advertise unretained pages or source ranges. +pub async fn retain_browse_indexes( + layout: &StoreLayout, + indexes: &BrowseIndexes, + generation: u64, + checkpoint: Option<&LayeredCheckpoint>, + run_sources: &[PackSourceDescriptor], +) -> Result<()> { + let previous = load_head(layout).await?; + let mut sources = BTreeMap::new(); + for source in checkpoint + .into_iter() + .flat_map(LayeredCheckpoint::sources) + .chain(run_sources) + { + if sources + .insert(source.object_hash().to_owned(), source.kind()) + .is_some_and(|kind| kind != source.kind()) + { + return Err(corrupt("retained source has conflicting kinds")); + } + } + let mut payload = Payload { + version: 1, + generation, + indexes: indexes.clone(), + previous: previous.as_ref().map(|(head, _)| head.hash.clone()), + checkpoint: checkpoint.map(|value| value.hash().to_owned()), + sources, + }; + if let Some((head, _)) = &previous { + let (old, _) = load_payload(layout, &head.hash, head.bytes).await?; + if old.generation > generation { + return Err(corrupt("retained browse generation moved backwards")); + } + payload.previous = old.previous.clone(); + if payload == old { + return Ok(()); + } + payload.previous = Some(head.hash.clone()); + } + let bytes = payload.encode()?; + let head = Head::append(previous.as_ref().map(|(head, _)| head), &bytes)?; + layout + .store() + .put_if_absent_verified(&layout.browse_retained_root_path(&head.hash), bytes) + .await?; + publish_head(layout, &head, previous.map(|(_, etag)| etag)).await +} + +async fn load_chain( + layout: &StoreLayout, + head: &Head, +) -> Result> { + let mut next = Some(head.hash.clone()); + let mut seen = BTreeSet::new(); + let mut records: Vec = Vec::new(); + let mut remaining = head.bytes; + while let Some(hash) = next { + if records.len() == head.roots || !seen.insert(hash.clone()) { + return Err(corrupt("retained browse count or cycle mismatch")); + } + let (payload, bytes) = load_payload(layout, &hash, remaining).await?; + remaining = remaining + .checked_sub(bytes) + .ok_or_else(|| corrupt("retained browse chain exceeds byte limit"))?; + if records + .last() + .is_some_and(|newer| newer.generation() < payload.generation) + { + return Err(corrupt("retained browse generations are not ordered")); + } + next = payload.previous.clone(); + records.push(RetainedBrowseIndexes { hash, payload }); + } + if remaining != 0 || records.len() != head.roots { + return Err(corrupt("retained browse accounting mismatch")); + } + Ok(records) +} + +/// Authenticate retained captures for maintenance; foreground lookup never walks this chain. +pub async fn load_retained_browse_indexes( + layout: &StoreLayout, +) -> Result> { + match load_head(layout).await? { + Some((head, _)) => load_chain(layout, &head).await, + None => Ok(Vec::new()), + } +} + +/// Trim captured generations after authoritative history pruning commits. +/// +/// The caller holds the exclusive repository GC lease. Failure leaves extra +/// retention and is retryable; trimming before history CAS could strand readers. +pub async fn prune_retained_browse_indexes( + layout: &StoreLayout, + minimum_generation: u64, +) -> Result<()> { + let Some((head, etag)) = load_head(layout).await? else { + return Ok(()); + }; + let current = super::load_browse_indexes(layout).await?; + let records = load_chain(layout, &head).await?; + let keep = |position: usize, record: &RetainedBrowseIndexes| { + position == 0 + || record.generation() >= minimum_generation + || current.as_ref() == Some(record.indexes()) + }; + if records + .iter() + .enumerate() + .all(|(position, record)| keep(position, record)) + { + return Ok(()); + } + let mut previous: Option = None; + for (position, record) in records.into_iter().enumerate().rev() { + if !keep(position, &record) { + continue; + } + let payload = Payload { + previous: previous.as_ref().map(|head| head.hash.clone()), + ..record.payload + }; + let bytes = payload.encode()?; + let head = Head::append(previous.as_ref(), &bytes)?; + layout + .store() + .put_if_absent_verified(&layout.browse_retained_root_path(&head.hash), bytes) + .await?; + previous = Some(head); + } + let head = previous.ok_or_else(|| corrupt("pruning removed current browse retention"))?; + publish_head(layout, &head, Some(etag)).await +} + +fn corrupt(reason: &str) -> MetadataError { + corrupt_object("browse retention", reason) +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs new file mode 100644 index 000000000..698385e52 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -0,0 +1,224 @@ +use std::sync::Arc; + +use object_store::memory::InMemory; + +use super::*; + +fn fixture() -> StoreLayout { + StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "browse-retention".into(), + ) +} + +fn indexes(byte: char) -> BrowseIndexes { + BrowseIndexes::new(byte.to_string().repeat(64), "b".repeat(64), None).unwrap() +} + +#[tokio::test] +async fn unchanged_capture_does_not_grow_retention_or_rewrite_its_head() { + let layout = fixture(); + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + let before = layout + .store() + .get_with_etag_bounded(&layout.browse_retained_head_path(), 4096) + .await + .unwrap(); + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + let after = layout + .store() + .get_with_etag_bounded(&layout.browse_retained_head_path(), 4096) + .await + .unwrap(); + assert_eq!(before, after); +} + +#[tokio::test] +async fn replacement_retains_prior_coverage_even_without_query_readiness() { + let layout = fixture(); + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + retain_browse_indexes(&layout, &indexes('c'), 2, None, &[]) + .await + .unwrap(); + let roots = load_retained_browse_indexes(&layout).await.unwrap(); + assert_eq!( + roots + .iter() + .map(|root| root.indexes().state_digest()) + .collect::>(), + vec!["c".repeat(64), "a".repeat(64)] + ); +} + +#[tokio::test] +async fn explicit_pruning_keeps_current_readiness_and_relinks_retained_generations() { + let layout = fixture(); + for (generation, byte) in [(1, 'a'), (2, 'c'), (3, 'd'), (4, 'e')] { + retain_browse_indexes(&layout, &indexes(byte), generation, None, &[]) + .await + .unwrap(); + } + // A lifetime publication may precede a failed readiness CAS. Preserve the + // older advertised root even when it is below the explicit pruning floor. + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + indexes('a').encode().unwrap(), + ) + .await + .unwrap(); + prune_retained_browse_indexes(&layout, 3).await.unwrap(); + let roots = load_retained_browse_indexes(&layout).await.unwrap(); + assert_eq!( + roots + .iter() + .map(RetainedBrowseIndexes::generation) + .collect::>(), + vec![4, 3, 1] + ); +} + +#[tokio::test] +async fn corrupt_or_missing_retained_roots_fail_closed() { + for missing in [false, true] { + let layout = fixture(); + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + let roots = load_retained_browse_indexes(&layout).await.unwrap(); + let path = layout.browse_retained_root_path(roots[0].hash()); + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + } + assert!(load_retained_browse_indexes(&layout).await.is_err()); + } +} + +#[tokio::test] +async fn backwards_publication_preserves_the_lifetime_head() { + let layout = fixture(); + retain_browse_indexes(&layout, &indexes('a'), 2, None, &[]) + .await + .unwrap(); + let before = layout + .store() + .get_with_etag_bounded(&layout.browse_retained_head_path(), 4096) + .await + .unwrap(); + assert!( + retain_browse_indexes(&layout, &indexes('c'), 1, None, &[]) + .await + .is_err() + ); + assert_eq!( + layout + .store() + .get_with_etag_bounded(&layout.browse_retained_head_path(), 4096) + .await + .unwrap(), + before + ); +} + +#[test] +fn retention_head_bounds_are_admitted_before_publishing() { + for (roots, bytes) in [ + (MAX_HISTORY_CHAIN_SEGMENTS, 1), + (1, MAX_HISTORY_CHAIN_BYTES), + ] { + let old = Head { + version: 1, + hash: "a".repeat(64), + roots, + bytes, + }; + assert!(Head::append(Some(&old), &Bytes::from_static(b"new root")).is_err()); + } +} + +#[tokio::test] +async fn an_expired_publisher_cannot_replace_a_successor_lifetime_head() { + let layout = fixture(); + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + let (old, etag) = load_head(&layout).await.unwrap().unwrap(); + retain_browse_indexes(&layout, &indexes('c'), 2, None, &[]) + .await + .unwrap(); + let result = publish_head(&layout, &old, Some(etag)).await; + assert!(matches!( + result, + Err(MetadataError::Storage { + source: crab_storage::StorageError::StateConflict { .. } + }) + )); +} + +#[tokio::test] +async fn captures_keep_original_source_bodies_independently_of_current_packaging() { + use crate::capsule_protocol::{Capsule, CapsuleRefEdit, CapsuleRun, CapsuleTransaction}; + let layout = fixture(); + let transaction = CapsuleTransaction::new( + &"a".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("1".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + let source = PackSourceDescriptor::from_capsule_run(&run).unwrap(); + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[source]) + .await + .unwrap(); + retain_browse_indexes(&layout, &indexes('c'), 2, None, &[]) + .await + .unwrap(); + let roots = load_retained_browse_indexes(&layout).await.unwrap(); + assert_eq!( + roots[1].source_paths(&layout), + vec![layout.capsule_path(run.hash())] + ); +} + +#[tokio::test] +async fn retrying_completed_pruning_does_not_rewrite_the_lifetime_head() { + let layout = fixture(); + for (generation, byte) in [(1, 'a'), (2, 'c'), (3, 'd')] { + retain_browse_indexes(&layout, &indexes(byte), generation, None, &[]) + .await + .unwrap(); + } + prune_retained_browse_indexes(&layout, 2).await.unwrap(); + let before = layout + .store() + .get_with_etag_bounded(&layout.browse_retained_head_path(), 4096) + .await + .unwrap(); + prune_retained_browse_indexes(&layout, 2).await.unwrap(); + assert_eq!( + layout + .store() + .get_with_etag_bounded(&layout.browse_retained_head_path(), 4096) + .await + .unwrap(), + before + ); +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 0da15ce80..0f9bda89d 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -3,6 +3,8 @@ use bstr::ByteSlice; mod browse; +#[cfg(feature = "storage")] +mod browse_retention; mod capsule; #[cfg(feature = "storage")] mod captured_visibility; @@ -27,6 +29,11 @@ mod visibility; #[cfg(feature = "storage")] pub use browse::load_browse_indexes; pub use browse::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES}; +#[cfg(feature = "storage")] +pub use browse_retention::{ + RetainedBrowseIndexes, load_retained_browse_indexes, prune_retained_browse_indexes, + retain_browse_indexes, +}; pub use capsule::{ Capsule, CapsuleGitPack, CapsuleGitPackDescriptor, CapsuleSection, CapsuleSectionKind, CapsuleSectionLocation, diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index 56661eeb9..cd5eff2b2 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -1,4 +1,6 @@ //! Measure isolated cold/warm capsule reads and emit independent-oracle evidence. +#[path = "support/captured_browse.rs"] +mod captured_browse; mod support; use std::io; @@ -36,7 +38,7 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|catalog-current|ref-point|resolve-oid|blame-history|blame-history-uncached]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|catalog-current|capture-browse|ref-point|resolve-oid|blame-history|blame-history-uncached]", ) .into()), }; @@ -44,6 +46,7 @@ async fn main() -> Result<()> { && selection != "build-indexes" && selection != "catalog" && selection != "catalog-current" + && selection != "capture-browse" && selection != "checkpoint" && selection != "graph-pages" && selection != "graph-ancestry" @@ -74,7 +77,20 @@ async fn main() -> Result<()> { .with_storage_observer(counters.clone()); let layout = StoreLayout::new(store, prefix.to_owned()); let identity = RepositoryIdentity::new(format!("s3:{bucket}"), prefix.as_str(), 1)?; - if selection == "ref-point" { + if selection == "capture-browse" { + let runtime = Arc::new(RemoteGitRuntime::default()); + let result = captured_browse::qualify( + &layout, + &identity, + runtime.clone(), + options, + &path, + &counters, + ) + .await; + runtime.shutdown().await; + result?; + } else if selection == "ref-point" { let root = counters .measure("open", "cold", async { Ok(crab_metadata::capsule_protocol::load_root(&layout).await?) diff --git a/crates/crab-remote/examples/support/captured_browse.rs b/crates/crab-remote/examples/support/captured_browse.rs new file mode 100644 index 000000000..009dccdcb --- /dev/null +++ b/crates/crab-remote/examples/support/captured_browse.rs @@ -0,0 +1,67 @@ +//! Hold a cold reader while an external driver moves refs, rebuilds, and runs repository GC. +use std::io::{self, Write}; +use std::sync::Arc; + +use crab_remote_git::{GitPath, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, Revision}; +use crab_storage::{Store, StoreLayout}; +use serde_json::json; +use tokio_util::sync::CancellationToken; + +use super::support::{Measurements, Result, open_repository}; + +pub async fn qualify( + layout: &StoreLayout, + identity: &RepositoryIdentity, + runtime: Arc, + options: RepositoryOptions, + path: &GitPath, + counters: &Measurements, +) -> Result<()> { + let cancel = CancellationToken::new(); + let repository = counters + .measure( + "open", + "cold", + open_repository(layout.clone(), identity.clone(), runtime, options, &cancel), + ) + .await?; + let head = repository + .refs() + .head + .as_ref() + .ok_or_else(|| io::Error::other("repository has no HEAD"))?; + let revision = Revision::Reference(head.name.clone()); + // Do not resolve a commit or read a directory here: that would warm the + // exact physical/graph/path pages whose lifetime this experiment measures. + println!( + "{}", + json!({"kind":"captured", "schema":1, "commit":head.target.to_string(), "generation":repository.generation()}) + ); + io::stdout().flush()?; + let command = tokio::task::spawn_blocking(|| -> io::Result { + let mut command = String::new(); + io::stdin().read_line(&mut command)?; + Ok(command) + }) + .await??; + if command.trim() != "continue" { + return Err(io::Error::other( + "capture probe requires a continue command after external GC", + ) + .into()); + } + for name in ["blob", "attribution", "path-history"] { + let value = counters + .measure( + name, + "after-external-gc", + super::query(name, &repository, &revision, path, &cancel), + ) + .await?; + println!( + "{}", + json!({"kind":"captured-result", "operation":name, "value":value}) + ); + } + Ok(()) +} diff --git a/crates/crab-remote/src/browse_indexes.rs b/crates/crab-remote/src/browse_indexes.rs index ebf1fa4f2..31818bbf2 100644 --- a/crates/crab-remote/src/browse_indexes.rs +++ b/crates/crab-remote/src/browse_indexes.rs @@ -2,7 +2,9 @@ use std::{sync::Arc, time::Duration}; -use crab_metadata::capsule_protocol::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES, load_root}; +use crab_metadata::capsule_protocol::{ + BrowseIndexes, MAX_BROWSE_INDEXES_BYTES, load_root, retain_browse_indexes, +}; use crab_read::capsule_protocol::{ CapsuleReadLimits, open_view_from_root_with_metadata, read_activity_from_root, }; @@ -97,6 +99,12 @@ pub async fn ensure( } indexes = indexes.with_object_index(hash)?; } + // Lifetime publication precedes query readiness. If the later CAS + // loses, extra retained data is safe; the opposite order can strand readers. + retain_browse_indexes( + layout, &indexes, snapshot.manifest.generation, view.layered_checkpoint(), + view.capsule_run_sources(), + ).await?; if previous.as_ref() == Some(&indexes) && indexes.path_state_hash().is_some() { ready = Some(indexes); return Ok(()); @@ -111,7 +119,7 @@ pub async fn ensure( }; // Bind the just-verified physical pages to this same view before path // construction. Reopening a fresh root could mix index and ref identities. - let repository = view.with_browse_indexes(Some(indexes.clone())) + let repository = view.clone().with_browse_indexes(Some(indexes.clone())) .git_repository_from_store(layout.clone(), identity.clone(), runtime, options, max_bytes, cancel) .await?; let complete = crab_write::generation::browse::build_path_state( @@ -123,6 +131,10 @@ pub async fn ensure( cancel, ) .await?; + retain_browse_indexes( + layout, &complete, snapshot.manifest.generation, view.layered_checkpoint(), + view.capsule_run_sources(), + ).await?; if publish_current(layout, &complete, etag, cancel) .await? .is_some() diff --git a/crates/crab-storage/src/layout.rs b/crates/crab-storage/src/layout.rs index 35c9a5aee..895ad7c52 100644 --- a/crates/crab-storage/src/layout.rs +++ b/crates/crab-storage/src/layout.rs @@ -188,6 +188,18 @@ impl StoreLayout { self.repo_path("v2/browse-indexes") } + /// Path to the mutable lifetime root for published browse captures. + #[must_use] + pub fn browse_retained_head_path(&self) -> ObjectPath { + self.repo_path("v2/browse-retention") + } + + /// Path to one authenticated browse capture retained until explicit history pruning. + #[must_use] + pub fn browse_retained_root_path(&self, hash: &str) -> ObjectPath { + self.repo_path(&format!("metadata/browse-roots/{hash}")) + } + /// Path to one immutable capsule-protocol capsule. #[must_use] pub fn capsule_path(&self, hash: &str) -> ObjectPath { From 5cd6d6bf3d0ba5acbc9e06c792ddeda9f91b2eed Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 03:33:27 -0700 Subject: [PATCH 067/127] fix(gc): retain ref-only capture dependencies Captured visibility and catalog pages can live in runs without Git members. Retain every captured capsule pointer instead of only the physical pack-source directory. The metadata regression uses a real ref-only run and retains its source-path assertion. No dependency contract or assertion is weakened. --- crates/crab-metadata/README.md | 5 +++-- .../src/capsule_protocol/browse_retention.rs | 22 ++++++++++--------- .../browse_retention/tests.rs | 14 ++++++++++-- crates/crab-remote/src/browse_indexes.rs | 4 ++-- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 1f0dd789f..1104648c6 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -122,8 +122,9 @@ records; absent indexes remain a readiness state. GC retains dependencies of the published readiness record even when newer refs make it unusable for new reads. Storage retention does not grant inventory membership or ref visibility. The separate `v2/browse-retention` CAS head owns a bounded chain of authenticated -`metadata/browse-roots/` captures. Maintenance records original checkpoint and -Git source identities before advertising replacement readiness. GC follows every +`metadata/browse-roots/` captures. Maintenance records original checkpoint, Git source and all pending capsule-run +identities before advertising replacement readiness. Ref-only visibility/catalog +runs remain dependencies even when they contain no Git pack members. GC follows every retained capture's graph, path-state, physical pages and original source bodies; foreground reads still load only the current bounded readiness record. An unchanged capture does not append a node. Missing/corrupt lifetime dependencies diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 3c476f26e..89a06c4b9 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -5,8 +5,8 @@ use crab_storage::{ETag, Store, StoreLayout}; use serde::{Deserialize, Serialize}; use super::{ - BrowseIndexes, LayeredCheckpoint, MAX_BROWSE_INDEXES_BYTES, MAX_HISTORY_CHAIN_BYTES, - MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, PackSourceDescriptor, PackSourceKind, + BrowseIndexes, CapsulePointer, LayeredCheckpoint, MAX_BROWSE_INDEXES_BYTES, + MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, PackSourceKind, }; use crate::error::{MetadataError, Result}; use crate::validation::{corrupt_object, validate_content_hash}; @@ -198,19 +198,21 @@ pub async fn retain_browse_indexes( indexes: &BrowseIndexes, generation: u64, checkpoint: Option<&LayeredCheckpoint>, - run_sources: &[PackSourceDescriptor], + run_pointers: &[CapsulePointer], ) -> Result<()> { let previous = load_head(layout).await?; let mut sources = BTreeMap::new(); - for source in checkpoint + let checkpoint_sources = checkpoint .into_iter() .flat_map(LayeredCheckpoint::sources) - .chain(run_sources) - { - if sources - .insert(source.object_hash().to_owned(), source.kind()) - .is_some_and(|kind| kind != source.kind()) - { + .map(|source| (source.object_hash().to_owned(), source.kind())); + let capsule_sources = run_pointers + .iter() + .map(|pointer| (pointer.hash().to_owned(), PackSourceKind::CapsuleRun)); + // Ref-only runs carry visibility/catalog pages despite having no Git + // members. Captures own those lazy reads as well as physical pack ranges. + for (hash, kind) in checkpoint_sources.chain(capsule_sources) { + if sources.insert(hash, kind).is_some_and(|old| old != kind) { return Err(corrupt("retained source has conflicting kinds")); } } diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index 698385e52..61412516f 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -169,7 +169,7 @@ async fn an_expired_publisher_cannot_replace_a_successor_lifetime_head() { } #[tokio::test] -async fn captures_keep_original_source_bodies_independently_of_current_packaging() { +async fn captures_keep_ref_only_visibility_sources_independently_of_current_packaging() { use crate::capsule_protocol::{Capsule, CapsuleRefEdit, CapsuleRun, CapsuleTransaction}; let layout = fixture(); let transaction = CapsuleTransaction::new( @@ -184,7 +184,17 @@ async fn captures_keep_original_source_bodies_independently_of_current_packaging .unwrap(); let capsule = Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap(); let run = CapsuleRun::leaf(capsule).unwrap(); - let source = PackSourceDescriptor::from_capsule_run(&run).unwrap(); + let source = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); retain_browse_indexes(&layout, &indexes('a'), 1, None, &[source]) .await .unwrap(); diff --git a/crates/crab-remote/src/browse_indexes.rs b/crates/crab-remote/src/browse_indexes.rs index 31818bbf2..d1ff7b312 100644 --- a/crates/crab-remote/src/browse_indexes.rs +++ b/crates/crab-remote/src/browse_indexes.rs @@ -103,7 +103,7 @@ pub async fn ensure( // loses, extra retained data is safe; the opposite order can strand readers. retain_browse_indexes( layout, &indexes, snapshot.manifest.generation, view.layered_checkpoint(), - view.capsule_run_sources(), + view.capsule_run_pointers(), ).await?; if previous.as_ref() == Some(&indexes) && indexes.path_state_hash().is_some() { ready = Some(indexes); @@ -133,7 +133,7 @@ pub async fn ensure( .await?; retain_browse_indexes( layout, &complete, snapshot.manifest.generation, view.layered_checkpoint(), - view.capsule_run_sources(), + view.capsule_run_pointers(), ).await?; if publish_current(layout, &complete, etag, cancel) .await? From 91a26c34aca5d430957e564bd5e6c23e9faa8634 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 04:10:30 -0700 Subject: [PATCH 068/127] refactor(gc): enumerate index metadata once Keep immutable capture roots in the v2 control namespace and scan derived index bodies through one bounded ownership filter. Report seven actual LIST streams without changing the existing assertions. Cover lookalike metadata directories and recorded LIST prefixes. Move the captured view through readiness attachment instead of cloning its metadata. --- crab/src/cmd/gc/captured_index_tests.rs | 110 +++++++++++++++++++++++ crab/src/cmd/gc/mod.rs | 31 +++---- crab/src/cmd/history_recovery_v2.rs | 2 +- crates/crab-metadata/README.md | 2 +- crates/crab-remote/src/browse_indexes.rs | 6 +- crates/crab-storage/src/layout.rs | 2 +- 6 files changed, 132 insertions(+), 21 deletions(-) diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index 4fd56e2a1..f135038bc 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -32,6 +32,7 @@ struct AgedObjects { inner: InMemory, paths: Mutex>, modified: SystemTime, + list_calls: Mutex>, } impl std::fmt::Display for AgedObjects { @@ -75,6 +76,9 @@ impl ObjectStore for AgedObjects { } fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + if let Some(prefix) = prefix { + self.list_calls.lock().unwrap().push(prefix.clone()); + } let paths = self.paths.lock().unwrap().clone(); let modified = self.modified; Box::pin(self.inner.list(prefix).map_ok(move |mut meta| { @@ -193,6 +197,7 @@ async fn cold_capture_after_gc(replace_readiness: bool) -> crab_metadata::error: inner: InMemory::new(), paths: Mutex::new(HashSet::new()), modified: SystemTime::now() - Duration::from_secs(7200), + list_calls: Mutex::new(Vec::new()), }); let store = Store::new(backend.clone()); let layout = StoreLayout::new(store.as_storage().clone(), "org/captured-gc".to_owned()); @@ -305,3 +310,108 @@ async fn gc_preserves_cold_captures_after_readiness_replacement() { "cold capture lost after readiness replacement: {result:?}" ); } + +#[tokio::test] +async fn capsule_gc_lists_each_candidate_namespace_once() { + let backend = Arc::new(AgedObjects { + inner: InMemory::new(), + paths: Mutex::new(HashSet::new()), + modified: SystemTime::now(), + list_calls: Mutex::new(Vec::new()), + }); + let store = Store::new(backend.clone()); + let layout = StoreLayout::new(store.as_storage().clone(), "org/gc-listing".to_owned()); + let root = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let mut manifest = crab_metadata::manifests::Manifest::default_for_repo("refs/heads/main"); + manifest.seal_git_validation(); + backend.list_calls.lock().unwrap().clear(); + let outcome = sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + root.record().root(), + &[], + &manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now(), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + let mut actual = backend.list_calls.lock().unwrap().clone(); + actual.sort(); + let mut expected = [ + "v2/capsules/", + "v2/checkpoints/", + "v2/pack-layers/", + "v2/history/", + "metadata/", + "v2/browse-roots/", + "manifests/", + ] + .map(|suffix| layout.repo_path(suffix)) + .to_vec(); + expected.sort(); + assert_eq!( + (actual, outcome.list_requests, outcome.list_parallelism), + (expected, 7, 7) + ); +} + +#[tokio::test] +async fn capsule_gc_preserves_metadata_outside_owned_index_directories() { + let backend = Arc::new(AgedObjects { + inner: InMemory::new(), + paths: Mutex::new(HashSet::new()), + modified: SystemTime::now() - Duration::from_secs(7200), + list_calls: Mutex::new(Vec::new()), + }); + let store = Store::new(backend.clone()); + let layout = StoreLayout::new(store.as_storage().clone(), "org/gc-metadata".to_owned()); + let root = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let unrelated = [ + "unrelated/keep", + "path-state-backups/keep", + "commit-graph-backups/keep", + "git-object-index-backups/keep", + ] + .map(|suffix| layout.repo_path(&format!("metadata/{suffix}"))); + for path in &unrelated { + store + .put(path, Bytes::from_static(b"unowned metadata")) + .await + .unwrap(); + backend.paths.lock().unwrap().insert(path.clone()); + } + let mut manifest = crab_metadata::manifests::Manifest::default_for_repo("refs/heads/main"); + manifest.seal_git_validation(); + sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + root.record().root(), + &[], + &manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now(), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + let retained = + futures_util::future::try_join_all(unrelated.iter().map(|path| store.head(path))).await; + assert!( + retained.is_ok(), + "GC removed metadata outside its owned directories: {retained:?}" + ); +} diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index bcd60d9e0..1f4a279f8 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3240,10 +3240,10 @@ async fn sweep_capsule_objects( let checkpoint_prefix = layout.repo_path("v2/checkpoints/"); let pack_layer_prefix = layout.repo_path("v2/pack-layers/"); let history_prefix = layout.repo_path("v2/history/"); - let path_state_prefix = layout.repo_path("metadata/path-state/"); - let commit_graph_prefix = layout.repo_path("metadata/commit-graph/"); - let object_index_prefix = layout.repo_path("metadata/git-object-index/"); - let browse_root_prefix = layout.repo_path("metadata/browse-roots/"); + let metadata_prefix = layout.repo_path("metadata/"); + let index_prefixes = ["path-state", "commit-graph", "git-object-index"] + .map(|kind| format!("{metadata_prefix}/{kind}/")); + let browse_root_prefix = layout.repo_path("v2/browse-roots/"); let manifests_prefix = layout.repo_path("manifests/"); let path_state_descriptor_prefix = layout.repo_path("manifests/path-state-"); let commit_graph_descriptor_prefix = layout.repo_path("manifests/commit-graph-"); @@ -3253,9 +3253,7 @@ async fn sweep_capsule_objects( checkpoints, pack_layers, history, - path_state_objects, - commit_graph_objects, - object_index_objects, + metadata_objects, browse_root_objects, manifests, ) = tokio::try_join!( @@ -3263,16 +3261,19 @@ async fn sweep_capsule_objects( store.list_prefix(&checkpoint_prefix), store.list_prefix(&pack_layer_prefix), store.list_prefix(&history_prefix), - store.list_prefix(&path_state_prefix), - store.list_prefix(&commit_graph_prefix), - store.list_prefix(&object_index_prefix), + store.list_prefix(&metadata_prefix), store.list_prefix(&browse_root_prefix), store.list_prefix(&manifests_prefix), )?; - let derived_index_objects = path_state_objects + // One metadata scan covers derived index bodies. Lifetime roots belong to + // protocol control; unrelated metadata never becomes a deletion candidate. + let derived_index_objects = metadata_objects .into_iter() - .chain(commit_graph_objects) - .chain(object_index_objects) + .filter(|object| { + index_prefixes + .iter() + .any(|prefix| object.location.as_ref().starts_with(prefix)) + }) .chain(browse_root_objects) .chain(manifests.into_iter().filter(|object| { let key = object.location.as_ref(); @@ -3369,8 +3370,8 @@ async fn sweep_capsule_objects( } } Ok(GcOutcome { - list_requests: 9, - list_parallelism: 9, + list_requests: 7, + list_parallelism: 7, list_wall_seconds: started.elapsed().as_secs_f64(), dry_run: args.dry_run, ..accounting diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index c951f80f6..ac82e116e 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -462,7 +462,7 @@ async fn prune_history( let minimum = chain .iter() .take(args.keep_last) - .last() + .next_back() .map(|segment| segment.checkpoint().covered_generation()); if payload.roots_pruned == 0 { // Retry capture cleanup after an earlier history CAS committed but diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 1104648c6..961517e1c 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -122,7 +122,7 @@ records; absent indexes remain a readiness state. GC retains dependencies of the published readiness record even when newer refs make it unusable for new reads. Storage retention does not grant inventory membership or ref visibility. The separate `v2/browse-retention` CAS head owns a bounded chain of authenticated -`metadata/browse-roots/` captures. Maintenance records original checkpoint, Git source and all pending capsule-run +`v2/browse-roots/` captures. Maintenance records original checkpoint, Git source and all pending capsule-run identities before advertising replacement readiness. Ref-only visibility/catalog runs remain dependencies even when they contain no Git pack members. GC follows every retained capture's graph, path-state, physical pages and original source bodies; diff --git a/crates/crab-remote/src/browse_indexes.rs b/crates/crab-remote/src/browse_indexes.rs index d1ff7b312..14b7b07e5 100644 --- a/crates/crab-remote/src/browse_indexes.rs +++ b/crates/crab-remote/src/browse_indexes.rs @@ -53,7 +53,7 @@ pub async fn ensure( max_capsule_bytes: max_bytes, max_frontier_bytes: max_bytes, }; - let view = open_view_from_root_with_metadata(layout, root, limits).await?; + let mut view = open_view_from_root_with_metadata(layout, root, limits).await?; let snapshot = view.git_snapshot()?; if snapshot.manifest.refs.is_empty() { return Ok(()); @@ -119,8 +119,8 @@ pub async fn ensure( }; // Bind the just-verified physical pages to this same view before path // construction. Reopening a fresh root could mix index and ref identities. - let repository = view.clone().with_browse_indexes(Some(indexes.clone())) - .git_repository_from_store(layout.clone(), identity.clone(), runtime, options, max_bytes, cancel) + view = view.with_browse_indexes(Some(indexes.clone())); + let repository = view.git_repository_from_store(layout.clone(), identity.clone(), runtime, options, max_bytes, cancel) .await?; let complete = crab_write::generation::browse::build_path_state( layout, diff --git a/crates/crab-storage/src/layout.rs b/crates/crab-storage/src/layout.rs index 895ad7c52..2e4641f8d 100644 --- a/crates/crab-storage/src/layout.rs +++ b/crates/crab-storage/src/layout.rs @@ -197,7 +197,7 @@ impl StoreLayout { /// Path to one authenticated browse capture retained until explicit history pruning. #[must_use] pub fn browse_retained_root_path(&self, hash: &str) -> ObjectPath { - self.repo_path(&format!("metadata/browse-roots/{hash}")) + self.repo_path(&format!("v2/browse-roots/{hash}")) } /// Path to one immutable capsule-protocol capsule. From 1261ea7a408ed6f1e98437c5166a34cab6e047d6 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:05:44 -0700 Subject: [PATCH 069/127] test(gc): keep captured index pages cold --- crates/crab-remote/README.md | 4 ++ .../examples/support/captured_browse.rs | 37 ++++++++++++------- crates/crab-remote/examples/support/mod.rs | 31 ++++++++++------ 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 20c9bf085..43b41b9a4 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -95,6 +95,10 @@ Foreground qualification uses `open_git_view_from_root`; browse-index builders use the complete metadata entry point. Pack payloads stay in authenticated origin ranges before and after checkpoints. Checkpoint catalog/visibility decoding and ref capture remain measured opening work, not a completed bounded-opening gate. +`capture-browse` captures the immutable view and readiness before its stdin +handshake, then constructs the repository after external GC. Its separate +`reader-open` measurement includes normal tip validation, keeping graph, physical +and path pages cold until the external lifecycle transition completes. `qualify_remote graph-pages` measures sampled ordinal/parent records and duplicate OID batches through `CommitGraphReader`. It verifies the derived graph's captured state and Git identity before reads. diff --git a/crates/crab-remote/examples/support/captured_browse.rs b/crates/crab-remote/examples/support/captured_browse.rs index 009dccdcb..2f6ba4c9d 100644 --- a/crates/crab-remote/examples/support/captured_browse.rs +++ b/crates/crab-remote/examples/support/captured_browse.rs @@ -7,7 +7,7 @@ use crab_storage::{Store, StoreLayout}; use serde_json::json; use tokio_util::sync::CancellationToken; -use super::support::{Measurements, Result, open_repository}; +use super::support::{Measurements, Result, capture_repository_view}; pub async fn qualify( layout: &StoreLayout, @@ -18,24 +18,19 @@ pub async fn qualify( counters: &Measurements, ) -> Result<()> { let cancel = CancellationToken::new(); - let repository = counters - .measure( - "open", - "cold", - open_repository(layout.clone(), identity.clone(), runtime, options, &cancel), - ) + let view = counters + .measure("open", "cold", capture_repository_view(layout)) .await?; - let head = repository + let head = view .refs() - .head - .as_ref() + .get(view.head()) .ok_or_else(|| io::Error::other("repository has no HEAD"))?; - let revision = Revision::Reference(head.name.clone()); - // Do not resolve a commit or read a directory here: that would warm the - // exact physical/graph/path pages whose lifetime this experiment measures. + let revision = Revision::Reference(view.head().to_owned()); + // Repository construction proves visible tips through graph pages. Delay + // it until after GC so even those pages stay cold in the captured view. println!( "{}", - json!({"kind":"captured", "schema":1, "commit":head.target.to_string(), "generation":repository.generation()}) + json!({"kind":"captured", "schema":1, "commit":head, "generation":view.root().root().generation()}) ); io::stdout().flush()?; let command = tokio::task::spawn_blocking(|| -> io::Result { @@ -50,6 +45,20 @@ pub async fn qualify( ) .into()); } + let repository = counters + .measure("reader-open", "after-external-gc", async { + Ok(view + .git_repository_from_store( + layout.clone(), + identity.clone(), + runtime, + options, + 2 * 1024 * 1024 * 1024, + &cancel, + ) + .await?) + }) + .await?; for name in ["blob", "attribution", "path-history"] { let value = counters .measure( diff --git a/crates/crab-remote/examples/support/mod.rs b/crates/crab-remote/examples/support/mod.rs index 221add4c4..d8c490bfb 100644 --- a/crates/crab-remote/examples/support/mod.rs +++ b/crates/crab-remote/examples/support/mod.rs @@ -233,18 +233,7 @@ pub async fn open_repository( options: RepositoryOptions, cancel: &CancellationToken, ) -> Result { - let root = crab_metadata::capsule_protocol::load_root(&layout).await?; - let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&layout).await?; - let view = crab_read::capsule_protocol::open_git_view_from_root( - &layout, - root, - crab_read::capsule_protocol::CapsuleReadLimits { - max_capsule_bytes: 2 * 1024 * 1024 * 1024, - max_frontier_bytes: 2 * 1024 * 1024 * 1024, - }, - ) - .await? - .with_browse_indexes(indexes); + let view = capture_repository_view(&layout).await?; Ok(view .git_repository_from_store( layout, @@ -257,6 +246,24 @@ pub async fn open_repository( .await?) } +pub async fn capture_repository_view( + layout: &StoreLayout, +) -> Result { + let root = crab_metadata::capsule_protocol::load_root(layout).await?; + let indexes = crab_metadata::capsule_protocol::load_browse_indexes(layout).await?; + let view = crab_read::capsule_protocol::open_git_view_from_root( + layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 2 * 1024 * 1024 * 1024, + max_frontier_bytes: 2 * 1024 * 1024 * 1024, + }, + ) + .await? + .with_browse_indexes(indexes); + Ok(view) +} + #[cfg(test)] mod tests { use super::*; From ef784e4e7ed48daf852705a0372fe09f67845898 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:30:33 -0700 Subject: [PATCH 070/127] test(catalog): probe cold capture lifetime --- crates/crab-remote/README.md | 4 + crates/crab-remote/examples/qualify_remote.rs | 9 ++- .../examples/support/captured_catalog.rs | 75 +++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 crates/crab-remote/examples/support/captured_catalog.rs diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 43b41b9a4..1ba312482 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -99,6 +99,10 @@ ref capture remain measured opening work, not a completed bounded-opening gate. handshake, then constructs the repository after external GC. Its separate `reader-open` measurement includes normal tip validation, keeping graph, physical and path pages cold until the external lifecycle transition completes. +`capture-catalog` takes a word-ordered file hash instead of a text path. It holds +the canonical current-state file-index session across the same stdin handshake, +then measures cold/warm duplicate lookups before closing and running the bulk +oracle. This tests catalog source lifetime without requiring browse indexes. `qualify_remote graph-pages` measures sampled ordinal/parent records and duplicate OID batches through `CommitGraphReader`. It verifies the derived graph's captured state and Git identity before reads. diff --git a/crates/crab-remote/examples/qualify_remote.rs b/crates/crab-remote/examples/qualify_remote.rs index cd5eff2b2..0bdf39b2a 100644 --- a/crates/crab-remote/examples/qualify_remote.rs +++ b/crates/crab-remote/examples/qualify_remote.rs @@ -1,6 +1,8 @@ //! Measure isolated cold/warm capsule reads and emit independent-oracle evidence. #[path = "support/captured_browse.rs"] mod captured_browse; +#[path = "support/captured_catalog.rs"] +mod captured_catalog; mod support; use std::io; @@ -38,7 +40,7 @@ async fn main() -> Result<()> { [bucket, prefix, path] => (bucket, prefix, path, "all"), [bucket, prefix, path, selection] => (bucket, prefix, path, selection.as_str()), _ => return Err(io::Error::other( - "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|catalog-current|capture-browse|ref-point|resolve-oid|blame-history|blame-history-uncached]", + "usage: qualify_remote [operation|all|build-indexes|checkpoint|catalog|catalog-current|capture-browse|capture-catalog|ref-point|resolve-oid|blame-history|blame-history-uncached]", ) .into()), }; @@ -47,6 +49,7 @@ async fn main() -> Result<()> { && selection != "catalog" && selection != "catalog-current" && selection != "capture-browse" + && selection != "capture-catalog" && selection != "checkpoint" && selection != "graph-pages" && selection != "graph-ancestry" @@ -77,7 +80,9 @@ async fn main() -> Result<()> { .with_storage_observer(counters.clone()); let layout = StoreLayout::new(store, prefix.to_owned()); let identity = RepositoryIdentity::new(format!("s3:{bucket}"), prefix.as_str(), 1)?; - if selection == "capture-browse" { + if selection == "capture-catalog" { + captured_catalog::qualify(&layout, &path, &counters).await?; + } else if selection == "capture-browse" { let runtime = Arc::new(RemoteGitRuntime::default()); let result = captured_browse::qualify( &layout, diff --git a/crates/crab-remote/examples/support/captured_catalog.rs b/crates/crab-remote/examples/support/captured_catalog.rs new file mode 100644 index 000000000..24ede0e39 --- /dev/null +++ b/crates/crab-remote/examples/support/captured_catalog.rs @@ -0,0 +1,75 @@ +//! Keep a canonical catalog session cold across externally driven compaction and GC. +use std::io::{self, Write}; + +use crab_metadata::file_index_lookup::FileIndexLookupSession; +use crab_remote_git::GitPath; +use crab_storage::{Store, StoreLayout}; +use crab_xet::hash::MerkleHash; +use serde_json::json; + +use super::support::{Measurements, Result}; + +pub async fn qualify( + layout: &StoreLayout, + key: &GitPath, + counters: &Measurements, +) -> Result<()> { + let key = std::str::from_utf8(key.as_bytes())?; + let hash = MerkleHash::from_hex(key)?; + let session = counters + .measure("open", "cold", async { + Ok( + FileIndexLookupSession::open_for_storage(layout.store(), layout.repo_prefix()) + .await?, + ) + }) + .await?; + let result: Result<_> = async { + // Capturing the current-state session must not prefetch its selected + // file pages or require browse maintenance to establish their lifetime. + println!("{}", json!({"kind":"captured", "schema":1, "file":key})); + io::stdout().flush()?; + let command = tokio::task::spawn_blocking(|| -> io::Result { + let mut command = String::new(); + io::stdin().read_line(&mut command)?; + Ok(command) + }) + .await??; + if command.trim() != "continue" { + return Err(io::Error::other( + "catalog capture requires a continue command after external GC", + ) + .into()); + } + let mut previous = None; + for cache in ["cold", "warm"] { + let selected = counters + .measure("catalog-current", cache, async { + Ok(session.lookup_batch(&[hash, hash]).await?) + }) + .await?; + if previous.as_ref().is_some_and(|value| value != &selected) { + return Err(io::Error::other("captured cold/warm catalog results differ").into()); + } + println!("{}", json!({"kind":"captured-result", "operation":"catalog-current", "cache":cache, + "shards":selected.iter().map(|hash| hash.as_ref().map(MerkleHash::hex)).collect::>()})); + previous = Some(selected); + } + Ok(previous) + } + .await; + // EOF, failed catalog reads and oracle mismatches must all drain the same + // session. The bulk oracle runs only after close, outside measured reads. + let closed = session.close().await; + let selected = result?; + closed?; + let full = crab_metadata::capsule_protocol::load_pointer_catalog(layout).await?; + let expected = full.files().get(&hash.hex()).ok_or_else(|| { + io::Error::other("captured catalog file is absent from maintenance oracle") + })?; + let expected = MerkleHash::from_hex(expected.shard_hash())?; + if selected != Some(vec![Some(expected), Some(expected)]) { + return Err(io::Error::other("captured catalog differs from maintenance oracle").into()); + } + Ok(()) +} From 9ae233a404ad5d7ba856a947eaa926fa6c35e0d5 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 06:28:25 -0700 Subject: [PATCH 071/127] fix(catalog): retain compacted capture sources Publish per-ref source lifetime before replacing compacted frontiers. Wire repository GC, fsck and explicit history pruning to the same authenticated retention records. --- crab/src/cmd/fsck_store.rs | 32 +- .../cmd/fsck_store/capsule_history_tests.rs | 60 +++ crab/src/cmd/gc/captured_index_tests.rs | 135 ++++++ crab/src/cmd/gc/mod.rs | 12 + crab/src/cmd/history_recovery_v2.rs | 20 +- crab/src/core/error.rs | 1 + crates/crab-metadata/README.md | 11 + .../crab-metadata/src/capsule_protocol/mod.rs | 7 + .../src/capsule_protocol/source_retention.rs | 445 ++++++++++++++++++ .../source_retention/tests.rs | 291 ++++++++++++ crates/crab-metadata/src/error.rs | 5 + crates/crab-storage/src/layout.rs | 25 + crates/crab-write/src/capsule_protocol.rs | 54 ++- 13 files changed, 1079 insertions(+), 19 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/source_retention.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 58eef16df..17c2cabd0 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -877,18 +877,32 @@ async fn verify_capsule_history( layout: &crab_storage::StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, ) -> Result<()> { - let Some(history) = root.history() else { - return Ok(()); + let segments = match root.history() { + Some(history) => { + crab_metadata::capsule_protocol::load_history_chain( + layout, + history, + crab_metadata::capsule_protocol::MAX_HISTORY_CHAIN_SEGMENTS, + crab_metadata::capsule_protocol::MAX_HISTORY_CHAIN_BYTES, + ) + .await? + } + None => Vec::new(), }; - let segments = crab_metadata::capsule_protocol::load_history_chain( - layout, - history, - crab_metadata::capsule_protocol::MAX_HISTORY_CHAIN_SEGMENTS, - crab_metadata::capsule_protocol::MAX_HISTORY_CHAIN_BYTES, - ) - .await?; let mut checkpoints = BTreeMap::new(); let mut runs = BTreeMap::new(); + let heads = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await? + .into_iter() + .map(|meta| meta.location) + .collect::>(); + let retained = + crab_metadata::capsule_protocol::load_retained_capsule_sources(layout, &heads).await?; + for run in retained.runs() { + insert_history_pointer(&mut runs, run.hash(), run.clone(), "retired capsule run")?; + } for segment in segments { insert_history_pointer( &mut checkpoints, diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index e6acdb55c..c8b0cb691 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -104,6 +104,66 @@ struct SourceReads { count: AtomicUsize, } +#[tokio::test] +async fn retired_sources_are_checked_without_checkpoint_history() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + verify_capsule_history(&layout, root.record().root()) + .await + .unwrap(); + layout + .store() + .delete(&layout.capsule_path(run.hash())) + .await + .unwrap(); + assert!( + verify_capsule_history(&layout, root.record().root()) + .await + .is_err() + ); +} + +#[tokio::test] +async fn retired_sources_share_full_body_verification_with_checkpoint_history() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + std::slice::from_ref(&pointer), + ) + .await + .unwrap(); + let root = checkpoint( + &layout, + root, + PackSourceDescriptor::from_capsule_run(&run).unwrap(), + pointer, + ) + .await; + let reads = Arc::new(SourceReads { + size: run.bytes().len() as u64, + count: AtomicUsize::new(0), + }); + let observed = crab_storage::StoreLayout::new( + layout.store().clone().with_storage_observer(reads.clone()), + layout.repo_prefix().to_owned(), + ); + verify_capsule_history(&observed, root.record().root()) + .await + .unwrap(); + assert_eq!(reads.count.load(Ordering::Relaxed), 1); +} + impl StorageObserver for SourceReads { fn started(&self, _operation: StorageOperation) {} diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index f135038bc..a210fc397 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -311,6 +311,141 @@ async fn gc_preserves_cold_captures_after_readiness_replacement() { ); } +#[tokio::test(flavor = "multi_thread")] +async fn gc_preserves_cold_catalog_sessions_without_browse_indexes() { + use crab_metadata::capsule_protocol::{ + CAPSULE_REF_COMPACTION_FAN_IN, FileCatalogEntry, PointerCatalog, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, + }; + let backend = Arc::new(AgedObjects { + inner: InMemory::new(), + paths: Mutex::new(HashSet::new()), + modified: SystemTime::now() - Duration::from_secs(7200), + list_calls: Mutex::new(Vec::new()), + }); + let store = Store::new(backend.clone()); + let layout = StoreLayout::new(store.as_storage().clone(), "org/catalog-gc".to_owned()); + let root = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let file_hash = MerkleHash::from([10; 32]); + let shard_hash = MerkleHash::from([13; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + "a".repeat(64), + XorbCatalogEntry::new( + 1, + "b".repeat(64), + vec![XorbChunkEntry::new("c".repeat(64), 5)], + ), + ) + .unwrap(); + catalog + .insert_shard( + shard_hash.hex(), + ShardCatalogEntry::new(1, vec!["a".repeat(64)]), + ) + .unwrap(); + catalog + .insert_file(file_hash.hex(), FileCatalogEntry::new(5, shard_hash.hex())) + .unwrap(); + let mut previous = None; + let mut session = None; + let mut original = None; + for sequence in 0..=CAPSULE_REF_COMPACTION_FAN_IN { + let oid = format!("{:040x}", sequence + 1); + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + previous.clone(), + Some(oid.clone()), + None, + )], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/main".to_owned(), + GitVisibilityEdit::from_replacement_objects( + previous.clone(), + oid.clone(), + vec![oid.clone()], + ), + )])) + .unwrap(); + let mut sections = vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + )]; + if sequence == 0 { + sections.push(CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )); + } + let capsule = Capsule::build(&transaction, Vec::new(), sections).unwrap(); + crab_write::capsule_protocol::publish(&layout, root.clone(), &transaction, &capsule) + .await + .unwrap(); + previous = Some(oid); + if sequence == 0 { + let captured = view(&layout).await; + let path = layout.capsule_path(captured.capsule_run_pointers()[0].hash()); + backend.paths.lock().unwrap().insert(path.clone()); + original = Some(path); + // Opening the canonical session captures controls only. Its first + // selected catalog-page query must survive source replacement and GC. + session = Some( + crab_metadata::file_index_lookup::FileIndexLookupSession::open_for_storage( + layout.store(), + layout.repo_prefix(), + ) + .await + .unwrap(), + ); + } + } + let current = view(&layout).await; + let original = original.unwrap(); + assert!( + current + .capsule_run_pointers() + .iter() + .all(|pointer| layout.capsule_path(pointer.hash()) != original) + ); + assert!( + crab_metadata::capsule_protocol::load_browse_indexes(&layout) + .await + .unwrap() + .is_none() + ); + let snapshot = current.git_snapshot().unwrap(); + sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + current.root().root(), + current.capsule_run_pointers(), + &snapshot.manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now(), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + let session = session.unwrap(); + let result = session.lookup_batch(&[file_hash, file_hash]).await; + session.close().await.unwrap(); + assert!( + matches!(result, Ok(ref values) if values == &[Some(shard_hash), Some(shard_hash)]), + "cold catalog capture lost after real run compaction and GC: {result:?}" + ); +} + #[tokio::test] async fn capsule_gc_lists_each_candidate_namespace_once() { let backend = Arc::new(AgedObjects { diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index 1f4a279f8..ce5fd23b6 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3265,6 +3265,18 @@ async fn sweep_capsule_objects( store.list_prefix(&browse_root_prefix), store.list_prefix(&manifests_prefix), )?; + let source_head_prefix = format!("{}/", layout.capsule_source_heads_prefix()); + let source_heads = history + .iter() + .filter(|object| object.location.as_ref().starts_with(&source_head_prefix)) + .map(|object| object.location.clone()) + .collect::>(); + // Source lifetime is established by publishers, independent of optional + // browse readiness. Reuse the history enumeration rather than add another LIST. + let retained = + crab_metadata::capsule_protocol::load_retained_capsule_sources(layout, &source_heads) + .await?; + reachable.extend(retained.paths().iter().map(ToString::to_string)); // One metadata scan covers derived index bodies. Lifetime roots belong to // protocol control; unrelated metadata never becomes a deletion candidate. let derived_index_objects = metadata_objects diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index ac82e116e..0aa66055d 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -468,8 +468,7 @@ async fn prune_history( // Retry capture cleanup after an earlier history CAS committed but // its derived-retention update failed. Empty history has no cutoff. if let Some(minimum) = minimum { - crab_metadata::capsule_protocol::prune_retained_browse_indexes(layout, minimum) - .await?; + prune_capture_sources(layout, minimum).await?; } return Ok(payload); } @@ -497,12 +496,7 @@ async fn prune_history( let (primary, release_base) = match replacement { Ok(root) => { let cleanup = match minimum { - Some(minimum) => { - crab_metadata::capsule_protocol::prune_retained_browse_indexes( - layout, minimum, - ) - .await - } + Some(minimum) => prune_capture_sources(layout, minimum).await, None => Ok(()), }; (cleanup.err().map(CrabError::from), root) @@ -527,6 +521,16 @@ async fn prune_history( } } +async fn prune_capture_sources( + layout: &crab_storage::StoreLayout, + minimum_generation: u64, +) -> crab_metadata::error::Result<()> { + crab_metadata::capsule_protocol::prune_retained_browse_indexes(layout, minimum_generation) + .await?; + crab_metadata::capsule_protocol::prune_retained_capsule_sources(layout, minimum_generation) + .await +} + fn prune_payload(chain: &[HistorySegment], keep_last: usize, applied: bool) -> HistoryPrunePayload { let pruned = chain.iter().skip(keep_last).collect::>(); HistoryPrunePayload { diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index 24cdac161..88963550d 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -2047,6 +2047,7 @@ impl From for CrabError { | crab_metadata::error::MetadataError::CapsuleContract { .. } | crab_metadata::error::MetadataError::PathStateSize { .. } | crab_metadata::error::MetadataError::BrowseIndexRecord { .. } + | crab_metadata::error::MetadataError::SourceRetentionRecord { .. } | crab_metadata::error::MetadataError::GitObjectIndexRecord { .. } | crab_metadata::error::MetadataError::CapsuleVisibilityRecord { .. } | crab_metadata::error::MetadataError::PointerCatalogRecord { .. } diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 961517e1c..62c676373 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -139,6 +139,17 @@ length. Pruning is the boundary that can invalidate captures from discarded generations; ordinary rebuilding, checkpointing and GC do not expire readers. The new lifetime format is version 1; no release tag contains capsule browse readiness, and its existing version 3 encoding remains unchanged. +Compaction also retains replaced capsule sources independently of browse +readiness. Before publishing a replacement frontier, the ref owner CAS-publishes +a per-ref head under `v2/history/source-heads/` and authenticated source-pointer +segments under `v2/history/source-roots/`. Foreground catalog sessions stay +read-only and keep their original source ranges. GC discovers these roots in its +existing history enumeration; fsck authenticates their original run proofs. +Explicit history pruning advances a persistent generation floor after history +CAS, trims older segments, and prevents expired publishers from restoring them. +These new lifetime records do not change capsule, checkpoint, or catalog formats. +They cover sources retired by updated publishers; backfill/rollout qualification +for previously compacted unreleased repositories remains a release gate. Rebuilding an immutable index verifies the generated content identity and repairs corrupt stored bytes only with a conditional update against the observed version, followed by readback. diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 0f9bda89d..d456fb982 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -21,6 +21,8 @@ mod ref_view; mod root; mod run; #[cfg(feature = "storage")] +mod source_retention; +#[cfg(feature = "storage")] mod store; mod transaction; mod transaction_record; @@ -89,6 +91,11 @@ pub use run::{ #[cfg(feature = "storage")] pub use run::{CapsuleRunAdmissionReader, RunCatalogSource}; #[cfg(feature = "storage")] +pub use source_retention::{ + RetainedCapsuleSources, load_retained_capsule_sources, prune_retained_capsule_sources, + retain_capsule_sources, +}; +#[cfg(feature = "storage")] pub use store::{ RootSnapshot, capture_ref_heads, create_root, load_capsule_run, load_capsule_run_control, load_capsule_run_git_control, load_history_chain, load_history_segment, diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention.rs b/crates/crab-metadata/src/capsule_protocol/source_retention.rs new file mode 100644 index 000000000..65bf1df2e --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/source_retention.rs @@ -0,0 +1,445 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crab_storage::{ETag, Store, StoreLayout}; +use object_store::path::Path; +use serde::{Deserialize, Serialize}; + +use super::{ + CapsulePointer, MAX_CAPSULES_PER_RUN, MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, + MAX_HISTORY_SEGMENT_BYTES, +}; +use crate::error::{MetadataError, Result}; +use crate::validation::{corrupt_object, validate_content_hash}; + +const MAX_HEAD_BYTES: u64 = 4096; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Floor { + version: u32, + minimum_generation: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Tip { + hash: String, + roots: usize, + bytes: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Head { + version: u32, + ref_key: String, + minimum_generation: u64, + tip: Option, +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Segment { + version: u32, + ref_key: String, + generation: u64, + sources: Vec, + previous: Option, +} + +/// Authenticated lifetime dependencies; they never authorize ref or object visibility. +pub struct RetainedCapsuleSources { + paths: Vec, + runs: Vec, +} + +impl RetainedCapsuleSources { + /// Return live history heads, immutable segments and original source bodies for GC. + #[must_use] + pub fn paths(&self) -> &[Path] { + &self.paths + } + + /// Return complete original source proofs for maintenance integrity checks. + #[must_use] + pub fn runs(&self) -> &[CapsulePointer] { + &self.runs + } +} + +impl Tip { + fn append(previous: Option<&Self>, hash: String, bytes: usize) -> Result { + let tip = Self { + hash, + roots: previous.map_or(0, |tip| tip.roots).saturating_add(1), + bytes: previous + .map_or(0, |tip| tip.bytes) + .saturating_add(bytes as u64), + }; + tip.validate()?; + Ok(tip) + } + + fn validate(&self) -> Result<()> { + validate_content_hash(&self.hash, "source history root", "source retention")?; + if self.roots == 0 + || self.roots > MAX_HISTORY_CHAIN_SEGMENTS + || self.bytes == 0 + || self.bytes > MAX_HISTORY_CHAIN_BYTES + { + return Err(corrupt("source history exceeds resource bounds")); + } + Ok(()) + } +} + +impl Segment { + fn validate(&self, head: &Head) -> Result<()> { + if self.version != 1 + || self.ref_key != head.ref_key + || self.generation < head.minimum_generation + || self.sources.is_empty() + { + return Err(corrupt("source segment differs from lifetime head")); + } + let mut hashes = BTreeSet::new(); + let mut count = 0usize; + for source in &self.sources { + super::root::validate_capsule_pointer(source)?; + if !hashes.insert(source.hash()) { + return Err(corrupt("source segment repeats an immutable source")); + } + count = count.saturating_add(source.capsule_count() as usize); + } + if count > MAX_CAPSULES_PER_RUN { + return Err(corrupt("source segment exceeds compaction intake bound")); + } + if let Some(hash) = &self.previous { + validate_content_hash(hash, "source history predecessor", "source retention")?; + } + Ok(()) + } +} + +async fn load_floor(layout: &StoreLayout) -> Result> { + let (bytes, etag) = match layout + .store() + .get_with_etag_bounded(&layout.capsule_source_floor_path(), MAX_HEAD_BYTES) + .await + { + Ok(value) => value, + Err(crab_storage::StorageError::NotFound { .. }) => return Ok(None), + Err(error) => return Err(error.into()), + }; + let floor: Floor = serde_json::from_slice(&bytes).map_err(record_error)?; + if floor.version != 1 { + return Err(corrupt("unsupported source lifetime floor")); + } + Ok(Some((floor.minimum_generation, etag))) +} + +async fn admit_generation(layout: &StoreLayout, generation: u64) -> Result { + let floor = load_floor(layout).await?.map_or(0, |(floor, _)| floor); + if generation < floor { + return Err(corrupt( + "source retirement precedes explicit history pruning", + )); + } + Ok(floor) +} + +async fn advance_floor(layout: &StoreLayout, minimum_generation: u64) -> Result { + let previous = load_floor(layout).await?; + if let Some((floor, _)) = &previous + && *floor >= minimum_generation + { + return Ok(*floor); + } + let bytes = serde_json::to_vec(&Floor { + version: 1, + minimum_generation, + }) + .map_err(record_error)?; + let path = layout.capsule_source_floor_path(); + match previous { + Some((_, etag)) => { + layout.store().update(&path, bytes.into(), etag).await?; + } + None => { + layout + .store() + .create_strict_with_etag(&path, bytes.into()) + .await?; + } + } + Ok(minimum_generation) +} + +async fn load_head(layout: &StoreLayout, path: &Path) -> Result> { + let (bytes, etag) = match layout + .store() + .get_with_etag_bounded(path, MAX_HEAD_BYTES) + .await + { + Ok(value) => value, + Err(crab_storage::StorageError::NotFound { .. }) => return Ok(None), + Err(error) => return Err(error.into()), + }; + let head: Head = serde_json::from_slice(&bytes).map_err(record_error)?; + validate_content_hash(&head.ref_key, "source history ref key", "source retention")?; + if head.version != 1 || *path != layout.capsule_source_head_path(&head.ref_key) { + return Err(corrupt("source lifetime head has an invalid identity")); + } + if let Some(tip) = &head.tip { + tip.validate()?; + } + Ok(Some((head, etag))) +} + +async fn publish_head(layout: &StoreLayout, head: &Head, etag: Option) -> Result<()> { + let bytes = serde_json::to_vec(head).map_err(record_error)?; + if bytes.len() as u64 > MAX_HEAD_BYTES { + return Err(corrupt("source head exceeds byte bound")); + } + let path = layout.capsule_source_head_path(&head.ref_key); + match etag { + Some(etag) => { + layout.store().update(&path, bytes.into(), etag).await?; + } + None => { + layout + .store() + .create_strict_with_etag(&path, bytes.into()) + .await?; + } + } + Ok(()) +} + +async fn read_segment( + layout: &StoreLayout, + head: &Head, + hash: &str, + maximum: u64, +) -> Result<(Segment, usize)> { + let expected = + blake3::Hash::from_hex(hash).map_err(|_| corrupt("invalid source segment identity"))?; + let bytes = layout + .store() + .verify_bounded( + &layout.capsule_source_segment_path(hash), + expected.as_bytes(), + maximum.min(MAX_HISTORY_SEGMENT_BYTES), + ) + .await?; + let segment: Segment = serde_json::from_slice(&bytes).map_err(record_error)?; + segment.validate(head)?; + Ok((segment, bytes.len())) +} + +async fn write_segment( + layout: &StoreLayout, + head: &Head, + segment: &Segment, + previous: Option<&Tip>, +) -> Result { + segment.validate(head)?; + let bytes = serde_json::to_vec(segment).map_err(record_error)?; + if bytes.len() as u64 > MAX_HISTORY_SEGMENT_BYTES { + return Err(corrupt("source segment exceeds byte bound")); + } + let hash = blake3::hash(&bytes).to_hex().to_string(); + let tip = Tip::append(previous, hash, bytes.len())?; + layout + .store() + .put_if_absent_verified(&layout.capsule_source_segment_path(&tip.hash), bytes.into()) + .await?; + Ok(tip) +} + +async fn read_chain(layout: &StoreLayout, head: &Head) -> Result> { + let Some(tip) = &head.tip else { + return Ok(Vec::new()); + }; + let mut next = Some(tip.hash.clone()); + let mut remaining = tip.bytes; + let mut records = Vec::new(); + let mut seen = BTreeSet::new(); + while let Some(hash) = next { + if records.len() == tip.roots || !seen.insert(hash.clone()) { + return Err(corrupt("source history cycle/count mismatch")); + } + let (segment, bytes) = read_segment(layout, head, &hash, remaining).await?; + remaining = remaining + .checked_sub(bytes as u64) + .ok_or_else(|| corrupt("source history exceeds byte bound"))?; + next = segment.previous.clone(); + records.push((hash, segment)); + } + if remaining != 0 || records.len() != tip.roots { + return Err(corrupt("source history accounting mismatch")); + } + Ok(records) +} + +/// Retain replaced physical sources before a compacted ref frontier becomes visible. +/// +/// Caller owns the edited ref lease and GC writer fences. Per-ref CAS prevents +/// expired publishers from overwriting newer lifetime/pruning authority. +pub async fn retain_capsule_sources( + layout: &StoreLayout, + ref_name: &str, + generation: u64, + sources: &[CapsulePointer], +) -> Result<()> { + if sources.is_empty() { + return Ok(()); + } + if !ref_name.starts_with("refs/") || !super::valid_ref_name(ref_name) { + return Err(corrupt("source retention requires a valid ref name")); + } + // Ref-head keys encode the full name. Lifetime heads need only a bounded + // per-ref identity, so long ref names cannot expand these control records. + let ref_key = blake3::hash(ref_name.as_bytes()).to_hex().to_string(); + let minimum_generation = admit_generation(layout, generation).await?; + let path = layout.capsule_source_head_path(&ref_key); + let previous = load_head(layout, &path).await?; + let (mut head, etag) = match previous { + Some((head, etag)) => (head, Some(etag)), + None => ( + Head { + version: 1, + ref_key, + minimum_generation, + tip: None, + }, + None, + ), + }; + let mut segment = Segment { + version: 1, + ref_key: head.ref_key.clone(), + generation, + sources: sources.to_vec(), + previous: head.tip.as_ref().map(|tip| tip.hash.clone()), + }; + if let Some(tip) = &head.tip { + let (old, _) = read_segment(layout, &head, &tip.hash, tip.bytes).await?; + segment.previous = old.previous.clone(); + if segment == old { + admit_generation(layout, generation).await?; + return Ok(()); + } + segment.previous = Some(tip.hash.clone()); + } + head.tip = Some(write_segment(layout, &head, &segment, head.tip.as_ref()).await?); + publish_head(layout, &head, etag).await?; + // A formerly unseen ref can race pruning after its first floor read. + // Recheck before publication; GC also excludes bodies below the global floor. + admit_generation(layout, generation).await?; + Ok(()) +} + +/// Authenticate the supplied per-ref lifetime heads and their complete dependency closure. +pub async fn load_retained_capsule_sources( + layout: &StoreLayout, + heads: &[Path], +) -> Result { + let mut paths = BTreeSet::new(); + let mut runs = BTreeMap::new(); + let mut roots = 0usize; + let mut bytes = 0u64; + let floor = load_floor(layout).await?; + if floor.is_some() { + paths.insert(layout.capsule_source_floor_path()); + } + let minimum_generation = floor.map_or(0, |(floor, _)| floor); + for path in heads { + if !paths.insert(path.clone()) { + continue; + } + let (head, _) = load_head(layout, path) + .await? + .ok_or_else(|| corrupt("source lifetime head disappeared"))?; + if head.minimum_generation > minimum_generation { + return Err(corrupt("source head exceeds the repository pruning floor")); + } + if let Some(tip) = &head.tip { + roots = roots.saturating_add(tip.roots); + bytes = bytes.saturating_add(tip.bytes); + if roots > MAX_HISTORY_CHAIN_SEGMENTS || bytes > MAX_HISTORY_CHAIN_BYTES { + return Err(corrupt("aggregate source history exceeds resource bounds")); + } + } + for (hash, segment) in read_chain(layout, &head).await? { + paths.insert(layout.capsule_source_segment_path(&hash)); + if segment.generation < minimum_generation { + continue; + } + for source in segment.sources { + paths.insert(layout.capsule_path(source.hash())); + if runs + .insert(source.hash().to_owned(), source.clone()) + .is_some_and(|old| old != source) + { + return Err(corrupt( + "source histories name conflicting immutable source proofs", + )); + } + } + } + } + Ok(RetainedCapsuleSources { + paths: paths.into_iter().collect(), + runs: runs.into_values().collect(), + }) +} + +/// Prune superseded sources only after authoritative history pruning commits. +/// +/// Caller holds the exclusive repository GC lease. The persistent generation +/// floor prevents late publishers from reviving explicitly discarded captures. +pub async fn prune_retained_capsule_sources( + layout: &StoreLayout, + minimum_generation: u64, +) -> Result<()> { + // Record the policy before enumerating heads. This also fences old + // publishers whose refs have never created a source-lifetime head yet. + let minimum_generation = advance_floor(layout, minimum_generation).await?; + let heads = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await?; + for meta in heads { + let (mut head, etag) = load_head(layout, &meta.location) + .await? + .ok_or_else(|| corrupt("source head disappeared during pruning"))?; + if head.minimum_generation >= minimum_generation { + continue; + } + let records = read_chain(layout, &head).await?; + head.minimum_generation = minimum_generation; + let mut tip = None; + for (_, mut segment) in records + .into_iter() + .rev() + .filter(|(_, segment)| segment.generation >= minimum_generation) + { + segment.previous = tip.as_ref().map(|tip: &Tip| tip.hash.clone()); + tip = Some(write_segment(layout, &head, &segment, tip.as_ref()).await?); + } + head.tip = tip; + publish_head(layout, &head, Some(etag)).await?; + } + Ok(()) +} + +fn record_error(source: serde_json::Error) -> MetadataError { + MetadataError::SourceRetentionRecord { source } +} +fn corrupt(reason: &str) -> MetadataError { + corrupt_object("capsule source retention", reason) +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs new file mode 100644 index 000000000..62d06c3d0 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs @@ -0,0 +1,291 @@ +use std::sync::Arc; + +use bytes::Bytes; +use object_store::memory::InMemory; + +use super::*; +use crate::capsule_protocol::{Capsule, CapsuleRefEdit, CapsuleRun, CapsuleTransaction}; + +fn fixture() -> StoreLayout { + StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "source-lifetime".into(), + ) +} + +async fn source(layout: &StoreLayout, sequence: u64) -> CapsulePointer { + let transaction = CapsuleTransaction::new( + &"a".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(format!("{sequence:040x}")), + None, + )], + ) + .unwrap(); + let run = + CapsuleRun::leaf(Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap()).unwrap(); + layout + .store() + .put(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap() +} + +async fn heads(layout: &StoreLayout) -> Vec { + layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await + .unwrap() + .into_iter() + .map(|meta| meta.location) + .collect() +} + +#[tokio::test] +async fn repeated_retirement_does_not_append_or_rewrite_lifetime() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + std::slice::from_ref(&pointer), + ) + .await + .unwrap(); + let path = heads(&layout).await.remove(0); + let before = layout.store().get_with_etag(&path).await.unwrap(); + retain_capsule_sources(&layout, "refs/heads/main", 0, &[pointer]) + .await + .unwrap(); + assert_eq!(layout.store().get_with_etag(&path).await.unwrap(), before); +} + +#[tokio::test] +async fn independent_ref_heads_share_source_identity_without_sharing_cas() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + for name in ["refs/heads/main", "refs/heads/other"] { + retain_capsule_sources(&layout, name, 0, std::slice::from_ref(&pointer)) + .await + .unwrap(); + } + let paths = heads(&layout).await; + let retained = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + assert_eq!((paths.len(), retained.runs()), (2, [pointer].as_slice())); +} + +#[tokio::test] +async fn pruning_relinks_kept_generations_and_rejects_late_old_publishers() { + let layout = fixture(); + let mut sources = Vec::new(); + for generation in 0..3 { + let pointer = source(&layout, generation + 1).await; + retain_capsule_sources( + &layout, + "refs/heads/main", + generation, + std::slice::from_ref(&pointer), + ) + .await + .unwrap(); + sources.push(pointer); + } + prune_retained_capsule_sources(&layout, 2).await.unwrap(); + let paths = heads(&layout).await; + let retained = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + assert_eq!(retained.runs(), &sources[2..]); + assert!( + retain_capsule_sources(&layout, "refs/heads/main", 0, &sources[..1]) + .await + .is_err() + ); + let before = layout.store().get_with_etag(&paths[0]).await.unwrap(); + prune_retained_capsule_sources(&layout, 2).await.unwrap(); + assert_eq!( + layout.store().get_with_etag(&paths[0]).await.unwrap(), + before + ); +} + +#[tokio::test] +async fn empty_pruned_history_keeps_its_generation_floor() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + std::slice::from_ref(&pointer), + ) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 1).await.unwrap(); + let paths = heads(&layout).await; + let retained = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + let mut expected = paths.clone(); + expected.push(layout.capsule_source_floor_path()); + expected.sort(); + assert_eq!( + (retained.paths(), retained.runs()), + (expected.as_slice(), [].as_slice()) + ); + assert!( + retain_capsule_sources(&layout, "refs/heads/main", 0, &[pointer]) + .await + .is_err() + ); +} + +#[tokio::test] +async fn pruning_rejects_old_publishers_for_previously_unseen_refs() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + prune_retained_capsule_sources(&layout, 1).await.unwrap(); + assert!( + retain_capsule_sources(&layout, "refs/heads/new", 0, std::slice::from_ref(&pointer)) + .await + .is_err() + ); + retain_capsule_sources(&layout, "refs/heads/new", 1, std::slice::from_ref(&pointer)) + .await + .unwrap(); + let retained = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + assert_eq!(retained.runs(), &[pointer]); +} + +#[tokio::test] +async fn late_first_head_cannot_restore_sources_below_the_repository_floor() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + std::slice::from_ref(&pointer), + ) + .await + .unwrap(); + let path = heads(&layout).await.remove(0); + let (old, _) = load_head(&layout, &path).await.unwrap().unwrap(); + layout.store().delete(&path).await.unwrap(); + prune_retained_capsule_sources(&layout, 1).await.unwrap(); + // Model an expired publisher that read the absent head before pruning, + // then won its first-head create after the pruner's enumeration. + publish_head(&layout, &old, None).await.unwrap(); + let retained = load_retained_capsule_sources(&layout, &[path]) + .await + .unwrap(); + assert!(retained.runs().is_empty()); + assert!( + !retained + .paths() + .contains(&layout.capsule_path(pointer.hash())) + ); +} + +#[tokio::test] +async fn missing_pruning_floor_rejects_newer_per_ref_authority() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + prune_retained_capsule_sources(&layout, 1).await.unwrap(); + retain_capsule_sources(&layout, "refs/heads/main", 1, &[pointer]) + .await + .unwrap(); + layout + .store() + .delete(&layout.capsule_source_floor_path()) + .await + .unwrap(); + assert!( + load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .is_err() + ); +} + +#[tokio::test] +async fn expired_publisher_cannot_overwrite_successor_lifetime() { + let layout = fixture(); + let first = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[first]) + .await + .unwrap(); + let path = heads(&layout).await.remove(0); + let (old, etag) = load_head(&layout, &path).await.unwrap().unwrap(); + let second = source(&layout, 2).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[second]) + .await + .unwrap(); + assert!(matches!( + publish_head(&layout, &old, Some(etag)).await, + Err(MetadataError::Storage { + source: crab_storage::StorageError::StateConflict { .. } + }) + )); +} + +#[tokio::test] +async fn missing_or_corrupt_source_segments_stop_dependency_discovery() { + for missing in [false, true] { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[pointer]) + .await + .unwrap(); + let paths = heads(&layout).await; + let (head, _) = load_head(&layout, &paths[0]).await.unwrap().unwrap(); + let path = layout.capsule_source_segment_path(&head.tip.unwrap().hash); + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + } + assert!( + load_retained_capsule_sources(&layout, &paths) + .await + .is_err() + ); + } +} + +#[test] +fn segment_admission_checks_chain_bounds_before_immutable_upload() { + for (roots, bytes) in [ + (MAX_HISTORY_CHAIN_SEGMENTS, 1), + (1, MAX_HISTORY_CHAIN_BYTES), + ] { + let tip = Tip { + hash: "a".repeat(64), + roots, + bytes, + }; + assert!(Tip::append(Some(&tip), "b".repeat(64), 1).is_err()); + } +} diff --git a/crates/crab-metadata/src/error.rs b/crates/crab-metadata/src/error.rs index 814919a6d..958e35629 100644 --- a/crates/crab-metadata/src/error.rs +++ b/crates/crab-metadata/src/error.rs @@ -21,6 +21,11 @@ pub enum MetadataError { #[source] source: serde_json::Error, }, + #[error("invalid capsule source-retention JSON")] + SourceRetentionRecord { + #[source] + source: serde_json::Error, + }, #[error("Git object index record encoding is invalid")] GitObjectIndexRecord { #[source] diff --git a/crates/crab-storage/src/layout.rs b/crates/crab-storage/src/layout.rs index 2e4641f8d..0cc04761c 100644 --- a/crates/crab-storage/src/layout.rs +++ b/crates/crab-storage/src/layout.rs @@ -228,6 +228,31 @@ impl StoreLayout { self.repo_path(&format!("v2/history/{partition}/{hash}")) } + /// Prefix of per-ref source-lifetime heads, separate from ref visibility. + #[must_use] + pub fn capsule_source_heads_prefix(&self) -> ObjectPath { + self.repo_path("v2/history/source-heads/") + } + + /// Path to the source-lifetime generation floor committed by explicit pruning. + #[must_use] + pub fn capsule_source_floor_path(&self) -> ObjectPath { + self.repo_path("v2/history/source-floor") + } + + /// Path to one per-ref source-lifetime head. + #[must_use] + pub fn capsule_source_head_path(&self, ref_key: &str) -> ObjectPath { + self.repo_path(&format!("v2/history/source-heads/{ref_key}.json")) + } + + /// Path to one immutable source-lifetime history segment. + #[must_use] + pub fn capsule_source_segment_path(&self, hash: &str) -> ObjectPath { + let partition = hash.get(..GLOBAL_CONTENT_FANOUT_WIDTH).unwrap_or(hash); + self.repo_path(&format!("v2/history/source-roots/{partition}/{hash}")) + } + /// Prefix containing independently mutable capsule-protocol ref heads. #[must_use] pub fn capsule_ref_heads_prefix(&self) -> ObjectPath { diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index fbe55b854..737bf42be 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -283,7 +283,14 @@ pub async fn prepare_publication_with_ref_head_bases( let run = capsule_leaf_run(capsule)?; let prepared = try_join_all(transaction.edits().iter().zip(snapshots).map( |(edit, snapshot)| { - prepare_ref_successor(router, snapshot, edit, &transaction_id, run.clone()) + prepare_ref_successor( + router, + snapshot, + edit, + &transaction_id, + run.clone(), + base.record().root().generation(), + ) }, )) .await?; @@ -560,6 +567,7 @@ pub async fn materialize_coordinated_repair( edit, &descriptor.transaction_id, run.clone(), + base.record().root().generation(), ) }, )) @@ -1093,6 +1101,7 @@ async fn prepare_ref_successor( edit: &crab_metadata::capsule_protocol::CapsuleRefEdit, transaction_id: &str, leaf: CapsuleRun, + generation: u64, ) -> Result<(PreparedRefHead, Option)> { let mut frontier = snapshot.visible.frontier().to_vec(); frontier.push(CapsulePointer::new( @@ -1106,6 +1115,28 @@ async fn prepare_ref_successor( leaf.newest_base_root_digest(), )?); let compacted = compact_ref_frontier(router, &mut frontier, &leaf).await?; + if compacted.is_some() { + let remaining = frontier + .iter() + .map(CapsulePointer::hash) + .collect::>(); + let retired = snapshot + .visible + .frontier() + .iter() + .filter(|pointer| !remaining.contains(pointer.hash())) + .cloned() + .collect::>(); + // Captured catalog/visibility readers still address the old immutable + // bodies. Publish their lifetime before any replacement ref CAS can hide them. + crab_metadata::capsule_protocol::retain_capsule_sources( + router, + edit.ref_name(), + generation, + &retired, + ) + .await?; + } let state = snapshot.visible.successor( edit.new_oid().map(str::to_owned), edit.peeled_oid().map(str::to_owned), @@ -2748,6 +2779,7 @@ mod tests { &transaction.edits()[0], &transaction_id, leaf, + base.record().root().generation(), ) .await .unwrap(); @@ -2872,9 +2904,27 @@ mod tests { .iter() .filter(|operation| **operation == StorageOperation::Put) .count(), - 3 + // Source lifetime adds one immutable segment and its per-ref CAS + // before the ref can stop exposing the original bodies to GC. + 5 ); + let source_heads = router + .store() + .list_prefix(&router.capsule_source_heads_prefix()) + .await + .unwrap() + .into_iter() + .map(|object| object.location) + .collect::>(); + let retained = + crab_metadata::capsule_protocol::load_retained_capsule_sources(&router, &source_heads) + .await + .unwrap(); + let mut expected_sources = prior_frontier.clone(); + expected_sources.sort_by(|left, right| left.hash().cmp(right.hash())); + assert_eq!(retained.runs(), expected_sources); + let head = read_ref_head(&router, published.record().root(), "refs/heads/main") .await .unwrap(); From 970e9cf8a6e62a7636d1d39c7a6caedc82c6ff9d Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:18:07 -0700 Subject: [PATCH 072/127] fix(fsck): verify retained browse dependencies --- crab/src/cmd/fsck_store.rs | 29 ++ crates/crab-metadata/README.md | 5 + .../src/capsule_protocol/browse_retention.rs | 88 +++++- .../browse_retention/tests.rs | 293 ++++++++++++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 2 + .../src/git_object_locator/paged/reader.rs | 30 ++ 6 files changed, 446 insertions(+), 1 deletion(-) diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 17c2cabd0..8dc5aae48 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -151,6 +151,18 @@ impl StoreChecker { } async fn check_capsule_data_chain(&self, state: &CapsuleFsckState) -> Result> { + let layout = crab_storage::StoreLayout::with_global_prefix( + self.store.as_storage().clone(), + self.router.repo_prefix().to_owned(), + self.router.global_prefix().to_owned(), + ); + // Current catalog health cannot prove cold captures still have their + // original sources. Full fsck checks lifetime roots; mirror recipe probes stay narrow. + crab_metadata::capsule_protocol::verify_retained_browse_indexes( + &layout, + MAX_FSCK_FRONTIER_BYTES, + ) + .await?; let mut issues = Vec::new(); for (hash, entry) in state.catalog.xorbs() { @@ -2060,6 +2072,23 @@ mod tests { assert!(shard_issues.is_empty()); } + #[tokio::test] + async fn capsule_data_check_rejects_corrupt_browse_lifetime() { + let (store, prefix, root, _, _) = capsule_checker_fixture().await; + let checker = StoreChecker::for_capsule_repository(store.clone(), prefix, root) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + store + .put( + &checker.router.repo_path("v2/browse-retention"), + bytes::Bytes::from_static(b"corrupt"), + ) + .await + .unwrap(); + assert!(checker.check_data_chain().await.is_err()); + } + #[tokio::test] async fn capsule_checker_validates_repository_without_legacy_layout() { let (store, prefix, root, _, _) = capsule_checker_fixture().await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 62c676373..69c34c83d 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -130,6 +130,11 @@ foreground reads still load only the current bounded readiness record. An unchanged capture does not append a node. Missing/corrupt lifetime dependencies stop GC before deletion. A valid readiness record without lifetime ownership requires background browse maintenance before GC; there is no scan fallback. +Full capsule fsck also authenticates the retained chain, decodes complete +graph/path-state indexes, and verifies physical-index and original source bytes, +including superseded ref-only runs. Shared captures stream each source body once +during this phase. Lifetime/readiness movement during verification requires a retry; +mirror pointer-recipe probes do not acquire this full maintenance workload. Explicit history pruning trims superseded captures below the oldest retained checkpoint generation only after the authoritative history CAS succeeds. Current diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 89a06c4b9..48b26afb3 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -11,7 +11,7 @@ use super::{ use crate::error::{MetadataError, Result}; use crate::validation::{corrupt_object, validate_content_hash}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Head { version: u32, @@ -285,6 +285,92 @@ pub async fn load_retained_browse_indexes( } } +/// Verify retained index/source integrity against stable lifetime and readiness heads. +/// +/// Maintenance decodes complete graph/path indexes and streams original source +/// and physical-index bodies. This proves retained bytes, never ref visibility. +#[cfg(feature = "remote-index")] +pub async fn verify_retained_browse_indexes( + layout: &StoreLayout, + maximum_bytes: u64, +) -> Result<()> { + let head = load_head(layout).await?; + let current = super::load_browse_indexes(layout).await?; + let records = match &head { + Some((head, _)) => load_chain(layout, head).await?, + None => Vec::new(), + }; + if current + .as_ref() + .is_some_and(|current| !records.iter().any(|record| record.indexes() == current)) + { + return Err(corrupt("published browse indexes have no retained capture")); + } + let mut sources = BTreeSet::new(); + for record in records { + let graph = crate::split_commit_graph::load_split_commit_graph( + layout.store(), + layout, + record.indexes().commit_graph_hash(), + maximum_bytes.min(crate::split_commit_graph::DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES), + ) + .await?; + if graph.descriptor.generation != record.generation() { + return Err(corrupt( + "retained graph generation differs from its capture", + )); + } + if let Some(hash) = record.indexes().path_state_hash() { + crate::path_state::load_path_state( + layout.store(), + layout, + hash, + &graph, + maximum_bytes.min(crate::path_state::DEFAULT_MAX_PATH_STATE_BYTES), + ) + .await?; + } + if let Some(hash) = record.indexes().object_index_hash() { + crate::git_object_locator::PagedGitObjectIndex::verify_retained( + layout.store(), + layout, + hash, + record.indexes().state_digest(), + &graph.descriptor, + maximum_bytes, + ) + .await?; + } + // Captures may share original multi-gigabyte sources. Authenticate each + // immutable body once, including ref-only sources absent from current history. + for path in record.source_paths(layout) { + if !sources.insert(path.clone()) { + continue; + } + let size = layout.store().head(&path).await?.size; + if size > maximum_bytes { + return Err(corrupt("retained source exceeds byte limit")); + } + let hash = path + .filename() + .and_then(|hash| blake3::Hash::from_hex(hash).ok()) + .ok_or_else(|| corrupt("retained source path has no content identity"))?; + layout + .store() + .verify_size_and_hash(&path, size, hash.as_bytes()) + .await?; + } + } + // Browse maintenance can change lifetime without moving a Git ref. A + // concurrent prune/rebuild must yield a retry, never a clean mixed capture. + if load_head(layout).await? != head || super::load_browse_indexes(layout).await? != current { + return Err(corrupt( + "browse retention changed during verification; retry fsck", + )); + } + Ok(()) +} + /// Trim captured generations after authoritative history pruning commits. /// /// The caller holds the exclusive repository GC lease. Failure leaves extra diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index 61412516f..e5fb4d855 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -15,6 +15,299 @@ fn indexes(byte: char) -> BrowseIndexes { BrowseIndexes::new(byte.to_string().repeat(64), "b".repeat(64), None).unwrap() } +#[cfg(feature = "remote-index")] +async fn integrity_fixture() -> (StoreLayout, Vec) { + use super::super::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleRun, CapsuleTransaction, PackLayer, + PointerCatalog, + }; + use crate::git_object_locator::{ + GitObjectLocation, GitObjectLocator, GitObjectMetadata, GitPackInventoryEntry, + PagedGitObjectIndexWrite, + }; + use crate::path_state::{PathStateInput, append_path_state, upload_path_state}; + use crate::split_commit_graph::{ + CommitGraphInput, append_paged_commit_graph, load_split_commit_graph, + upload_split_commit_graph, + }; + + let layout = fixture(); + let mut manifest = crate::manifests::Manifest::default_for_repo("refs/heads/main"); + manifest.generation = 1; + let pack_id = crab_xet::hash::MerkleHash::from([3; 32]); + let inventory = [GitPackInventoryEntry { + pack_id, + object_count: 1, + pack_size: 42, + }]; + manifest.pack_index_hash = crate::manifests::compact_pack_index( + 1, + &[crate::manifests::PackManifestEntry { + pack_id: pack_id.to_string(), + size: 42, + content_hash: pack_id.to_string(), + ref_tips: Vec::new(), + object_count: 1, + }], + ) + .unwrap() + .0; + manifest.git_validation_digest = "b".repeat(64); + let graph = append_paged_commit_graph( + None, + 1, + manifest.pack_index_hash.clone(), + manifest.git_validation_digest.clone(), + &[[1; 20]], + vec![CommitGraphInput { + oid: [1; 20], + tree_oid: [2; 20], + commit_time: 1, + parents: Vec::new(), + }], + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(layout.store(), &layout, &graph) + .await + .unwrap(); + let graph_hash = graph.descriptor_hash.clone(); + let mut paths = vec![layout.repo_path(&graph.layers[0].reference.path)]; + let graph = load_split_commit_graph(layout.store(), &layout, &graph_hash, 1024 * 1024) + .await + .unwrap(); + let path_state = append_path_state( + None, + &graph, + vec![PathStateInput { + oid: [1; 20], + first_parent: None, + author: Vec::new(), + author_seconds: 1, + message: Vec::new(), + mutations: Vec::new(), + }], + ) + .unwrap(); + upload_path_state(layout.store(), &layout, &path_state, 1024 * 1024) + .await + .unwrap(); + paths.push(layout.repo_path(&path_state.layers[0].reference.path)); + let entries = [( + [1; 20], + GitObjectLocator { + ordinal: 0, + pack_id, + location: GitObjectLocation { + pack_offset: 12, + entry_len: 10, + crc32: 0, + }, + metadata: GitObjectMetadata::default(), + }, + )]; + let physical = + PagedGitObjectIndexWrite::build(&"a".repeat(64), &manifest, &inventory, &entries).unwrap(); + let hash = physical.upload(layout.store(), &layout).await.unwrap(); + paths.push( + crate::git_object_locator::PagedGitObjectIndex::retained_paths( + layout.store(), + &layout, + &hash, + &"a".repeat(64), + 1024 * 1024, + ) + .await + .unwrap()[1] + .clone(), + ); + let indexes = BrowseIndexes::new( + "a".repeat(64), + graph_hash.clone(), + Some(path_state.descriptor_hash), + ) + .unwrap() + .with_object_index(hash) + .unwrap(); + let transaction = CapsuleTransaction::new( + &"c".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("1".repeat(40)), + None, + )], + ) + .unwrap(); + let run = + CapsuleRun::leaf(Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap()).unwrap(); + let run_path = layout.capsule_path(run.hash()); + layout + .store() + .put(&run_path, run.bytes().clone()) + .await + .unwrap(); + paths.push(run_path); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from(vec![0x42; 1024]), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "3".repeat(40), + 1, + ) + .unwrap(); + let layer = PackLayer::build(&pack).unwrap(); + let layer_path = layout.capsule_pack_layer_path(layer.hash()); + layout + .store() + .put(&layer_path, layer.bytes().clone()) + .await + .unwrap(); + paths.push(layer_path); + let checkpoint = LayeredCheckpoint::build( + 1, + &"c".repeat(64), + vec![layer.source_descriptor().unwrap()], + PointerCatalog::new(), + None, + ) + .unwrap(); + let checkpoint_path = layout.capsule_checkpoint_path(checkpoint.hash()); + layout + .store() + .put(&checkpoint_path, checkpoint.bytes().clone()) + .await + .unwrap(); + paths.push(checkpoint_path); + retain_browse_indexes(&layout, &indexes, 1, Some(&checkpoint), &[pointer]) + .await + .unwrap(); + // The old bodies now belong only to retained capture lifetime, not the + // latest readiness or any authoritative checkpoint-history chain. + let latest = BrowseIndexes::new("d".repeat(64), graph_hash, None).unwrap(); + retain_browse_indexes(&layout, &latest, 1, None, &[]) + .await + .unwrap(); + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + latest.encode().unwrap(), + ) + .await + .unwrap(); + (layout, paths) +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn integrity_checks_every_retained_body_after_readiness_replacement() { + let (layout, paths) = integrity_fixture().await; + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); + for path in paths { + let original = layout.store().get_with_etag(&path).await.unwrap().0; + for missing in [false, true] { + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + } + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err(), + "missing={missing}, path={path}" + ); + layout + .store() + .put_overwrite(&path, original.clone()) + .await + .unwrap(); + } + } + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn advertised_readiness_requires_authenticated_lifetime_ownership() { + let layout = fixture(); + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + indexes('a').encode().unwrap(), + ) + .await + .unwrap(); + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err() + ); +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn retained_physical_index_must_match_its_graph_and_state() { + for field in [ + "state_digest", + "generation", + "pack_index_hash", + "git_validation_digest", + ] { + let (layout, _) = integrity_fixture().await; + let records = load_retained_browse_indexes(&layout).await.unwrap(); + let indexes = records[1].indexes(); + let path = + layout.bulk_manifest_path("git-object-index", indexes.object_index_hash().unwrap()); + let bytes = layout.store().get_with_etag(&path).await.unwrap().0; + let mut descriptor: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + descriptor[field] = if field == "generation" { + serde_json::json!(2) + } else { + serde_json::json!("e".repeat(64)) + }; + let bytes = Bytes::from(serde_json::to_vec(&descriptor).unwrap()); + let hash = blake3::hash(&bytes).to_hex().to_string(); + layout + .store() + .put(&layout.bulk_manifest_path("git-object-index", &hash), bytes) + .await + .unwrap(); + let indexes = indexes.clone().with_object_index(hash).unwrap(); + retain_browse_indexes(&layout, &indexes, 1, None, &[]) + .await + .unwrap(); + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err(), + "field={field}" + ); + } +} + #[tokio::test] async fn unchanged_capture_does_not_grow_retention_or_rewrite_its_head() { let layout = fixture(); diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index d456fb982..17b51aaab 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -31,6 +31,8 @@ mod visibility; #[cfg(feature = "storage")] pub use browse::load_browse_indexes; pub use browse::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES}; +#[cfg(feature = "remote-index")] +pub use browse_retention::verify_retained_browse_indexes; #[cfg(feature = "storage")] pub use browse_retention::{ RetainedBrowseIndexes, load_retained_browse_indexes, prune_retained_browse_indexes, diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index f26164933..2326d40e4 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -53,6 +53,36 @@ impl PagedGitObjectIndex { ]) } + /// Authenticate a retained body's bytes and graph binding without granting visibility. + /// + /// The caller supplies an authenticated graph descriptor from the same capture. + pub async fn verify_retained( + store: &Store, + layout: &StoreLayout, + hash: &str, + state_digest: &str, + graph: &crate::split_commit_graph::CommitGraphDescriptor, + maximum: u64, + ) -> Result<()> { + let (descriptor, _) = Self::descriptor(store, layout, hash, maximum).await?; + if descriptor.state_digest != state_digest + || descriptor.generation != graph.generation + || descriptor.pack_index_hash != graph.pack_index_hash + || descriptor.git_validation_digest != graph.git_validation_digest + || descriptor.body_bytes > maximum + { + return bad("retained object index does not match graph or byte limit"); + } + store + .verify_size_and_hash( + &body_path(layout, &descriptor.body_hash), + descriptor.body_bytes, + &hash_bytes(&descriptor.body_hash)?, + ) + .await?; + Ok(()) + } + async fn descriptor( store: &Store, layout: &StoreLayout, From 7c71656c5739be0a679a0da91da4340f8fa8a535 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 03:48:42 -0700 Subject: [PATCH 073/127] perf(add): skip absent-index cache writes An unopened global index cannot supply per-chunk absence facts. Keep that state in the existing OnceCell instead of persisting a negative row for every chunk in a new file. Preserve initialized-index cache policy, cancellation, close ownership and push-side proof validation. --- crab/docs/design/add.md | 5 ++ crab/src/git/push.rs | 103 ++++++++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index eda232407..d706ec4b7 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -224,6 +224,11 @@ repacking; push still revalidates every positive placement and origin proof. Cache failures, stale entries, and evictions are advisory misses; they cannot change the bytes selected for a push. +An uninitialized global chunk index is remembered only for the current add +session. It does not establish per-chunk absence, so it does not populate the +persistent negative cache. Existing index snapshots keep their normal cache +behavior; unavailable remote authority still leaves payload ownership local. + Push reads remote candidates in bounded pages under a shared deadline. It retains completed pages and local cache hits if the deadline expires or a later remote page fails; every retained placement still crosses normal proof diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index d9e7b97fe..c0a87fb0e 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -4494,17 +4494,23 @@ impl crab_staging::push_plan::ExistingChunkLookup for AddRemoteChunkClassifier { "remote add classifier failed: {error}" )), })?; + // Keep index absence in the session's OnceCell: persisting one miss + // per chunk makes cache writes scale with the entire new file. + let cache = self + .candidate_cache + .as_ref() + .filter(|_| matches!(self.remote_chunk_index.get(), Some(Some(_)))); let mut updates = Vec::with_capacity(misses.len()); for chunk_hash in misses { let candidate = fetched.get(&chunk_hash).copied(); - if self.candidate_cache.is_some() { + if cache.is_some() { updates.push((chunk_hash, candidate)); } if let Some(candidate) = candidate { candidates.insert(chunk_hash, candidate); } } - if let Some(cache) = &self.candidate_cache { + if let Some(cache) = cache { if let Err(error) = cache.memory_insert_batch(&updates) { warn!(error = %error, "add remote candidate memory cache update failed"); } @@ -22045,16 +22051,22 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn add_classifier_does_not_repeat_an_uninitialized_index_probe() { - let inner = Arc::new(object_store::memory::InMemory::new()); - let reads = Arc::new(Mutex::new(Vec::new())); - let recording_store: Arc = Arc::new(RecordingReadStore { - inner, - reads: Arc::clone(&reads), - }); - let store = Store::new(recording_store); - let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); + fn add_classifier_with_private_cache( + store: &Store, + router: &StoreLayout, + cache_root: &std::path::Path, + ) -> ( + AddRemoteChunkClassifier, + Arc, + ) { + let cache_root = cache_root.join("cache"); + crab_cache::ensure_private_cache_directory(&cache_root).expect("private cache root"); + let candidate_cache = Arc::new( + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( + &cache_root.join("candidates.sqlite"), + ) + .expect("open candidate cache"), + ); let pipeline = PushPipeline::new( PushConfig::default(), Vec::new(), @@ -22067,20 +22079,32 @@ mod tests { CancellationToken::new(), None, ); - pipeline.install_metadb(build_push_metadb_guard( - &store, - &router, - None, - &crate::core::config::MetaDbTomlConfig::default(), - true, - )); + let mut config = crate::core::config::MetaDbTomlConfig::default(); + config.chunk_index.local_path = Some(cache_root.join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard(store, router, None, &config, true)); let classifier = AddRemoteChunkClassifier { pipeline, remote_chunk_index: tokio::sync::OnceCell::new(), - candidate_cache: None, + candidate_cache: Some(Arc::clone(&candidate_cache)), candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), }; + (classifier, candidate_cache) + } + + #[tokio::test(flavor = "multi_thread")] + async fn add_classifier_keeps_uninitialized_index_absence_session_scoped() { + let cache_root = tempfile::tempdir().expect("private candidate cache"); + let inner = Arc::new(object_store::memory::InMemory::new()); + let reads = Arc::new(Mutex::new(Vec::new())); + let recording_store: Arc = Arc::new(RecordingReadStore { + inner, + reads: Arc::clone(&reads), + }); + let store = Store::new(recording_store); + let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); + let (classifier, candidate_cache) = + add_classifier_with_private_cache(&store, &router, cache_root.path()); let first = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( &classifier, &[(MerkleHash::from([1; 32]), 4)], @@ -22118,6 +22142,45 @@ mod tests { assert!(matches!(cancelled, crab_staging::StagingError::Cancelled)); assert_eq!(manifest_probes(), initial_probes); classifier.close().await; + assert!( + candidate_cache + .load_persistent(&[MerkleHash::from([1; 32]), MerkleHash::from([2; 32])]) + .expect("inspect persisted candidates") + .is_empty(), + "an absent index must not write a per-chunk negative cache" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn add_classifier_persists_misses_from_an_initialized_index() { + let cache_root = tempfile::tempdir().expect("private candidate cache"); + let (store, router) = test_store_router("repo-add-initialized-index"); + let seed = build_push_metadb_guard( + &store, + &router, + None, + &crate::core::config::MetaDbTomlConfig::default(), + false, + ); + seed.chunk_index_system_keys().await.expect("seed manifest"); + seed.close().await.expect("close seed writer"); + let (classifier, candidate_cache) = + add_classifier_with_private_cache(&store, &router, cache_root.path()); + let hash = MerkleHash::from([4; 32]); + let lookup = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( + &classifier, + &[(hash, 4)], + ) + .await; + classifier.close().await; + lookup.expect("initialized-index lookup"); + assert_eq!( + candidate_cache + .load_persistent(&[hash]) + .expect("inspect persisted miss") + .get(&hash), + Some(&None) + ); } #[tokio::test(flavor = "multi_thread")] From e245c4b144323b3c4a829414089b0d509099d16a Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 00:43:38 -0700 Subject: [PATCH 074/127] test(read): qualify captured catalog lifecycle Prove cold catalog pages survive checkpoint replacement and a real orphan sweep. Repair the async visibility test caller and carry forward the qualified staging-version fixture. Record the remaining shallow-date adapter failure without weakening its CLI smoke assertion. --- .../docs/architecture/indexed-remote-reads.md | 86 +++++++- .../e2e/run_add_commit_push_rustfs_smoke.py | 13 +- crab/src/cmd/gc/capsule_cleanup_tests.rs | 194 ++++++++++++++++++ 3 files changed, 282 insertions(+), 11 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 1b57dd83b..a13bda5eb 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -69,7 +69,9 @@ counts contain no raw paths, OIDs, endpoints, content or credentials; separate oracle evidence contains the paths/OIDs needed to prove correctness. `/usr/bin/time` supplies process CPU and peak RSS. Those cover opening and both -queries together; they are not per-query RSS. Index construction uses an explicit +queries together; they are not per-query RSS. Catalog qualifier totals also +include the complete inspection oracle, so they cannot establish selected-query +memory bounds. Index construction uses an explicit `build-indexes` operation, outside measured reads, with its own requests, bytes, CPU and peak RSS. Read-only qualifications do not trigger maintenance. @@ -2188,11 +2190,79 @@ two path heads, two leases, 65 payload records and 4,194,304 retained segment bytes remain. These receipts establish the oracle and staging component; they do not qualify the new integrated binary or catalog format. -Combined-source CLI compilation and runtime qualification are still pending. -Remaining catalog consumers include shared/protected preparation, pinned -hydration/mount lookup and the current-state file lookup opener. Complete +The combined release binary compiled and the focused RustFS smoke passed +163 commands and 84 checks, including exact retirement, committed restaging +before first push, both cold versions, force/ref updates and invalid roots. +The integrated native-first entropy 10-GiB run passes 75 commands and 57 checks: +all four pushes, exact retirement, eight cold historical SHA-256 checks, Git +integrity and remote fsck with zero errors or repairs. +Those receipts bind the earlier integrated source, not subsequent reader changes. + +Pinned hydration/mount lookup uses captured catalog pages in PR #581. PR #584 +extends this to current-state file lookup and shares ref ordering and captured +visibility with Git readers. Metadata tests pass (458), as do all 47 read capsule-protocol tests after +preserving corruption classification at the read boundary. CLI, runtime and +retained-root lifecycle checks are required before readiness. +A reproduced independent-ref fixture also requires complete inspection to +validate dependencies after assembling the visible set, while publication-time +delta validation remains strict. + +`qualify_remote ... catalog-current` measures the actual +current-state lookup session. The argument uses Xet `MerkleHash::hex()` encoding, +not the pointer wire format’s raw-byte hex. Hydration converts parsed pointer +bytes with `MerkleHash::from(pointer.file_hash)` before lookup; the qualifier +accepts that catalog identity directly. Opening, cold queries and warm queries are separate; the full +inspection oracle runs after measured queries and explicit session close. This +keeps unrelated catalog decoding out of cold-query preparation. Use a quiescent, +dedicated fixture so the post-query full oracle has the same visible authority. +Compare these +receipts across fixed small edits and increasing distinct historical chunks, +ref counts and uncompacted history; checkpoint row bounds alone are insufficient. + +Shared/protected preparation remains a separate integration boundary. Complete inspection remains appropriate for fsck, compaction and GC; foreground lookups -must not inherit those scans. Carry forward applicable CLI range/cancellation -contracts before final qualification. The complete integrated 10-GiB matrix, -history scaling, lifecycle/migration, fuller blame and representative S3 gates -remain open. +must not inherit those scans. CLI snapshot reads still install all Git packs; +carry forward the range/cancellation and cleanup contracts when removing that +cost. The final combined 10-GiB matrix, history scaling, lifecycle/migration, +fuller blame and representative S3 gates remain open. + +The GC regression passes with both existing cleanup tests (three tests total). +It captures a 20,000-file checkpoint catalog without a file query, replaces it, +and sweeps past the grace period. Actual orphan deletion is required; the old +reader then fetches cold pages and resolves duplicated file requests, while a +new session observes the replacement. Both checkpoints have a valid physical +source descriptor/body. This is an in-memory metadata lifecycle proof, separate +from payload hydration and concurrent real-store qualification. + +Current-state qualification against the four-version, two-file 10-GiB entropy +history passes with the updated reader. Opening uses 28 backend operations and +149,860 bytes. The duplicate-file cold query uses 497 range reads and 7,434,846 +bytes; the warm query repeats all 497 reads. Both results match full inspection, +operations drain and no writes occur. These are one local debug-profile sample, +not cloud latency or independent history-scaling proof. Repeated recipe metadata +reads are a measured remaining performance gap, not a successful warm-cache gate. +The first attempted receipt remains failed: it supplied pointer-wire hex where +the qualifier requires the distinct catalog `MerkleHash::hex()` representation. + +A differential run of the same qualifier binary against a real 4-MiB file passes: +five cold ranges / 3,755 bytes and zero warm backend operations. Combined with +the unchanged captured lookup and the existing 256-page / 8-MiB FIFO page cache, +this supports recipe-page eviction as the cause of repeated large-file warm +reads. A bounded cache of already-verified file bindings is the next candidate; +it must preserve the initial complete dependency proof, immutable capture, +ordered duplicates/misses, error retry and shared close/cancellation contracts. + +The current reader's minimal CLI/NFS check and debug build pass. The force-push +policy/visibility regression also passes after awaiting the now-async visibility +API in its test caller; all six policy cases retain their assertions. Eleven +Python fixture tests pass. The previously qualified staging-version assertion +from the combined branch is carried forward with exact post-push retirement. + +The complete current-source RustFS CLI smoke remains failed (152 commands, +61 checks). Its shallow-since case supplies an ISO date; the helper treats the +remote-helper option as an integer wire timestamp and rejects it before clone. +The harness still expects the earlier unsupported-option diagnostic. Git's +remote-helper boundary passes date text through, while wire fetch uses a +normalized timestamp. Fixing that adapter and qualifying dated boundaries +against native Git are required; changing the expected message alone would +hide the missing behavior. No full CLI E2E pass is claimed from this receipt. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index b79cbe9b9..79d217fea 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -2100,11 +2100,18 @@ def run_committed_restage_before_first_push_case(self) -> None: ["staging", "stats", "--json"], name=f"{case_name} staging ownership before push", ) - before_push = json.loads(self.read_stdout(before_push_record))["data"]["lifecycle"] + before_push_data = json.loads(self.read_stdout(before_push_record))["data"] + before_push = before_push_data["lifecycle"] + expected_versions = {first_pointer.file_hash, second_pointer.file_hash} + retained_versions = {entry["file_hash"] for entry in before_push_data["files"]} + # Native preparation and Git-clean retain independent handoff leases. + # Version coverage, followed by exact retirement, proves their lifetime; + # a fixed pre-push lease count cannot identify the preserved versions. self.check( f"{case_name}-retains-current-and-committed-history-owners", - before_push["path_heads"] == 2 - and before_push["path_leases"] == 2 + retained_versions == expected_versions + and before_push["recipes"] == len(expected_versions) + and before_push["path_leases"] >= len(expected_versions) and before_push["open_batches_without_publication"] == 0 and before_push["reclaimable_superseded_leases"] == 0 and before_push["reclaimable_files"] == 0, diff --git a/crab/src/cmd/gc/capsule_cleanup_tests.rs b/crab/src/cmd/gc/capsule_cleanup_tests.rs index a501550aa..9a3a7f618 100644 --- a/crab/src/cmd/gc/capsule_cleanup_tests.rs +++ b/crab/src/cmd/gc/capsule_cleanup_tests.rs @@ -19,6 +19,200 @@ use crate::core::error::CrabError; use crate::storage::StoreLayout; use crate::storage::store::Store; +#[tokio::test] +async fn captured_catalog_pages_survive_checkpoint_replacement_and_gc() { + use crab_metadata::capsule_protocol::{ + CapsuleGitPack, FileCatalogEntry, LayeredCheckpoint, PackLayer, PointerCatalog, + ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + }; + use crab_metadata::file_index_lookup::FileIndexLookupSession; + use crab_xet::hash::MerkleHash; + + let store = Store::new(Arc::new(InMemory::new())); + let router = StoreLayout::new(store.clone(), "org/catalog-gc".to_owned()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let base = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let mut catalog = PointerCatalog::new(); + let shard = "a".repeat(64); + let xorb = "b".repeat(64); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 80, + "c".repeat(64), + vec![XorbChunkEntry::new("d".repeat(64), 16)], + ), + ) + .unwrap(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + for index in 0..20_000 { + catalog + .insert_file( + format!("{index:064x}"), + FileCatalogEntry::new(16, shard.clone()), + ) + .unwrap(); + } + let transaction = CapsuleTransaction::new( + base.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap(); + crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) + .await + .unwrap(); + let view = crab_read::capsule_protocol::open_view( + &layout, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 8 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let layer = PackLayer::build( + &CapsuleGitPack::new( + Bytes::from_static(b"PACK"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "3".repeat(40), + 1, + ) + .unwrap(), + ) + .unwrap(); + let source = layer.source_descriptor().unwrap(); + store + .put( + &layout.capsule_pack_layer_path(layer.hash()), + layer.bytes().clone(), + ) + .await + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + view.root().root().generation(), + view.root().digest(), + vec![source.clone()], + catalog, + None, + ) + .unwrap(); + let first = crab_write::capsule_protocol::publish_ref_layered_checkpoint( + &layout, + view.root_snapshot().clone(), + &checkpoint, + view.refs().clone(), + view.peeled_refs().clone(), + view.visible_ref_transactions().clone(), + view.capsule_run_pointers().to_vec(), + ) + .await + .unwrap(); + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed_reads = reads.clone(); + let observed = layout + .store() + .clone() + .with_read_request_observer(Arc::new(move |_| { + observed_reads.fetch_add(1, Ordering::SeqCst); + })); + let captured = FileIndexLookupSession::open_for_storage(&observed, layout.repo_prefix()) + .await + .unwrap(); + // No file query has run: the old reader must fetch its catalog leaf after + // compaction and a real sweep, rather than succeed from a warmed page cache. + let next = LayeredCheckpoint::build( + first.record().root().generation(), + first.record().digest(), + vec![source], + PointerCatalog::new(), + None, + ) + .unwrap(); + let root = crab_write::capsule_protocol::publish_ref_layered_checkpoint( + &layout, + first.clone(), + &next, + first.record().root().refs().clone(), + first.record().root().peeled_refs().clone(), + first.record().root().compacted_ref_transactions().clone(), + Vec::new(), + ) + .await + .unwrap(); + let file = MerkleHash::from_hex(&format!("{:064x}", 19_999)).unwrap(); + let latest = FileIndexLookupSession::open_for_storage(layout.store(), layout.repo_prefix()) + .await + .unwrap(); + let latest_result = latest.lookup(&file).await; + latest.close().await.unwrap(); + assert_eq!(latest_result.unwrap(), None); + + let orphan = layout.capsule_checkpoint_path(&"e".repeat(64)); + store + .put(&orphan, Bytes::from_static(b"orphan")) + .await + .unwrap(); + let current = crab_read::capsule_protocol::open_view_from_root_with_control( + &layout, + root.clone(), + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 8 * 1024 * 1024, + max_frontier_bytes: 8 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let snapshot = current.git_snapshot().unwrap(); + let outcome = sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + root.record().root(), + current.capsule_run_pointers(), + ¤t.state_digest(), + &snapshot.manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now() + Duration::from_secs(2 * 3600), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + assert_eq!(outcome.packs_deleted, 1); + assert!(matches!( + store.head(&orphan).await, + Err(CrabError::NotFound { .. }) + )); + reads.store(0, Ordering::SeqCst); + let selected = captured.lookup_batch(&[file, file]).await; + captured.close().await.unwrap(); + assert!( + reads.load(Ordering::SeqCst) > 0, + "captured lookup must fetch cold pages after GC" + ); + let expected = MerkleHash::from_hex(&shard).unwrap(); + assert_eq!(selected.unwrap(), vec![Some(expected), Some(expected)]); +} + #[tokio::test] async fn failed_capsule_view_releases_root_and_sweep_fences() { let store = Store::new(Arc::new(InMemory::new())); From 750e07af18ae9d1580aa07f1cdcc190295069c4d Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 01:25:54 -0700 Subject: [PATCH 075/127] fix(git): honor dated shallow selections Remote helpers receive Git date expressions before wire normalization. Apply shallow selectors before requested-tip selection so an empty result cannot become a depth-one checkout. --- crab/docs/architecture/git-protocol-v2.md | 15 ++- .../e2e/run_add_commit_push_rustfs_smoke.py | 111 ++++++++------- crab/src/git/remote_helper.rs | 126 +++++++++++++++--- crates/crab-read/src/upload_pack.rs | 45 +++++-- crates/crab-remote-git/src/error.rs | 5 + 5 files changed, 217 insertions(+), 85 deletions(-) diff --git a/crab/docs/architecture/git-protocol-v2.md b/crab/docs/architecture/git-protocol-v2.md index e188bda4c..ad1d1efc2 100644 --- a/crab/docs/architecture/git-protocol-v2.md +++ b/crab/docs/architecture/git-protocol-v2.md @@ -55,9 +55,8 @@ the invocation rather than allowing every protocol globally: git -c protocol.crab.allow=always clone --recurse-submodules crab://bucket/team/repo ``` -"All commands" is not a support claim. Date/ref-exclusion shallow selectors -remain unimplemented; signed push certificates, push options intended for -server-side hooks, and remote archive service need separate protocol contracts. +"All commands" is not a support claim. Signed push certificates, push options +intended for server-side hooks, and remote archive service need separate protocol contracts. Signed commits/tags are Git objects and are distinct from signed push requests. PRs, CI execution, and mandatory server-side hooks cannot execute inside a passive bucket. Clients with unrestricted bucket write credentials can bypass @@ -76,7 +75,7 @@ select a v2 authority. The session supports: - `ls-refs` with ref prefixes, symrefs, peeled tags, unborn HEAD, and hidden refs; - `fetch` with wants, haves, done, tags, sideband/progress, shallow/deepen, - and `deepen-relative`; + `deepen-relative`, `deepen-since`, and `deepen-not`; - the bounded filter matrix: `blob:none`; `blob:limit=[kmg]`; `tree:`; `object:type={tag,commit,tree,blob}`; `sparse:oid=`; and repeated filters or percent-encoded `combine:` intersections; @@ -85,6 +84,12 @@ select a v2 authority. The session supports: required or assumed; - standard local Git pack installation on the v2 path. +The line helper normalizes `--shallow-since` date expressions through Git's +date parser before passing epoch seconds to the shared planner. Date and +ref-exclusion selectors apply to requested tips as well as their parents. +If no commits qualify, fetching fails before pack installation or shallow-file +publication; it does not substitute a depth-one checkout. + An empty manifest is a valid ref snapshot without a catalog. Empty clones and `ls-remote` complete through v2; clients requesting `unborn` receive the configured HEAD target unless hidden. Object wants against that empty snapshot @@ -229,7 +234,7 @@ takeover. ## Explicitly unsupported The first profile rejects stateful `connect`, `packfile-uris`, `object-info`, -`ref-in-want`, date/ref-exclusion shallow selectors, and other capabilities not +`ref-in-want`, and other capabilities not listed above. A v2 session cannot fall back to the line protocol after the terminal handoff; it reports the protocol failure instead. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 79d217fea..6af60724b 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -1097,53 +1097,70 @@ def run_atomic_rejection_case(self) -> None: def run_shallow_case(self) -> None: case_name = "shallow-history" repo, remote_url, _repo_prefix = self.prepare_git_repo(case_name) - first = self.commit_text(repo, "history.txt", "one\n", "one") - self.commit_text(repo, "history.txt", "two\n", "two") - self.commit_text(repo, "history.txt", "three\n", "three") - self.run_git(repo, ["push", "-u", "origin", "main"]) - - selector_clone = self.run_root / case_name / "shallow-since-clone" - selector_result = self.clone_git( - remote_url, - selector_clone, - name=f"{case_name} unsupported shallow-since clone", - extra_args=["--shallow-since", "2100-01-01T00:00:00Z"], - check=False, - ) - selector_stderr = Path(selector_result.stderr_log).read_text( - encoding="utf-8", errors="replace" - ) - self.check( - f"{case_name}-shallow-since-fails-explicitly", - selector_result.exit_code != 0 - and "deepen-since is not supported" in selector_stderr - and not (selector_clone / "history.txt").exists(), - { - "exit_code": selector_result.exit_code, - "stderr_log": selector_result.stderr_log, - }, - ) - exclude_clone = self.run_root / case_name / "shallow-exclude-clone" - exclude_result = self.clone_git( - remote_url, - exclude_clone, - name=f"{case_name} unsupported shallow-exclude clone", - extra_args=["--shallow-exclude", "main"], - check=False, - ) - exclude_stderr = Path(exclude_result.stderr_log).read_text( - encoding="utf-8", errors="replace" - ) - self.check( - f"{case_name}-shallow-exclude-fails-explicitly", - exclude_result.exit_code != 0 - and "deepen-not is not supported" in exclude_stderr - and not (exclude_clone / "history.txt").exists(), - { - "exit_code": exclude_result.exit_code, - "stderr_log": exclude_result.stderr_log, - }, - ) + commits = [] + for index, text in enumerate(("one", "two", "three"), start=1): + (repo / "history.txt").write_text(text + "\n", encoding="utf-8") + self.run_git(repo, ["add", "history.txt"]) + date = f"2023-11-{14 + index:02d}T22:13:20Z" + self.run_cmd( + f"{case_name} dated commit {text}", + ["git", "commit", "-m", text], + repo, + extra_env={"GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}, + ) + commits.append(self.rev_parse(repo, "HEAD")) + first = commits[0] + self.run_git(repo, ["tag", "archive", first]) + self.run_git(repo, ["push", "-u", "origin", "main", "--tags"]) + + # Match native upload-pack before asserting Crab's selector behavior. + # Fixed dates separate date parsing from commit-boundary selection. + selectors = ( + ("since", ["--shallow-since", "2023-11-16T22:13:20Z"]), + ("exclude", ["--shallow-exclude", "archive"]), + ("future", ["--shallow-since", "2030-01-01T00:00:00Z"]), + ("exclude-all", ["--shallow-exclude", "main"]), + ) + for selector, arguments in selectors: + expected_success = selector in ("since", "exclude") + transports = ( + ("native", repo.resolve().as_uri(), []), + ("crab-v2", remote_url, ["--config", "protocol.version=2"]), + ("crab-classic", remote_url, ["--config", "protocol.version=0"]), + ) + for transport, url, protocol in transports: + clone = self.run_root / case_name / f"{transport}-{selector}-clone" + result = self.clone_git( + url, clone, + name=f"{case_name} {transport} {selector} clone", + extra_args=[*protocol, *arguments], check=False, + ) + detail = {"exit_code": result.exit_code, "stderr_log": result.stderr_log} + if not expected_success: + diagnostic = Path(result.stderr_log).read_text( + encoding="utf-8", errors="replace" + ) + self.check( + f"{case_name}-{transport}-{selector}-rejects-empty-selection", + result.exit_code != 0 + and "no commits selected for shallow requests" in diagnostic + and not (clone / "history.txt").exists(), + detail, + ) + continue + self.check( + f"{case_name}-{transport}-{selector}-clone-succeeds", + result.exit_code == 0, detail, + ) + selected = self.read_stdout(self.run_git(clone, ["rev-list", "HEAD"])) + boundary = (clone / ".git" / "shallow").read_text().splitlines() + self.check( + f"{case_name}-{transport}-{selector}-exact-history-and-boundary", + selected.splitlines() == list(reversed(commits[1:])) + and boundary == [commits[1]] + and (clone / "history.txt").read_text() == "three\n", + {"commits": selected.splitlines(), "boundary": boundary}, + ) raw_fetch = self.run_root / case_name / "raw-fetch" raw_fetch.mkdir() diff --git a/crab/src/git/remote_helper.rs b/crab/src/git/remote_helper.rs index b95344ac3..107c4967a 100644 --- a/crab/src/git/remote_helper.rs +++ b/crab/src/git/remote_helper.rs @@ -1105,7 +1105,11 @@ where match cmd { HelperCommand::Option { key, value } => { - handle_option(&key, &value, options, writer).await?; + tokio::select! { + biased; + () = cancellation.cancelled() => return Err(CrabError::Cancelled), + result = handle_option(&key, &value, options, writer) => result?, + } } HelperCommand::Capabilities => { return Ok(Some(Batch::Capabilities)); @@ -1223,22 +1227,39 @@ async fn handle_option( } }, "deepen-since" => { - let timestamp = value - .parse::() - .map_err(|_| CrabError::Protocol(format!("invalid deepen-since value: {value}")))?; - if options - .fetch_options - .deepen_since - .replace(timestamp) - .is_some() - { + if options.fetch_options.deepen_since.is_some() { return Err(CrabError::Protocol( "duplicate deepen-since option".to_owned(), )); } + // Helpers receive Git date expressions; only upload-pack wire requests + // contain epoch seconds. Use Git's approxidate parser for the same grammar. + let date = decode_option_value(value)?; + let output = tokio::process::Command::new("git") + .arg("rev-parse") + .arg(format!("--since={date}")) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .output() + .await?; + if !output.status.success() { + return Err(CrabError::Protocol(format!( + "cannot parse deepen-since date: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + let timestamp = std::str::from_utf8(&output.stdout) + .ok() + .and_then(|line| line.trim().strip_prefix("--max-age=")) + .and_then(|seconds| seconds.parse::().ok()) + .ok_or_else(|| { + CrabError::Protocol("invalid Git deepen-since timestamp".to_owned()) + })?; + options.fetch_options.deepen_since = Some(timestamp); writer.write_all(b"ok\n").await?; } "deepen-not" => { + let value = decode_option_value(value)?; if value.is_empty() || value.bytes().any(|byte| byte.is_ascii_whitespace()) { return Err(CrabError::Protocol( "deepen-not requires one non-empty reference".to_owned(), @@ -1248,9 +1269,9 @@ async fn handle_option( .fetch_options .deepen_not .iter() - .any(|name| name == value) + .any(|name| name == &value) { - options.fetch_options.deepen_not.push(value.to_owned()); + options.fetch_options.deepen_not.push(value); } writer.write_all(b"ok\n").await?; } @@ -1356,19 +1377,25 @@ async fn handle_option( Ok(()) } -fn parse_ref_lease(value: &str) -> Result<(String, Option)> { +fn decode_option_value(value: &str) -> Result { if value.starts_with('"') && !value.ends_with('"') { return Err(CrabError::Protocol( - "unterminated quoted ref lease".to_owned(), + "unterminated quoted helper option".to_owned(), )); } let (decoded, consumed) = gix_quote::ansi_c::undo(value.as_bytes().into()) - .map_err(|error| CrabError::Protocol(format!("invalid quoted ref lease: {error}")))?; + .map_err(|error| CrabError::Protocol(format!("invalid quoted helper option: {error}")))?; if consumed != value.len() { - return Err(CrabError::Protocol("trailing data in ref lease".to_owned())); + return Err(CrabError::Protocol( + "trailing data in helper option".to_owned(), + )); } - let decoded = std::str::from_utf8(decoded.as_ref()) - .map_err(|error| CrabError::Protocol(format!("ref lease is not UTF-8: {error}")))?; + String::from_utf8(decoded.to_vec()) + .map_err(|error| CrabError::Protocol(format!("helper option is not UTF-8: {error}"))) +} + +fn parse_ref_lease(value: &str) -> Result<(String, Option)> { + let decoded = decode_option_value(value)?; let (name, oid) = decoded .rsplit_once(':') .ok_or_else(|| CrabError::Protocol("ref lease requires ref:oid".to_owned()))?; @@ -5732,7 +5759,7 @@ mod tests { let (source, store, router, commits, _tag) = capsule_history_fixture().await; let cutoff = String::from_utf8(run_git( source.path(), - &["show", "-s", "--format=%ct", &commits[1]], + &["show", "-s", "--format=%cI", &commits[1]], )) .expect("commit timestamp is utf8") .trim() @@ -5764,6 +5791,40 @@ mod tests { assert!(!git_object_exists(target.path(), &commits[0])); } + #[tokio::test] + async fn classic_capsule_empty_shallow_selectors_install_nothing() { + let (_source, store, router, commits, _tag) = capsule_history_fixture().await; + for selector in ["deepen-since 1893456000", "deepen-not main"] { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let git_dir = target.path().join(".git"); + let _guard = GitEnvCwdGuard::set(target.path(), &git_dir, target.path()); + let tip = commits.last().expect("history tip"); + let input = format!("option {selector}\nfetch {tip} refs/heads/main\n\n"); + let context = test_context( + store.clone(), + router.repo_prefix(), + target.path().join("push-state"), + ); + let (_, result) = + run_with_context(&input, context, tokio_util::sync::CancellationToken::new()).await; + let error = result.expect_err("empty shallow selection must fail"); + assert!( + error + .to_string() + .contains("no commits selected for shallow requests") + ); + assert_eq!( + ( + git_object_exists(target.path(), tip), + git_dir.join("shallow").exists() + ), + (false, false), + "{selector} must fail before pack installation or shallow publication" + ); + } + } + #[tokio::test] async fn capsule_promisor_fetch_installs_the_authorized_object() { let (store, router, blob) = capsule_promisor_fixture().await; @@ -7386,6 +7447,9 @@ mod tests { #[tokio::test] async fn shallow_selectors_are_accepted_without_degrading_to_full_fetch() { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let _guard = GitEnvCwdGuard::set(target.path(), &target.path().join(".git"), target.path()); let mut options = HelperOptions::default(); let mut output = Vec::new(); handle_option("deepen-since", "1700000000", &mut options, &mut output) @@ -7404,6 +7468,30 @@ mod tests { assert_eq!(String::from_utf8(output).unwrap(), "ok\nok\n"); } + #[tokio::test] + async fn option_shallow_since_normalizes_git_dates_and_quoting() { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let _guard = GitEnvCwdGuard::set(target.path(), &target.path().join(".git"), target.path()); + for date in [ + "1700000000", + "2023-11-14T22:13:20Z", + "2023-11-14T23:13:20+01:00", + "\"2023-11-14 22:13:20 +0000\"", + ] { + let mut options = HelperOptions::default(); + let mut output = Vec::new(); + handle_option("deepen-since", date, &mut options, &mut output) + .await + .unwrap(); + assert_eq!( + (options.fetch_options.deepen_since, output), + (Some(1_700_000_000), b"ok\n".to_vec()), + "{date}" + ); + } + } + #[tokio::test] async fn option_update_shallow_accepts_booleans_and_rejects_other_values() { assert_eq!(run("option update-shallow true\n").await, "ok\n"); diff --git a/crates/crab-read/src/upload_pack.rs b/crates/crab-read/src/upload_pack.rs index 5fb59eeb3..3bed3dc62 100644 --- a/crates/crab-read/src/upload_pack.rs +++ b/crates/crab-read/src/upload_pack.rs @@ -1128,6 +1128,7 @@ async fn plan_with_operation( } let mut selected = HashSet::new(); + let mut selected_commit = false; let mut object_ids = Vec::new(); let mut shallow = HashSet::new(); let mut unshallow = HashSet::new(); @@ -1141,7 +1142,11 @@ async fn plan_with_operation( let Some(item) = queue.pop_front() else { break; }; - if common_haves.contains(&item.oid) && request.deepen.is_none() { + if common_haves.contains(&item.oid) + && request.deepen.is_none() + && request.deepen_since.is_none() + && request.deepen_not.is_empty() + { continue; } // Tree entries prove that blobs are leaves. Rejecting a known omitted blob here @@ -1241,9 +1246,29 @@ async fn plan_with_operation( invariant: "batched upload-pack read is missing an object", })? }; - if excluded_commits.contains(&item.oid) && !roots.contains(&item.oid) { + // Wants authorize access but do not override shallow selectors. Applying + // these before selection also prevents an excluded tip's tree leaking in. + if excluded_commits.contains(&item.oid) { continue; } + if object.kind == gix_object::Kind::Commit + && let Some(since) = request.deepen_since + { + let commit = gix_object::CommitRef::from_bytes(&object.data, gix_hash::Kind::Sha1) + .map_err(|_| RemoteGitError::Corrupt { + stage: CorruptionStage::Commit, + })?; + let timestamp = commit + .time() + .map_err(|_| RemoteGitError::Corrupt { + stage: CorruptionStage::Commit, + })? + .seconds; + if timestamp < since { + continue; + } + } + selected_commit |= object.kind == gix_object::Kind::Commit; let encountered_client_have = matches!(visibility, VisibilitySource::TipBound { .. }) && client_haves.contains(&item.oid) && object.kind == gix_object::Kind::Commit; @@ -1278,6 +1303,10 @@ async fn plan_with_operation( } } + if (request.deepen_since.is_some() || !request.deepen_not.is_empty()) && !selected_commit { + return Err(RemoteGitError::EmptyShallowSelection); + } + if request.include_tags { for reference in &repository.refs().entries { if !reference.name.starts_with("refs/tags/") @@ -2396,18 +2425,6 @@ async fn enqueue_children( maximum_objects, deduplicate_by_oid, )?; - if let Some(since) = request.deepen_since { - let timestamp = commit - .time() - .map_err(|_| RemoteGitError::Corrupt { - stage: CorruptionStage::Commit, - })? - .seconds; - if timestamp < since { - shallow.insert(item.oid); - return Ok(()); - } - } match item.depth { TraversalDepth::RelativeBoundary => { if existing_shallow.contains(&item.oid) { diff --git a/crates/crab-remote-git/src/error.rs b/crates/crab-remote-git/src/error.rs index 15bcdeb54..0e40a9f61 100644 --- a/crates/crab-remote-git/src/error.rs +++ b/crates/crab-remote-git/src/error.rs @@ -237,6 +237,10 @@ pub enum Error { #[error("repository is empty")] EmptyRepository, + /// Date and ref exclusions selected no commits for a shallow fetch. + #[error("no commits selected for shallow requests")] + EmptyShallowSelection, + /// Locator publication has not reached the required manifest generation. #[error( "repository object locator is not ready: observed generation {observed:?}, required {required}" @@ -516,6 +520,7 @@ impl Error { | Self::InvalidPath { .. } | Self::InvalidCursor { .. } | Self::InvalidLimit { .. } + | Self::EmptyShallowSelection | Self::Revision { reason: RevisionError::InvalidReference From aa24b292359c2f0478c893074aa98280f81c22de Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 01:37:48 -0700 Subject: [PATCH 076/127] test(git): keep raw SHA probe unadvertised Advertise an archive side tip instead of the ancestor under raw-fetch admission test. Its exclusion closure still contains that ancestor, preserving both selector and admission assertions. --- .../scripts/e2e/run_add_commit_push_rustfs_smoke.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 6af60724b..ff9097936 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -1110,7 +1110,18 @@ def run_shallow_case(self) -> None: ) commits.append(self.rev_parse(repo, "HEAD")) first = commits[0] - self.run_git(repo, ["tag", "archive", first]) + # Exclude the first commit through a side tip, keeping that ancestor + # unadvertised for the raw-SHA policy check below. + archive = self.read_stdout(self.run_cmd( + f"{case_name} archive side tip", + ["git", "commit-tree", f"{first}^{{tree}}", "-p", first, "-m", "archive"], + repo, + extra_env={ + "GIT_AUTHOR_DATE": "2023-11-15T23:13:20Z", + "GIT_COMMITTER_DATE": "2023-11-15T23:13:20Z", + }, + )) + self.run_git(repo, ["tag", "archive", archive]) self.run_git(repo, ["push", "-u", "origin", "main", "--tags"]) # Match native upload-pack before asserting Crab's selector behavior. From 85183e82734969b00e178946f3f6e2643d59f110 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 02:00:04 -0700 Subject: [PATCH 077/127] test(git): fault current capsule payloads V2 stores Git packs and indexes inside capsule runs. Target those ranges and compare authority bytes directly: damaged discovery controls cannot prove unchanged refs through ls-remote. --- crab/docs/architecture/git-protocol-v2.md | 7 + .../e2e/run_add_commit_push_rustfs_smoke.py | 190 ++++++++---------- .../e2e/test_add_commit_push_source.py | 31 +++ 3 files changed, 121 insertions(+), 107 deletions(-) diff --git a/crab/docs/architecture/git-protocol-v2.md b/crab/docs/architecture/git-protocol-v2.md index ad1d1efc2..24d6f8bcb 100644 --- a/crab/docs/architecture/git-protocol-v2.md +++ b/crab/docs/architecture/git-protocol-v2.md @@ -254,6 +254,13 @@ filter reads must leave canonical object counts and bytes unchanged; generated response-pack cache and coordination objects are accounted for separately. This is not a claim that first-read repair works with read-only object-store credentials. +The add/commit/push RustFS smoke exercises the current physical fault boundary: +loss of a published capsule and corruption confined to its embedded Git pack +or index range. Each rejection probe uses an empty cache and requires no local +HEAD or checked-out payload. It compares the remote root and ref-head bytes +before and after the fault; discovery can itself reject a damaged capsule, +so successful `ls-remote` is not the mutation oracle. + Direct pushes and protected/service publication paths publish visibility before the corresponding ref or manifest commit. Protected receive extracts the closure from the already-verified materialization ODB before that workspace diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index ff9097936..5e9f70b77 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -10,7 +10,7 @@ clone, hydration, and byte identity. A separate ordinary-Git matrix proves missing/corrupt metadata-root handling, update/fetch, deletion, force and non-fast-forward behavior, atomic rejection, tag following, shallow history, -immutable-pack failures, and concurrent push serialization. +capsule loss, embedded pack/index corruption, and concurrent push serialization. With ``--source``, run only the two managed-file workflows on disposable clones of a read-only real repository. Use a run root on the workspace volume @@ -205,6 +205,31 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def capsule_member_range(body: bytes, kind: str) -> tuple[int, int]: + """Locate one pack/index fault inside the synthetic single-member run.""" + if len(body) < 48 or body[-8:] != b"CRBRUN01" or kind not in ("pack", "index"): + raise SmokeError("fault fixture is not a capsule run with a pack/index target") + footer_length = int.from_bytes(body[-48:-40], "big") + footer_start = len(body) - 48 - footer_length + if not 0 < footer_length <= 8 * 1024 * 1024 or footer_start < 0: + raise SmokeError("fault fixture footer is outside its container") + try: + footer = json.loads(body[footer_start:-48]) + members = footer["git_packs"] + if footer["version"] != 4 or len(members) != 1: + raise SmokeError("fault fixture requires one current-format pack member") + offset, length = members[0][kind]["offset"], members[0][kind]["length"] + except (KeyError, TypeError, ValueError) as exc: + raise SmokeError("fault fixture has no usable pack member") from exc + if (type(offset) is not int or type(length) is not int + or offset < 0 or length < 4 or offset + length > footer_start): + raise SmokeError("fault fixture range overlaps control bytes or escapes its container") + magic = b"PACK" if kind == "pack" else b"\xfftOc" + if body[offset:offset + 4] != magic: + raise SmokeError("fault fixture member does not contain the selected Git format") + return offset, length + + def parse_pointer(blob: str) -> PointerRecord: raw = blob.encode("utf-8") if len(raw) > 256: @@ -1314,122 +1339,74 @@ def run_corrupt_root_case(self) -> None: {"stderr_log": clone.stderr_log}, ) - def run_immutable_object_failure_case(self, object_kind: str) -> None: - case_name = f"missing-{object_kind}" - repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) - remote_tip = self.commit_text(repo, "payload.txt", "immutable\n", "immutable") - self.run_git(repo, ["push", "-u", "origin", "main"]) - keys = self.list_keys(repo_prefix) - suffix = f".{object_kind}" - candidates = sorted(key for key in keys if key.endswith(suffix)) - self.check( - f"{case_name}-fixture-object-found", - bool(candidates), - {"suffix": suffix, "keys": sorted(keys)}, - ) - target_key = candidates[0] - status, _, body = self.signed_s3_request("DELETE", target_key) - self.check( - f"{case_name}-fixture-object-deleted", - status in (200, 204), - { - "status": status, - "key": target_key, - "body": body.decode("utf-8", errors="replace")[:200], - }, - ) - - clone_dir = self.run_root / case_name / "clone" - clone = self.clone_git( - remote_url, clone_dir, name=f"{case_name} git clone", check=False - ) - head = self.run_git( - self.run_root, - ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], - name=f"{case_name} verify no local ref update", - check=False, - ) - self.check( - f"{case_name}-clone-fails-before-checkout", - clone.exit_code != 0 - and head.exit_code != 0 - and not (clone_dir / "payload.txt").exists(), - { - "clone_exit_code": clone.exit_code, - "head_exit_code": head.exit_code, - "target_key": target_key, - }, - ) - refs = self.ls_remote(remote_url, name=f"{case_name} refs remain coherent") - self.check( - f"{case_name}-remote-ref-remains-original", - refs.get("refs/heads/main") == remote_tip, - {"expected": remote_tip, "actual": refs.get("refs/heads/main")}, - ) - - def run_corrupt_index_failure_case(self) -> None: - case_name = "corrupt-idx" + def run_capsule_failure_case(self, fault: str) -> None: + case_name = "missing-capsule" if fault == "missing" else f"corrupt-{fault}" repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) remote_tip = self.commit_text(repo, "payload.txt", "immutable\n", "immutable") self.run_git(repo, ["push", "-u", "origin", "main"]) - candidates = sorted( - key for key in self.list_keys(repo_prefix) if key.endswith(".idx") - ) self.check( - f"{case_name}-fixture-object-found", - bool(candidates), - {"suffix": ".idx", "keys": candidates}, + f"{case_name}-healthy-tip-before-fault", + self.ls_remote(remote_url, name=f"{case_name} healthy refs").get("refs/heads/main") == remote_tip, ) + keys = self.list_keys(repo_prefix) + candidates = sorted(key for key in keys if key.startswith(f"{repo_prefix}/v2/capsules/")) + self.check(f"{case_name}-fixture-object-found", len(candidates) == 1, {"keys": candidates}) target_key = candidates[0] - status, _, body = self.signed_s3_request( - "PUT", target_key, body=b"not a valid git pack index\n" - ) - self.check( - f"{case_name}-fixture-object-corrupted", - status in (200, 201), - { - "status": status, - "key": target_key, - "body": body.decode("utf-8", errors="replace")[:200], - }, - ) - + authority = {} + for key in sorted(key for key in keys if key == f"{repo_prefix}/v2/root" or key.startswith(f"{repo_prefix}/v2/refs/")): + status, _, value = self.signed_s3_request("GET", key) + self.check(f"{case_name}-authority-readable-before-fault", status == 200, {"key": key}) + authority[key] = value + status, headers, original = self.signed_s3_request("GET", target_key) + self.check(f"{case_name}-fixture-body-readable", status == 200 and len(original) < 1024 * 1024) + if fault == "missing": + status, _, _ = self.signed_s3_request("DELETE", target_key) + absent, _, _ = self.signed_s3_request("GET", target_key) + self.check(f"{case_name}-fixture-object-deleted", status in (200, 204) and absent == 404) + else: + offset, length = capsule_member_range(original, fault) + damaged = bytearray(original) + damaged[offset] ^= 1 + status, _, _ = self.signed_s3_request( + "PUT", target_key, body=bytes(damaged), extra_headers={"if-match": headers["etag"]}, + ) + read_status, _, stored = self.signed_s3_request("GET", target_key) + self.check( + f"{case_name}-fixture-member-corrupted", + status in (200, 201) and read_status == 200 and stored == bytes(damaged), + {"key": target_key, "member_offset": offset, "member_length": length}, + ) clone_dir = self.run_root / case_name / "clone" - clone = self.clone_git( - remote_url, clone_dir, name=f"{case_name} git clone", check=False + clone_cache = self.run_root / case_name / "cold-cache" + self.check(f"{case_name}-clone-cache-starts-empty", not clone_cache.exists()) + clone = self.run_cmd( + f"{case_name} git clone", ["git", "clone", remote_url, str(clone_dir)], self.run_root, + check=False, extra_env={"CRAB_CACHE_DIR": str(clone_cache)}, ) head = self.run_git( - self.run_root, - ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], - name=f"{case_name} verify no local ref update", - check=False, - ) - clone_stderr = Path(clone.stderr_log).read_text( - encoding="utf-8", errors="replace" + self.run_root, ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], + name=f"{case_name} verify no local ref update", check=False, ) self.check( f"{case_name}-clone-fails-before-checkout", - clone.exit_code != 0 - and head.exit_code != 0 - and not (clone_dir / "payload.txt").exists(), - { - "clone_exit_code": clone.exit_code, - "head_exit_code": head.exit_code, - "target_key": target_key, - }, - ) - self.check( - f"{case_name}-clone-surfaces-integrity-error", - "CRAB-E0020" in clone_stderr - or "corrupt object" in clone_stderr.lower(), - {"stderr_log": clone.stderr_log}, - ) - refs = self.ls_remote(remote_url, name=f"{case_name} refs remain coherent") - self.check( - f"{case_name}-remote-ref-remains-original", - refs.get("refs/heads/main") == remote_tip, - {"expected": remote_tip, "actual": refs.get("refs/heads/main")}, + clone.exit_code != 0 and head.exit_code != 0 and not (clone_dir / "payload.txt").exists(), + {"clone_exit_code": clone.exit_code, "head_exit_code": head.exit_code, "target_key": target_key}, ) + if fault != "missing": + diagnostic = Path(clone.stderr_log).read_text(encoding="utf-8", errors="replace") + self.check( + f"{case_name}-clone-surfaces-integrity-error", + "CRAB-E0020" in diagnostic or "corrupt object" in diagnostic.lower(), + {"stderr_log": clone.stderr_log}, + ) + # Payloads share their physical object with discovery controls. Compare + # authority bytes directly; failed discovery cannot prove unchanged refs. + after = {} + for key in authority: + status, _, value = self.signed_s3_request("GET", key) + self.check(f"{case_name}-authority-readable-after-fault", status == 200, {"key": key}) + after[key] = value + self.check(f"{case_name}-remote-authority-unchanged", after == authority) def run_concurrent_push_case(self) -> None: case_name = "concurrent-cas" @@ -2312,9 +2289,8 @@ def run(self) -> None: self.run_ref_update_delete_force_and_tag_case() self.run_atomic_rejection_case() self.run_shallow_case() - self.run_immutable_object_failure_case("pack") - self.run_immutable_object_failure_case("idx") - self.run_corrupt_index_failure_case() + for fault in ("missing", "pack", "index"): + self.run_capsule_failure_case(fault) self.run_concurrent_push_case() self.run_committed_restage_before_first_push_case() self.run_partial_overlap_case() diff --git a/crab/scripts/e2e/test_add_commit_push_source.py b/crab/scripts/e2e/test_add_commit_push_source.py index 5b9ef5052..4f2c42e1a 100644 --- a/crab/scripts/e2e/test_add_commit_push_source.py +++ b/crab/scripts/e2e/test_add_commit_push_source.py @@ -214,5 +214,36 @@ def failed_run(smoke: SMOKE.AddCommitPushSmoke) -> None: self.assertNotIn(secret, report_text + stderr.getvalue()) +class CapsuleFaultRangeTests(unittest.TestCase): + @staticmethod + def fixture(*, offset: int = 8, length: int = 9, version: int = 4) -> bytes: + body = b"PACKbody\xfftOcindex" + footer = json.dumps({ + "version": version, + "git_packs": [{ + "pack": {"offset": 0, "length": 8}, + "index": {"offset": offset, "length": length}, + }], + }).encode() + return body + footer + len(footer).to_bytes(8, "big") + bytes(32) + b"CRBRUN01" + + def test_targets_the_absolute_pack_and_index_ranges(self) -> None: + body = self.fixture() + for kind, expected in (("pack", (0, 8)), ("index", (8, 9))): + with self.subTest(kind=kind): + self.assertEqual(SMOKE.capsule_member_range(body, kind), expected) + + def test_refuses_ranges_that_could_damage_unselected_controls(self) -> None: + for arguments in ({"offset": -1}, {"offset": 8, "length": 10}, {"offset": True}, {"length": 3}): + with self.subTest(arguments=arguments), self.assertRaises(SMOKE.SmokeError): + SMOKE.capsule_member_range(self.fixture(**arguments), "index") + + def test_refuses_other_container_formats_and_malformed_trailers(self) -> None: + body = self.fixture() + for malformed in (body[:-1], body[:20], self.fixture(version=3), body[:-48] + bytes(48)): + with self.subTest(size=len(malformed)), self.assertRaises(SMOKE.SmokeError): + SMOKE.capsule_member_range(malformed, "pack") + + if __name__ == "__main__": unittest.main() From 949439734b370b0cc00819595566b98e59b35497 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 02:32:53 -0700 Subject: [PATCH 078/127] fix(storage): retain immutable object identity Keep the provider token beside the bytes it identifies. Preserve both ETag and version from the PUT or verified GET so origin proofs need no later HEAD. Update all consumers of this unreleased Rust outcome shape; stored receipt formats remain unchanged. --- crab/src/git/xet_publication.rs | 6 +- crab/src/optimize/xorbs/executor.rs | 6 +- crates/crab-remote/src/prepare.rs | 8 +- crates/crab-storage/README.md | 4 + crates/crab-storage/src/store.rs | 122 ++++++++++++++++++++++++---- 5 files changed, 122 insertions(+), 24 deletions(-) diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 3e13c2392..4029b8c41 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -719,7 +719,7 @@ async fn publish_xorb_candidate( .create_or_read_immutable(&path, bytes, MAX_XORB_SIZE as u64) .await? { - crab_storage::ImmutableCreateOutcome::Created => { + crab_storage::ImmutableCreateOutcome::Created(_) => { if let (Some(cache), Some(bytes)) = (caching_store, cache_bytes) { warm_published_cache_object( cache, @@ -731,7 +731,9 @@ async fn publish_xorb_candidate( } Ok((local_entry, local_placements, true)) } - crab_storage::ImmutableCreateOutcome::Existing(existing) => { + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => { let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? .ok_or_else(|| CrabError::CorruptObject { path: path.to_string(), diff --git a/crab/src/optimize/xorbs/executor.rs b/crab/src/optimize/xorbs/executor.rs index cd51c621c..3c7115ccc 100644 --- a/crab/src/optimize/xorbs/executor.rs +++ b/crab/src/optimize/xorbs/executor.rs @@ -476,8 +476,10 @@ async fn upload_destination( () = cancel.cancelled() => return Err(CrabError::Cancelled), }; match publication { - crab_storage::ImmutableCreateOutcome::Created => {} - crab_storage::ImmutableCreateOutcome::Existing(existing) => { + crab_storage::ImmutableCreateOutcome::Created(_) => {} + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => { let parser = XorbParser::parse(existing).map_err(CrabError::from)?; if parser.hash() != destination_hash { return Err(CrabError::CorruptObject { diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 3c526c14e..4d8ba63bf 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -1147,10 +1147,10 @@ async fn upload_capsule_content( ) .await? { - crab_storage::ImmutableCreateOutcome::Created => local_entry.clone(), - crab_storage::ImmutableCreateOutcome::Existing(existing) => { - catalog_xorb_from_existing(&xorb_hash, local_entry, existing)? - } + crab_storage::ImmutableCreateOutcome::Created(_) => local_entry.clone(), + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => catalog_xorb_from_existing(&xorb_hash, local_entry, existing)?, }; delta.insert_xorb(xorb_hash, entry)?; } diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index 6291ec7c5..f82e41126 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -45,6 +45,10 @@ writes, bounded reads, byte/request observers, and provider-neutral `StorageError` values. `put_if_absent` distinguishes a new immutable object from an identical retry without a preceding HEAD, and streamed size/hash verification uses the GET response metadata instead of a separate HEAD. +`create_or_read_immutable` retains both ETag and version from the successful +PUT, verified readback, or occupied-key GET alongside its outcome. Origin proof +consumers must carry that identity with the verified bytes; a later HEAD may +observe another object version. Occupied-key bytes still require caller validation. `Store::verify_bounded` combines that content-hash contract with pre-body size admission and streamed length checks. It uses one GET for a successful read; the limit applies to buffered encoded bytes, not the caller's decoded objects. diff --git a/crates/crab-storage/src/store.rs b/crates/crab-storage/src/store.rs index 77583e604..edfbb3d96 100644 --- a/crates/crab-storage/src/store.rs +++ b/crates/crab-storage/src/store.rs @@ -60,10 +60,10 @@ pub enum ImmutableWriteVerification { /// Result of a create-only immutable write whose occupied key is returned to the caller. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ImmutableCreateOutcome { - /// This call created and verified the requested bytes. - Created, - /// The key was already occupied; these bounded bytes remain untrusted. - Existing(Bytes), + /// This call created and verified the bytes; identity belongs to that PUT or verified readback. + Created(ETag), + /// The key was occupied; identity belongs to this bounded GET, whose bytes remain untrusted. + Existing { bytes: Bytes, identity: ETag }, } /// Bounded-memory byte stream returned by object reads. @@ -727,11 +727,11 @@ impl Store { }; let payload = upload_payload(bytes); match self.inner.put_opts(&path, payload, options).await { - Ok(_) => Ok(true), + Ok(result) => Ok(Some(ETag::from(result))), Err(error) => { let mapped = map_object_store_error(error, path.as_ref()); if matches!(mapped, StorageError::StateConflict { .. }) { - Ok(false) + Ok(None) } else { Err(mapped) } @@ -740,13 +740,13 @@ impl Store { } }) .await?; - if !created { - let (existing, _) = self.get_with_etag_bounded(path, max_existing_bytes).await?; - return Ok(ImmutableCreateOutcome::Existing(existing)); - } + let Some(mut identity) = created else { + let (bytes, identity) = self.get_with_etag_bounded(path, max_existing_bytes).await?; + return Ok(ImmutableCreateOutcome::Existing { bytes, identity }); + }; if self.immutable_write_verification == ImmutableWriteVerification::ReadbackRequired { let maximum = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - let (stored, _) = self.get_with_etag_bounded(path, maximum).await?; + let (stored, observed_identity) = self.get_with_etag_bounded(path, maximum).await?; let actual_hash = *blake3::hash(&stored).as_bytes(); if actual_hash != expected_hash { return Err(StorageError::CorruptObject { @@ -758,8 +758,11 @@ impl Store { ), }); } + // Bind proof to the object version whose bytes were verified. A + // later HEAD may observe a replacement with a different encoding. + identity = observed_identity; } - Ok(ImmutableCreateOutcome::Created) + Ok(ImmutableCreateOutcome::Created(identity)) } /// Writes `bytes` at `path` iff nothing exists there yet. @@ -3763,6 +3766,7 @@ mod tests { Body(Vec), BodyError, Range(Range), + DistinctIdentities, } #[derive(Debug)] @@ -3785,7 +3789,12 @@ mod tests { payload: PutPayload, options: PutOptions, ) -> object_store::Result { - self.inner.put_opts(location, payload, options).await + let mut result = self.inner.put_opts(location, payload, options).await?; + if matches!(self.fault, ReadFault::DistinctIdentities) { + result.e_tag = Some("put-etag".to_owned()); + result.version = Some("put-version".to_owned()); + } + Ok(result) } async fn put_multipart_opts( @@ -3840,6 +3849,10 @@ mod tests { } ReadFault::Range(range) => response.range = range.clone(), ReadFault::Transient(_) => {} + ReadFault::DistinctIdentities => { + response.meta.e_tag = Some("get-etag".to_owned()); + response.meta.version = Some("get-version".to_owned()); + } } } Ok(response) @@ -4194,7 +4207,78 @@ mod tests { .await .unwrap(); - assert_eq!(outcome, ImmutableCreateOutcome::Existing(existing)); + let observed = store.head(&path).await.unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Existing { + bytes: existing, + identity: ETag { + e_tag: observed.e_tag, + version: observed.version + }, + } + ); + } + + #[tokio::test] + async fn logical_immutable_identity_belongs_to_the_verified_operation() { + for (existing, verification) in [ + (false, ImmutableWriteVerification::Sha256Checksum), + (false, ImmutableWriteVerification::ReadbackRequired), + (true, ImmutableWriteVerification::ReadbackRequired), + ] { + let inner = Arc::new(InMemory::new()); + let path = Path::from("blobs/operation-identity"); + let bytes = Bytes::from_static(b"candidate"); + if existing { + inner.put(&path, bytes.clone().into()).await.unwrap(); + } + let store = memory_store_with_inner(Arc::new(ReadFaultStore { + inner, + fault: ReadFault::DistinctIdentities, + })) + .with_immutable_write_verification(verification); + let outcome = store + .create_or_read_immutable(&path, bytes.clone(), 1024) + .await + .unwrap(); + let observed = + if existing || verification == ImmutableWriteVerification::ReadbackRequired { + "get" + } else { + "put" + }; + let identity = ETag { + e_tag: Some(format!("{observed}-etag")), + version: Some(format!("{observed}-version")), + }; + let expected = if existing { + ImmutableCreateOutcome::Existing { bytes, identity } + } else { + ImmutableCreateOutcome::Created(identity) + }; + assert_eq!( + outcome, expected, + "existing={existing}, verification={verification:?}" + ); + } + } + + #[tokio::test] + async fn logical_immutable_create_withholds_identity_when_readback_is_corrupt() { + let store = memory_store_with_inner(Arc::new(ReadFaultStore { + inner: Arc::new(InMemory::new()), + fault: ReadFault::Body(vec![Bytes::from_static(b"changed!!")]), + })); + let error = store + .create_or_read_immutable( + &Path::from("blobs/corrupt-readback"), + Bytes::from_static(b"candidate"), + 1024, + ) + .await + .expect_err("invalid bytes cannot acquire a verified identity"); + assert!(matches!(error, StorageError::CorruptObject { .. })); } #[tokio::test] @@ -4232,8 +4316,14 @@ mod tests { .await .unwrap(); - assert_eq!(outcome, ImmutableCreateOutcome::Created); let result = inner.get(&path).await.unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Created(ETag { + e_tag: result.meta.e_tag.clone(), + version: result.meta.version.clone(), + }) + ); assert_eq!( result.attributes.get(&Attribute::StorageClass), Some(&"STANDARD_IA".to_owned().into()) @@ -4269,7 +4359,7 @@ mod tests { .await .unwrap(); - assert!(matches!(outcome, ImmutableCreateOutcome::Existing(_))); + assert!(matches!(outcome, ImmutableCreateOutcome::Existing { .. })); let result = inner.get(&path).await.unwrap(); assert_eq!( result.attributes.get(&Attribute::StorageClass), From c23ec4ffdc36d08eaf8e635f1d5e846d1f3d74a8 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 03:56:49 -0700 Subject: [PATCH 079/127] feat(index): publish capsule chunk candidates Bind shared chunk hints to authenticated source shards and exact immutable object identities. Keep capsule provenance separate from released manifest receipt keys. Draft: exact staged-proof revalidation and current-source real-store qualification remain incomplete. --- crab/docs/design/add.md | 16 + crab/src/git/capsule_push.rs | 82 ++++- crab/src/git/push.rs | 282 ++++++++++++++++-- crab/src/git/xet_publication.rs | 120 +++++--- crab/src/metadata/metadb/mod.rs | 3 +- .../src/metadata/metadb/stores/chunk_index.rs | 54 ++++ .../crab-auth-server/src/receive/capsule.rs | 93 ++++-- .../src/receive/capsule_dependencies.rs | 108 +++++-- .../src/capsule_protocol/chunk_candidates.rs | 171 +++++++++++ .../chunk_candidates/tests.rs | 147 +++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 7 + .../capsule_protocol/pointer/paged/reader.rs | 54 +++- crates/crab-metadata/src/key_codec.rs | 22 ++ crates/crab-metadata/src/remote_index.rs | 183 +++++++++++- .../src/remote_index/capsule_tests.rs | 86 ++++++ 15 files changed, 1297 insertions(+), 131 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs create mode 100644 crates/crab-metadata/src/remote_index/capsule_tests.rs diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index d706ec4b7..94426f02c 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -203,6 +203,22 @@ SlateDB contains only committed, origin-bound receipts and is updated after a successful push CAS; it never stores add claims, local paths, or pending uploads. +Capsule pushes publish chunk hints into the same shared database after ref +visibility. Their head/source keys are separate from the released manifest +receipt keys. A source assertion carries the whole-xorb catalog digest and an +origin receipt captured from the verified PUT or GET; the receipt's payload +digest remains distinct. Readers batch requested chunks, authenticate the +source repository's current paged catalog, check each exact chunk position and +size, then verify canonical origin identity. An absent source or changed +physical descriptor cannot turn an index hint into authority. + +Direct publication and protected receive use the same bounded candidate +encoder/writer. Protected promotion carries the canonical operation identity; +client staging-prefix writes do not confer canonical origin proof. Every index +writer is explicitly closed, including batch failures. A post-commit index +failure leaves a successful ref publication visible and reports repair work; +local packing remains available for future lookup misses. + Concurrent add lookups own their receipt snapshot and validation result across remote awaits. They share keyed advisory caches, not the push pipeline's mutable per-push proof accumulator; otherwise one file can erase another's diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index 5d6c9e128..e0fbb8cc6 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -396,6 +396,7 @@ async fn run_inner( .collect::>() }; let mut xet_stats = super::xet_publication::XetPublicationStats::default(); + let mut pointer_delta = crab_metadata::capsule_protocol::PointerCatalog::new(); let publication: Result>> = async { // LFS bytes share the ref visibility boundary with Git and Xet data. @@ -409,19 +410,21 @@ async fn run_inner( cancel, ) .await?; - let (pointer_delta, prepared_xet_stats) = super::xet_publication::prepare_delta( - &layout, - &base, - &prepared.pointers, - staging, - caching_store, - metrics, - config.upload_concurrency, - config.protected_push.is_none(), - cancel, - ) - .await?; + let (prepared_pointer_delta, prepared_xet_stats) = + super::xet_publication::prepare_delta( + &layout, + &base, + &prepared.pointers, + staging, + caching_store, + metrics, + config.upload_concurrency, + config.protected_push.is_none(), + cancel, + ) + .await?; xet_stats = prepared_xet_stats; + pointer_delta = prepared_pointer_delta; if config.protected_push.is_none() && let Some(replication) = config.active_active_replication.as_ref() { @@ -477,6 +480,7 @@ async fn run_inner( } let result = if changes_namespace { let commit_layout = layout.clone(); + let commit_pointer_delta = &pointer_delta; let commit_push_ref_head_bases = push_ref_head_bases.clone(); crab_write::with_ref_namespaces_wait( layout.store(), @@ -501,7 +505,7 @@ async fn run_inner( base, &transaction, &capsule, - &pointer_delta, + commit_pointer_delta, &lfs_objects, &commit_push_ref_head_bases, ) @@ -545,6 +549,58 @@ async fn run_inner( } }; if let Some(active_active_commit) = committed { + if layout.store().staging_write_prefix().is_none() { + let mut index_config = + crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + if let Some(path) = config.metadb.chunk_index.db.path.as_ref() { + index_config.chunk_index_path.clone_from(path); + } + // The ref is already committed. Index failures only remove acceleration; + // reporting an uncommitted push here would invite a duplicate publication. + match crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &index_config, + layout.repo_prefix(), + &pointer_delta, + &xet_stats.origins, + ) + .await + { + Ok(()) => { + let cache_path = crate::cache::add_remote_candidate_cache_path( + &crate::cache::default_cache_root(), + &store.bucket_identity(), + layout.global_prefix(), + ); + if cache_path.is_file() { + let hashes = pointer_delta + .xorbs() + .values() + .flat_map(|entry| entry.chunks()) + .filter_map(|chunk| MerkleHash::from_hex(chunk.hash()).ok()) + .collect::>(); + let invalidation = tokio::task::spawn_blocking(move || { + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( + &cache_path, + ) + .and_then(|cache| cache.forget_negatives(&hashes)) + }) + .await; + if !matches!(invalidation, Ok(Ok(()))) { + tracing::warn!( + "capsule committed; candidate cache invalidation failed" + ); + } + } + } + Err(error) => { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair") + } + } + } if let Some(snapshot) = staging_snapshot && let Err(error) = snapshot.commit() { diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index c0a87fb0e..571615cfa 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -15015,6 +15015,103 @@ impl PushPipeline { Ok(Some((hits, skipped_remote))) } + async fn lookup_capsule_chunks_for_add( + &self, + hashes: &[MerkleHash], + chunk_store: &crate::metadata::ChunkIndexStore, + ) -> Result> { + let candidates = chunk_store.capsule_candidates(hashes).await?; + let Some(store) = &self.store else { + return Ok(HashMap::new()); + }; + #[derive(Default)] + struct SourceCandidates<'a> { + shards: HashSet<&'a str>, + placements: Vec<&'a crab_metadata::receipts::CommittedChunkPlacement>, + } + let mut by_repo: BTreeMap<&str, SourceCandidates<'_>> = BTreeMap::new(); + for placement in candidates.placements.values() { + if let Some(anchor) = candidates.anchors.get(&placement.source_anchor_id) { + let source = by_repo.entry(&anchor.source_repo_prefix).or_default(); + source.shards.insert(&anchor.source_shard_hash); + source.placements.push(placement); + } + } + let mut refs = HashMap::new(); + let mut origins = HashMap::new(); + let mut proofs = HashMap::new(); + for (repo, source) in by_repo { + check_cancelled(&self.cancel)?; + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + repo.to_owned(), + self.router.global_prefix().to_owned(), + ); + let root = match crab_metadata::capsule_protocol::load_root(&layout).await { + Ok(root) => root, + Err(crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::NotFound { .. }, + }) => continue, + Err(error) => return Err(error.into()), + }; + let lookup = crab_metadata::capsule_protocol::load_pointer_catalog_lookup_from_root( + &layout, &root, + ) + .await?; + let hashes = source + .shards + .into_iter() + .map(str::to_owned) + .collect::>(); + // Validate and release one source closure before opening the next; + // otherwise each source could retain its entire format byte budget. + let catalog = lookup.shards(layout.store(), &hashes).await?; + for placement in source.placements { + let Some(anchor) = candidates.anchors.get(&placement.source_anchor_id) else { + continue; + }; + let xorb_hash = MerkleHash::from(placement.xorb_hash); + let path = self.router.xorb_path(&xorb_hash); + if anchor + .validate_placement(placement, &catalog, path.as_ref()) + .is_err() + { + continue; + } + let hash = MerkleHash::from(placement.chunk_hash); + refs.insert( + hash, + XorbRef { + xorb_hash, + chunk_index: placement.chunk_index, + uncompressed_size: placement.uncompressed_size, + }, + ); + origins.insert(xorb_hash, anchor.origin.clone()); + proofs.insert(hash, (placement.placement_id(), placement.origin_proof_id)); + } + } + // Source catalogs establish commitment; the existing origin verifier + // binds payload/index checks to the exact object version in the receipt. + let verified = self + .verify_xorb_refs_with_committed_receipts(&refs, &origins) + .await?; + Ok(verified + .into_iter() + .filter_map(|(hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proofs.get(&hash).copied()?; + Some(( + hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + }) + .collect()) + } + async fn lookup_proven_remote_chunks_for_add( &self, chunk_hashes: &[MerkleHash], @@ -15024,12 +15121,8 @@ impl PushPipeline { return Ok(HashMap::new()); } check_cancelled(&self.cancel)?; - let remote_candidates = tokio::time::timeout( - GLOBAL_CHUNK_LOOKUP_BUDGET, - chunk_store.get_committed_candidates_batch(chunk_hashes), - ) - .await - .map_err(|_| { + let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; + let timeout_error = || { CrabError::NetworkTransient(object_store::Error::Generic { store: "chunk_index_db", source: Box::new(std::io::Error::new( @@ -15037,7 +15130,27 @@ impl PushPipeline { "add remote classification proof budget exhausted", )), }) - })??; + }; + let mut capsule = tokio::time::timeout_at( + deadline, + self.lookup_capsule_chunks_for_add(chunk_hashes, &chunk_store), + ) + .await + .map_err(|_| timeout_error())??; + let remaining = chunk_hashes + .iter() + .copied() + .filter(|hash| !capsule.contains_key(hash)) + .collect::>(); + if remaining.is_empty() { + return Ok(capsule); + } + let remote_candidates = tokio::time::timeout_at( + deadline, + chunk_store.get_committed_candidates_batch(&remaining), + ) + .await + .map_err(|_| timeout_error())??; let mut refs = HashMap::with_capacity(remote_candidates.placements.len()); let mut proof_ids = HashMap::with_capacity(remote_candidates.placements.len()); @@ -15066,20 +15179,18 @@ impl PushPipeline { let verified = self .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins) .await?; - Ok(verified - .into_iter() - .filter_map(|(chunk_hash, xorb_ref)| { - let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; - (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( - chunk_hash, - crab_staging::push_plan::ExistingChunkCandidate { - xorb_ref, - placement_id, - origin_proof_id, - }, - )) - }) - .collect()) + capsule.extend(verified.into_iter().filter_map(|(chunk_hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; + (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( + chunk_hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + })); + Ok(capsule) } async fn lookup_cache_service_chunk_refs( @@ -22051,6 +22162,135 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { + use crab_staging::push_plan::ExistingChunkLookup; + use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; + + let _git_env = CleanGitEnvGuard::new(); + let source = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .args(args) + .current_dir(source.path()) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + git(&["init", "-b", "main"]); + let staging_root = source.path().join("staging"); + let staging = crab_staging::StagingArea::open(staging_root.clone()) + .await + .unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(b"shared capsule payload"), + std::path::Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let chunks = staging + .recipe_page(&staged.recipe, 0) + .unwrap() + .chunks + .into_iter() + .map(|chunk| (chunk.chunk_hash, chunk.len)) + .collect::>(); + staging + .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + let pointer = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); + git(&["add", "model.bin"]); + git(&["commit", "-m", "shared payload"]); + staging.close().await.unwrap(); + let staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(staging_root) + .await + .unwrap(), + ); + let (store, source_router) = test_store_router("repos/source"); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..Default::default() + }; + let (result, _) = super::super::capsule_push::run( + &config, + &[crate::git::remote_helper::PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &source_router, + None, + &[], + Some(&staging), + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(result.all_ok(), "source publication failed: {result:?}"); + drop(staging); + + let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); + let pipeline = PushPipeline::new( + PushConfig::default(), + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + CancellationToken::new(), + None, + ); + let cache_root = tempfile::tempdir().unwrap(); + let mut metadata = crate::core::config::MetaDbTomlConfig::default(); + metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard( + &store, &router, None, &metadata, true, + )); + // No advisory caches or source-repository state can manufacture a hit. + // This exercises the real publisher-to-classifier authority boundary. + let classifier = AddRemoteChunkClassifier { + pipeline, + remote_chunk_index: tokio::sync::OnceCell::new(), + candidate_cache: None, + candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), + candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), + }; + let candidates = classifier.lookup_existing_candidates(&chunks).await; + classifier.close().await; + let candidates = candidates.unwrap(); + assert!( + !candidates.is_empty() && candidates.iter().all(Option::is_some), + "committed source chunks must be reusable from a cold consumer: {candidates:?}" + ); + } + fn add_classifier_with_private_cache( store: &Store, router: &StoreLayout, diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 4029b8c41..b9b2c57d1 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -23,11 +23,12 @@ use crate::core::metrics::Metrics; // existing payload bound even when a caller requests more network concurrency. const MAX_IN_FLIGHT_XORBS: usize = 4; -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub(crate) struct XetPublicationStats { pub xorbs_uploaded: u64, pub shards_uploaded: u64, pub xorb_bytes_uploaded: u64, + pub origins: HashMap, } #[expect( @@ -133,6 +134,7 @@ pub(crate) async fn prepare_delta( let mut shard_session = PushShardSession::new(); let mut pending_files = Vec::with_capacity(unresolved.len()); let mut uploaded_xorbs = HashSet::new(); + let mut origins = HashMap::new(); let mut xorb_bytes_uploaded = 0_u64; let mut prepared_files = Vec::with_capacity(unresolved.len()); let mut prepared_candidates = Vec::::new(); @@ -278,18 +280,19 @@ pub(crate) async fn prepare_delta( |(ordinal, (expected_hash, expected_refs))| async move { check_cancelled(cancel)?; let path = layout.xorb_path(&expected_hash); - let bytes = match layout + let (bytes, identity) = match layout .store() .get_with_etag_bounded(&path, MAX_XORB_SIZE as u64) .await { - Ok((bytes, _)) => bytes, + Ok(result) => result, Err(crab_storage::StorageError::NotFound { .. }) => { return Ok::<_, CrabError>((ordinal, None)); } Err(error) => return Err(error.into()), }; - let verified = verify_external_xorb(&path, expected_hash, &expected_refs, bytes)?; + let verified = + verify_external_xorb(&path, expected_hash, &expected_refs, bytes, identity)?; Ok((ordinal, verified)) }, )) @@ -302,10 +305,11 @@ pub(crate) async fn prepare_delta( verified_external.sort_by_key(|(ordinal, _)| *ordinal); let mut reused_xorbs = 0_usize; for (_, verified) in verified_external { - let Some((xorb_hash, entry, xorb_placements)) = verified else { + let Some((xorb_hash, entry, xorb_placements, origin)) = verified else { continue; }; reused_xorbs += 1; + origins.insert(xorb_hash, origin); for placement in xorb_placements { placements.entry(placement.chunk_hash).or_insert(placement); } @@ -375,7 +379,7 @@ pub(crate) async fn prepare_delta( let bytes = Bytes::from(bytes); let (entry, xorb_placements) = verify_prepared_xorb(&path, planned_hash, &planned, bytes.clone())?; - let (entry, xorb_placements, created) = publish_xorb_candidate( + let (entry, xorb_placements, created, origin) = publish_xorb_candidate( layout, caching_store, planned_hash, @@ -391,6 +395,7 @@ pub(crate) async fn prepare_delta( entry, xorb_placements, uploaded_bytes, + origin, )) }, )) @@ -400,8 +405,11 @@ pub(crate) async fn prepare_delta( while let Some(result) = uploads.next().await { verified_candidates.push(result?); } - verified_candidates.sort_by_key(|(ordinal, _, _, _, _)| *ordinal); - for (_, planned_hash, entry, xorb_placements, uploaded_bytes) in verified_candidates { + verified_candidates.sort_by_key(|(ordinal, _, _, _, _, _)| *ordinal); + for (_, planned_hash, entry, xorb_placements, uploaded_bytes, origin) in verified_candidates { + if let Some(origin) = origin { + origins.insert(planned_hash, origin); + } if uploaded_bytes > 0 { uploaded_xorbs.insert(planned_hash); xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -449,6 +457,7 @@ pub(crate) async fn prepare_delta( &mut placements, &mut xorb_entries, &mut uploaded_xorbs, + &mut origins, ) .await?; xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -466,6 +475,7 @@ pub(crate) async fn prepare_delta( &mut placements, &mut xorb_entries, &mut uploaded_xorbs, + &mut origins, ) .await?; xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -608,6 +618,7 @@ pub(crate) async fn prepare_delta( xorbs_uploaded: uploaded_xorbs.len() as u64, shards_uploaded, xorb_bytes_uploaded, + origins, }, )) } @@ -619,11 +630,12 @@ async fn publish_built_xorb( placements: &mut ChunkPlacementMap, xorb_entries: &mut HashMap, uploaded_xorbs: &mut HashSet, + origins: &mut HashMap, ) -> Result { let encoded_size = result.bytes.len() as u64; let body_digest = blake3::hash(&result.bytes).to_hex().to_string(); let entry = catalog_xorb_entry(result.bytes.len() as u64, body_digest, &result.placements)?; - let (entry, xorb_placements, created) = publish_xorb_candidate( + let (entry, xorb_placements, created, origin) = publish_xorb_candidate( layout, caching_store, result.hash, @@ -632,6 +644,9 @@ async fn publish_built_xorb( result.placements, ) .await?; + if let Some(origin) = origin { + origins.insert(result.hash, origin); + } if created { uploaded_xorbs.insert(result.hash); } @@ -653,6 +668,7 @@ async fn publish_xorb_candidate( crab_metadata::capsule_protocol::XorbCatalogEntry, Vec, bool, + Option, )> { let path = layout.xorb_path(&xorb_hash); let staged = layout.store().staging_write_prefix().is_some(); @@ -691,16 +707,22 @@ async fn publish_xorb_candidate( .get_with_etag_bounded(&path, MAX_XORB_SIZE as u64) .await { - Ok((existing, _)) => { - let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? - .ok_or_else(|| CrabError::CorruptObject { - path: path.to_string(), - reason: format!( - "existing xorb {} does not authenticate as the requested logical content", - xorb_hash.hex() - ), - })?; - return Ok((verified.1, verified.2, false)); + Ok((existing, identity)) => { + let verified = verify_external_xorb( + &path, + xorb_hash, + &expected_refs, + existing, + identity, + )? + .ok_or_else(|| CrabError::CorruptObject { + path: path.to_string(), + reason: format!( + "existing xorb {} does not authenticate as the requested logical content", + xorb_hash.hex() + ), + })?; + return Ok((verified.1, verified.2, false, Some(verified.3))); } Err(crab_storage::StorageError::NotFound { .. }) => {} Err(error) => return Err(error.into()), @@ -711,15 +733,18 @@ async fn publish_xorb_candidate( // canonical encoding above when present; otherwise record verified // staged bytes for service-owned promotion before ref publication. let created = layout.store().put_if_absent_verified(&path, bytes).await?; - return Ok((local_entry, local_placements, created)); + return Ok((local_entry, local_placements, created, None)); } let cache_bytes = caching_store.map(|_| bytes.clone()); + // Local builder/prepared verification already authenticated these bytes. + // Carry the payload digest with the exact immutable PUT/readback identity. + let payload_digest = XorbParser::parse(bytes.clone())?.payload_digest(); match layout .store() .create_or_read_immutable(&path, bytes, MAX_XORB_SIZE as u64) .await? { - crab_storage::ImmutableCreateOutcome::Created(_) => { + crab_storage::ImmutableCreateOutcome::Created(identity) => { if let (Some(cache), Some(bytes)) = (caching_store, cache_bytes) { warm_published_cache_object( cache, @@ -729,20 +754,36 @@ async fn publish_xorb_candidate( ) .await; } - Ok((local_entry, local_placements, true)) + let origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + xorb_hash.into(), + payload_digest, + local_entry.encoded_size(), + identity.e_tag, + identity.version, + ); + Ok((local_entry, local_placements, true, Some(origin))) } crab_storage::ImmutableCreateOutcome::Existing { - bytes: existing, .. + bytes: existing, + identity, } => { - let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? - .ok_or_else(|| CrabError::CorruptObject { - path: path.to_string(), - reason: format!( - "existing xorb {} does not authenticate as the requested logical content", - xorb_hash.hex() - ), - })?; - Ok((verified.1, verified.2, false)) + let verified = verify_external_xorb( + &path, + xorb_hash, + &expected_refs, + existing, + identity, + )? + .ok_or_else(|| CrabError::CorruptObject { + path: path.to_string(), + reason: format!( + "existing xorb {} does not authenticate as the requested logical content", + xorb_hash.hex() + ), + })?; + Ok((verified.1, verified.2, false, Some(verified.3))) } } } @@ -844,11 +885,13 @@ fn verify_external_xorb( expected_hash: MerkleHash, expected_refs: &HashMap, bytes: Bytes, + identity: crab_storage::ETag, ) -> Result< Option<( MerkleHash, crab_metadata::capsule_protocol::XorbCatalogEntry, Vec, + crab_metadata::receipts::OriginReceipt, )>, > { let encoded_size = bytes.len() as u64; @@ -921,7 +964,16 @@ fn verify_external_xorb( } } let entry = catalog_xorb_entry(encoded_size, body_digest, &xorb_placements)?; - Ok(Some((expected_hash, entry, xorb_placements))) + let origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + expected_hash.into(), + parser.payload_digest(), + encoded_size, + identity.e_tag, + identity.version, + ); + Ok(Some((expected_hash, entry, xorb_placements, origin))) } fn verify_prepared_xorb( @@ -1685,7 +1737,7 @@ mod tests { layout.store().clone() }; let layout = crab_storage::StoreLayout::new(store, "repo".to_owned()); - let (entry, placements, created) = publish_xorb_candidate( + let (entry, placements, created, _origin) = publish_xorb_candidate( &layout, None, candidate.hash, diff --git a/crab/src/metadata/metadb/mod.rs b/crab/src/metadata/metadb/mod.rs index a4e4db9a0..7d5344963 100644 --- a/crab/src/metadata/metadb/mod.rs +++ b/crab/src/metadata/metadb/mod.rs @@ -27,6 +27,7 @@ use tracing::{debug, info, warn}; use crate::core::error::{CrabError, MetaDbError, Result}; use crate::core::metrics::Metrics; use crab_metadata::chunk_index::ChunkIndex; +use crab_metadata::key_codec::METADB_FORMAT_VERSION; use crab_metadata::persistent_chunk_index::PersistentChunkIndex; use crab_metadata::{key_codec, value_codec}; @@ -66,8 +67,6 @@ const DEFAULT_WAL_FLUSH_SIZE: u64 = 4 * 1024 * 1024; /// lookups. const DEFAULT_BLOOM_BITS_PER_KEY: u32 = 10; -const METADB_FORMAT_VERSION: u32 = 1; - /// Aggregated snapshot of `sys:*` keys in one database. /// /// Returned by [`MetaDb::file_index_system_keys`] and diff --git a/crab/src/metadata/metadb/stores/chunk_index.rs b/crab/src/metadata/metadb/stores/chunk_index.rs index 341d53898..89dd9597a 100644 --- a/crab/src/metadata/metadb/stores/chunk_index.rs +++ b/crab/src/metadata/metadb/stores/chunk_index.rs @@ -104,6 +104,60 @@ impl ChunkIndexStore { self } + /// Load capsule hints without interpreting them as released manifest receipts. + pub(crate) async fn capsule_candidates( + &self, + hashes: &[MerkleHash], + ) -> Result { + use crab_metadata::capsule_protocol::{ + CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, + capsule_chunk_head_key, + }; + let mut output = CapsuleChunkCandidates::default(); + // Stay below Db's dense range-scan threshold: unrelated chunk history + // must not turn an add-time point batch into a scan of the global index. + for hashes in hashes.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { + let keys = hashes + .iter() + .map(|hash| Bytes::copy_from_slice(&capsule_chunk_head_key(&(*hash).into()))) + .collect::>(); + let values = self.db.get_batch(&keys).await?; + let mut anchor_ids = HashSet::new(); + for (hash, value) in hashes.iter().zip(values) { + let Some(value) = value else { + continue; + }; + let placement = CommittedChunkPlacement::decode(&value)?; + if placement.chunk_hash != <[u8; 32]>::from(*hash) { + return Err(CrabError::StagingCorrupt( + "capsule candidate hash differs from index key".to_owned(), + )); + } + anchor_ids.insert(placement.source_anchor_id); + output.placements.insert(placement.chunk_hash, placement); + } + let ids = anchor_ids + .into_iter() + .filter(|id| !output.anchors.contains_key(id)) + .collect::>(); + let keys = ids + .iter() + .map(|id| Bytes::copy_from_slice(&capsule_chunk_anchor_key(id))) + .collect::>(); + let values = self.db.get_batch(&keys).await?; + for (id, value) in ids.into_iter().zip(values) { + let value = value.ok_or_else(|| { + CrabError::StagingCorrupt( + "capsule candidate source anchor is absent".to_owned(), + ) + })?; + let anchor = CapsuleChunkAnchor::decode(&id, &value)?; + output.anchors.insert(id, anchor); + } + } + Ok(output) + } + /// Look up a single chunk hash through all three tiers. /// /// Cache-first: in-memory → persistent, when available → remote. diff --git a/crates/crab-auth-server/src/receive/capsule.rs b/crates/crab-auth-server/src/receive/capsule.rs index 3a6475ac3..1a6a613eb 100644 --- a/crates/crab-auth-server/src/receive/capsule.rs +++ b/crates/crab-auth-server/src/receive/capsule.rs @@ -1,6 +1,6 @@ //! Protocol-v2 protected-push candidate verification. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use crab_auth::PushRefUpdate; use crab_coordination::write_coordinator::{CommitOutcome, CoordinatedRefUpdate}; @@ -27,6 +27,7 @@ pub(super) struct VerifiedCapsuleCandidate { pub replication_objects: Vec, pub publication: Capsule, dependency_copies: Vec, + origins: HashMap, } pub(super) async fn commit_capsule_candidate( @@ -102,9 +103,29 @@ pub(super) async fn commit_capsule_candidate( .map(Some); } - promote_staged_writes(ctx.store(), &plan.staged_objects).await?; - promote_dependency_copies(ctx, &verified.dependency_copies).await?; - if let Some(delta) = capsule.pointer_catalog_delta()? { + let mut origins = verified.origins; + let mut other_objects = Vec::new(); + for object in &plan.staged_objects { + if crab_storage::content_hash_from_path(&object.canonical_key, "xorbs").is_some() { + let bytes = read_verified_staged_object(ctx.store(), object).await?; + let origin = super::capsule_dependencies::promote_xorb_origin( + ctx.store(), + &object.canonical_key, + bytes, + ) + .await?; + origins.insert( + crab_xet::hash::MerkleHash::from(origin.content_hash), + origin, + ); + } else { + other_objects.push(object.clone()); + } + } + promote_staged_writes(ctx.store(), &other_objects).await?; + origins.extend(promote_dependency_copies(ctx, &verified.dependency_copies).await?); + let delta = capsule.pointer_catalog_delta()?.unwrap_or_default(); + if !delta.is_empty() { crab_metadata::ref_registry::union_register_repo_shards( ctx.store(), ctx.router(), @@ -122,12 +143,12 @@ pub(super) async fn commit_capsule_candidate( .iter() .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); let base = view.root_snapshot().clone(); - if changes_namespace { + let result = if changes_namespace { let layout = ctx.router().clone(); let repo_prefix = ctx.repo_prefix().to_owned(); let active_active = active_active.cloned(); let replication_objects = replication_objects.to_vec(); - return crab_write::with_ref_namespaces( + crab_write::with_ref_namespaces( ctx.store(), ctx.router(), &ref_names, @@ -158,19 +179,38 @@ pub(super) async fn commit_capsule_candidate( .await }, ) - .await; + .await + } else { + publish_verified_candidate( + ctx.store(), + ctx.router(), + ctx.repo_prefix(), + base, + &transaction, + &capsule, + active_active, + replication_objects, + ) + .await + }; + if result.is_ok() { + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + ctx.repo_prefix(), + ctx.router().global_prefix(), + ); + if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( + std::sync::Arc::clone(ctx.store().inner()), + &config, + ctx.repo_prefix(), + &delta, + &origins, + ) + .await + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + } } - publish_verified_candidate( - ctx.store(), - ctx.router(), - ctx.repo_prefix(), - base, - &transaction, - &capsule, - active_active, - replication_objects, - ) - .await + result } async fn publish_verified_candidate( @@ -358,7 +398,7 @@ pub(super) async fn verify_capsule_candidate( validate_capsule_transaction(plan, candidate)?; let candidate_delta = candidate.pointer_catalog_delta()?.unwrap_or_default(); - let (changed_paths, publication, replication_objects, dependency_copies) = + let (changed_paths, publication, replication_objects, dependency_copies, origins) = match prepare.view_scope.as_ref() { None => { if prepared_root != plan.base_root_digest { @@ -382,9 +422,9 @@ pub(super) async fn verify_capsule_candidate( ) .await?; catalog.apply(&candidate_delta)?; - let (objects, copies) = + let (objects, copies, origins) = verify_pointer_dependencies(ctx, plan, &catalog, &pointers, None).await?; - (changed_paths, candidate.clone(), objects, copies) + (changed_paths, candidate.clone(), objects, copies, origins) } Some(scope) => { let filtered_router = crab_storage::StoreLayout::with_global_prefix( @@ -445,7 +485,7 @@ pub(super) async fn verify_capsule_candidate( )?; let mut final_catalog = source_catalog; final_catalog.apply(&delta)?; - let (objects, copies) = verify_pointer_dependencies( + let (objects, copies, origins) = verify_pointer_dependencies( ctx, plan, &final_catalog, @@ -465,7 +505,13 @@ pub(super) async fn verify_capsule_candidate( } let publication = Capsule::build(&materialized.transaction, materialized.git_packs, sections)?; - (materialized.changed_paths, publication, objects, copies) + ( + materialized.changed_paths, + publication, + objects, + copies, + origins, + ) } }; let publication_transaction = publication.transaction()?; @@ -496,6 +542,7 @@ pub(super) async fn verify_capsule_candidate( replication_objects: replication_objects.into_iter().collect(), publication, dependency_copies, + origins, }) } diff --git a/crates/crab-auth-server/src/receive/capsule_dependencies.rs b/crates/crab-auth-server/src/receive/capsule_dependencies.rs index a3d692b3e..059be9987 100644 --- a/crates/crab-auth-server/src/receive/capsule_dependencies.rs +++ b/crates/crab-auth-server/src/receive/capsule_dependencies.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use bytes::Bytes; use crab_metadata::capsule_protocol::PointerCatalog; @@ -130,7 +130,11 @@ pub(super) async fn verify_pointer_dependencies( catalog: &PointerCatalog, pointers: &ReachablePointerScan, fallback: Option<&crab_storage::StoreLayout>, -) -> Result<(BTreeSet, Vec)> { +) -> Result<( + BTreeSet, + Vec, + HashMap, +)> { let mut referenced = BTreeSet::from([ctx .router() .capsule_path(&plan.run_hash) @@ -139,6 +143,7 @@ pub(super) async fn verify_pointer_dependencies( let mut verified_shards = BTreeMap::::new(); let mut verified_xorbs = BTreeSet::new(); let mut copies = Vec::new(); + let mut origins = HashMap::new(); for pointer in &pointers.crab_pointers { let file_hash = MerkleHash::from(pointer.file_hash).hex(); let file = catalog @@ -163,7 +168,7 @@ pub(super) async fn verify_pointer_dependencies( Some(bytes) => bytes.clone(), None => { let fallback_key = fallback.map(|layout| layout.shard_path(&shard_hash)); - let bytes = read_candidate_object( + let (bytes, _) = read_candidate_object( ctx, plan, &shard_key, @@ -217,7 +222,7 @@ pub(super) async fn verify_pointer_dependencies( referenced.insert(key.clone()); if verified_xorbs.insert(hash.to_owned()) { let fallback_key = fallback.map(|layout| layout.xorb_path(&xorb_hash)); - let bytes = read_candidate_object( + let (bytes, identity) = read_candidate_object( ctx, plan, &key, @@ -232,6 +237,12 @@ pub(super) async fn verify_pointer_dependencies( )); } validate_staged_xorb(&key, &bytes, &chunks)?; + if let Some(identity) = identity { + origins.insert( + xorb_hash, + verified_xorb_origin(&key, bytes.clone(), identity)?, + ); + } } } } @@ -252,7 +263,8 @@ pub(super) async fn verify_pointer_dependencies( ) .to_string(); referenced.insert(key.clone()); - let bytes = read_candidate_object(ctx, plan, &key, None, pointer.size, &mut copies).await?; + let (bytes, _) = + read_candidate_object(ctx, plan, &key, None, pointer.size, &mut copies).await?; if <[u8; 32]>::from(Sha256::digest(&bytes)) != pointer.oid { return Err(invalid( "candidate LFS object digest differs from its pointer", @@ -278,7 +290,7 @@ pub(super) async fn verify_pointer_dependencies( } } copies.dedup_by(|left, right| left.target_key == right.target_key); - Ok((referenced, copies)) + Ok((referenced, copies, origins)) } async fn read_candidate_object( @@ -288,8 +300,8 @@ async fn read_candidate_object( fallback_key: Option<&object_store::path::Path>, expected_size: u64, copies: &mut Vec, -) -> Result { - let bytes = match plan +) -> Result<(Bytes, Option)> { + let (bytes, identity) = match plan .staged_objects .iter() .find(|object| object.canonical_key == canonical_key) @@ -300,7 +312,10 @@ async fn read_candidate_object( "staged dependency size differs from the candidate catalog", )); } - read_verified_staged_object(ctx.store(), object).await? + ( + read_verified_staged_object(ctx.store(), object).await?, + None, + ) } None => match ctx .store() @@ -310,7 +325,7 @@ async fn read_candidate_object( ) .await { - Ok((bytes, _)) => bytes, + Ok((bytes, identity)) => (bytes, Some(identity)), Err(crab_storage::StorageError::NotFound { .. }) => { let fallback_key = fallback_key.ok_or_else(|| AuthServerError::NotFound { path: canonical_key.to_owned(), @@ -325,7 +340,7 @@ async fn read_candidate_object( size: expected_size, digest: blake3::hash(&bytes).to_hex().to_string(), }); - bytes + (bytes, None) } Err(error) => return Err(error.into()), }, @@ -335,13 +350,14 @@ async fn read_candidate_object( "candidate dependency size differs from the candidate catalog", )); } - Ok(bytes) + Ok((bytes, identity)) } pub(super) async fn promote_dependency_copies( ctx: &ReceiveContext, copies: &[DependencyCopy], -) -> Result<()> { +) -> Result> { + let mut origins = HashMap::new(); for copy in copies { let (bytes, _) = ctx .store() @@ -356,12 +372,64 @@ pub(super) async fn promote_dependency_copies( "filtered view dependency changed after protected verification", )); } - ctx.store() - .put_if_absent_verified( - &object_store::path::Path::from(copy.target_key.clone()), - bytes, - ) - .await?; + if content_hash_from_path(©.target_key, "xorbs").is_some() { + let origin = promote_xorb_origin(ctx.store(), ©.target_key, bytes).await?; + origins.insert(MerkleHash::from(origin.content_hash), origin); + } else { + ctx.store() + .put_if_absent_verified( + &object_store::path::Path::from(copy.target_key.clone()), + bytes, + ) + .await?; + } } - Ok(()) + Ok(origins) +} + +// Verification checked the complete body before promotion. Keep the immutable +// operation identity with that body so a later HEAD cannot certify a replacement. +pub(super) async fn promote_xorb_origin( + store: &crab_storage::Store, + key: &str, + bytes: Bytes, +) -> Result { + let expected = *blake3::hash(&bytes).as_bytes(); + let maximum = bytes.len() as u64; + let outcome = store + .create_or_read_immutable(&object_store::path::Path::from(key), bytes.clone(), maximum) + .await?; + let identity = match outcome { + crab_storage::ImmutableCreateOutcome::Created(identity) => identity, + crab_storage::ImmutableCreateOutcome::Existing { + bytes: stored, + identity, + } => { + if *blake3::hash(&stored).as_bytes() != expected { + return Err(conflict( + "canonical xorb differs from the verified protected body", + )); + } + identity + } + }; + verified_xorb_origin(key, bytes, identity) +} + +fn verified_xorb_origin( + key: &str, + bytes: Bytes, + identity: crab_storage::ETag, +) -> Result { + let size = bytes.len() as u64; + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes)?; + Ok(crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + key.to_owned(), + parser.hash().into(), + parser.payload_digest(), + size, + identity.e_tag, + identity.version, + )) } diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs new file mode 100644 index 000000000..9553a0c83 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs @@ -0,0 +1,171 @@ +//! Rebuildable chunk hints whose source authority is a capsule catalog. +use std::collections::{BTreeMap, HashMap}; + +use serde::{Deserialize, Serialize}; + +use super::XorbCatalogEntry; +use crate::error::{MetadataError, Result}; +use crate::receipts::{CommittedChunkPlacement, OriginReceipt, RECEIPT_SCHEMA_VERSION}; +use crab_xet::hash::MerkleHash; + +pub use crate::key_codec::{capsule_chunk_anchor_key, capsule_chunk_head_key}; + +/// Capsule source and verified physical origin for one xorb. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapsuleChunkAnchor { + pub source_repo_prefix: String, + pub source_shard_hash: String, + pub body_digest: String, + pub origin: OriginReceipt, +} + +impl CapsuleChunkAnchor { + /// Decode bounded provenance and verify its immutable index identity. + pub fn decode(id: &[u8; 32], bytes: &[u8]) -> Result { + if bytes.len() > 16 * 1024 { + return Err(corrupt("capsule source anchor exceeds its byte bound")); + } + let anchor: Self = serde_json::from_slice(bytes) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if anchor.id()? != *id { + return Err(corrupt("capsule source anchor digest differs")); + } + Ok(anchor) + } + + /// Identify this immutable source and origin assertion. + pub fn id(&self) -> Result<[u8; 32]> { + if self.source_repo_prefix.len() > 4096 + || self.source_shard_hash.len() != 64 + || self.body_digest.len() != 64 + { + return Err(corrupt("invalid capsule source identity")); + } + let bytes = serde_json::to_vec(self) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if bytes.len() > 16 * 1024 { + return Err(corrupt("capsule source anchor exceeds its byte bound")); + } + let mut hash = blake3::Hasher::new_derive_key("crab capsule chunk source v1"); + hash.update(&bytes); + Ok(*hash.finalize().as_bytes()) + } + + /// Check an exact placement against its committed shard closure and origin assertion. + pub fn validate_placement( + &self, + placement: &CommittedChunkPlacement, + catalog: &super::PointerCatalog, + object_key: &str, + ) -> Result<()> { + placement.validate()?; + let xorb_hash = MerkleHash::from(placement.xorb_hash).hex(); + let rooted = catalog + .shards() + .get(&self.source_shard_hash) + .is_some_and(|shard| shard.xorb_hashes().binary_search(&xorb_hash).is_ok()); + let entry = catalog + .xorbs() + .get(&xorb_hash) + .ok_or_else(|| corrupt("capsule placement xorb is absent from source catalog"))?; + self.validate_catalog_xorb(object_key, entry)?; + let chunk = entry.chunks().get(placement.chunk_index as usize); + if !rooted + || placement.source_anchor_id != self.id()? + || placement.origin_proof_id != self.origin.proof_id() + || placement.xorb_hash != self.origin.content_hash + || !chunk.is_some_and(|chunk| { + chunk.hash() == MerkleHash::from(placement.chunk_hash).hex() + && chunk.uncompressed_size() == placement.uncompressed_size + }) + { + return Err(corrupt( + "capsule placement differs from its source shard or origin", + )); + } + Ok(()) + } + + /// Check the source catalog's physical descriptor before verifying origin availability. + pub fn validate_catalog_xorb(&self, object_key: &str, entry: &XorbCatalogEntry) -> Result<()> { + if self.source_repo_prefix.is_empty() || self.body_digest != entry.body_digest() { + return Err(corrupt( + "source catalog differs from the capsule chunk anchor", + )); + } + self.origin.validate( + "canonical-origin", + object_key, + self.origin.content_hash, + entry.encoded_size(), + ) + } +} + +/// Untrusted hints, requiring current source-catalog and canonical-origin checks. +#[derive(Debug, Default)] +pub struct CapsuleChunkCandidates { + pub placements: HashMap<[u8; 32], CommittedChunkPlacement>, + pub anchors: HashMap<[u8; 32], CapsuleChunkAnchor>, +} + +/// Encode one bounded xorb's candidates after full verification and ref publication. +pub(crate) fn capsule_chunk_candidate_rows( + repo_prefix: &str, + source_shard_hash: &str, + xorb_hash: MerkleHash, + entry: &XorbCatalogEntry, + origin: &OriginReceipt, +) -> Result, Vec)>> { + let anchor = CapsuleChunkAnchor { + source_repo_prefix: repo_prefix.to_owned(), + source_shard_hash: source_shard_hash.to_owned(), + body_digest: entry.body_digest().to_owned(), + origin: origin.clone(), + }; + anchor.validate_catalog_xorb(&origin.object_key, entry)?; + if origin.content_hash != <[u8; 32]>::from(xorb_hash) { + return Err(corrupt("origin names a different xorb")); + } + let anchor_id = anchor.id()?; + let mut rows = vec![( + capsule_chunk_anchor_key(&anchor_id).to_vec(), + serde_json::to_vec(&anchor) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?, + )]; + let mut placements = BTreeMap::new(); + for (index, chunk) in entry.chunks().iter().enumerate() { + let chunk_hash = + MerkleHash::from_hex(chunk.hash()).map_err(|_| corrupt("invalid chunk hash"))?; + placements + .entry(<[u8; 32]>::from(chunk_hash)) + .or_insert(CommittedChunkPlacement { + schema_version: RECEIPT_SCHEMA_VERSION, + chunk_hash: chunk_hash.into(), + xorb_hash: xorb_hash.into(), + chunk_index: u32::try_from(index) + .map_err(|_| corrupt("chunk position overflows"))?, + uncompressed_size: chunk.uncompressed_size(), + origin_proof_id: origin.proof_id(), + source_anchor_id: anchor_id, + }); + } + for (hash, placement) in placements { + rows.push(( + capsule_chunk_head_key(&hash).to_vec(), + placement.encode()?.to_vec(), + )); + } + Ok(rows) +} + +fn corrupt(reason: &str) -> MetadataError { + MetadataError::CorruptObject { + path: "capsule chunk candidates".to_owned(), + reason: reason.to_owned(), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs new file mode 100644 index 000000000..aff1436c4 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs @@ -0,0 +1,147 @@ +use super::*; +use crate::capsule_protocol::XorbChunkEntry; + +fn fixture() -> (MerkleHash, XorbCatalogEntry, OriginReceipt) { + let hash = MerkleHash::from([3; 32]); + let entry = XorbCatalogEntry::new( + 100, + blake3::hash(b"whole body").to_hex().to_string(), + vec![ + XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9), + XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9), + ], + ); + let mut origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", hash.hex()), + hash.into(), + *blake3::hash(b"payload").as_bytes(), + 100, + Some("etag".to_owned()), + Some("version".to_owned()), + ); + origin.proven_at_unix_secs = 1; + (hash, entry, origin) +} + +#[test] +fn capsule_candidates_preserve_distinct_digests_and_first_occurrence() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + assert_eq!( + ( + rows.len(), + placement.chunk_index, + anchor.body_digest, + anchor.origin.payload_digest + ), + (2, 0, entry.body_digest().to_owned(), origin.payload_digest) + ); +} + +#[test] +fn capsule_anchor_rejects_changed_source_identity() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let mut anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + anchor.source_repo_prefix = "other".to_owned(); + assert!( + CapsuleChunkAnchor::decode( + &placement.source_anchor_id, + &serde_json::to_vec(&anchor).unwrap() + ) + .is_err() + ); +} + +#[test] +fn capsule_anchor_requires_the_authenticated_physical_descriptor() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + let other = XorbCatalogEntry::new(100, "4".repeat(64), entry.chunks().to_vec()); + assert!( + anchor + .validate_catalog_xorb(&origin.object_key, &other) + .is_err() + ); +} + +#[test] +fn capsule_anchor_rejects_oversized_records_before_decoding() { + assert!(CapsuleChunkAnchor::decode(&[1; 32], &vec![b' '; 16 * 1024 + 1]).is_err()); +} + +fn rooted_fixture() -> ( + CapsuleChunkAnchor, + CommittedChunkPlacement, + crate::capsule_protocol::PointerCatalog, +) { + let (hash, entry, origin) = fixture(); + let shard = "2".repeat(64); + let rows = capsule_chunk_candidate_rows("repo", &shard, hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + let mut catalog = crate::capsule_protocol::PointerCatalog::new(); + catalog.insert_xorb(hash.hex(), entry).unwrap(); + catalog + .insert_shard( + shard, + crate::capsule_protocol::ShardCatalogEntry::new(20, vec![hash.hex()]), + ) + .unwrap(); + (anchor, placement, catalog) +} + +#[test] +fn capsule_placement_requires_an_authenticated_source_shard() { + let (anchor, placement, rooted) = rooted_fixture(); + let mut unrooted = crate::capsule_protocol::PointerCatalog::new(); + for (hash, entry) in rooted.xorbs() { + unrooted.insert_xorb(hash.clone(), entry.clone()).unwrap(); + } + assert!( + anchor + .validate_placement(&placement, &unrooted, &anchor.origin.object_key) + .is_err() + ); +} + +#[test] +fn capsule_placement_accepts_exact_rooted_content() { + let (anchor, placement, catalog) = rooted_fixture(); + assert!( + anchor + .validate_placement(&placement, &catalog, &anchor.origin.object_key) + .is_ok() + ); +} + +#[test] +fn capsule_placement_rejects_tampered_occurrences_and_proofs() { + let (anchor, placement, catalog) = rooted_fixture(); + for field in ["size", "position", "chunk", "origin", "source"] { + let mut changed = placement.clone(); + match field { + "size" => changed.uncompressed_size += 1, + "position" => changed.chunk_index = 2, + "chunk" => changed.chunk_hash = [9; 32], + "origin" => changed.origin_proof_id = [9; 32], + "source" => changed.source_anchor_id = [9; 32], + _ => unreachable!(), + } + assert!( + anchor + .validate_placement(&changed, &catalog, &anchor.origin.object_key) + .is_err(), + "accepted changed {field}" + ); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 17b51aaab..07c1359c9 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -10,6 +10,7 @@ mod capsule; mod captured_visibility; #[cfg(feature = "storage")] mod catalog_capture; +mod chunk_candidates; mod history; mod layered; #[cfg(feature = "storage")] @@ -123,3 +124,9 @@ fn valid_ref_namespace<'a>(names: impl IntoIterator) -> bool { .all(|(index, _)| !names.contains(&name[..index])) }) } + +pub use chunk_candidates::{ + CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, capsule_chunk_head_key, +}; + +pub(crate) use chunk_candidates::capsule_chunk_candidate_rows; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index f9e3bf451..3b38e2e0d 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -280,7 +280,7 @@ impl PointerCatalogLookup { )?; let hashes = requested(files, &mut work)?; let mut shards = requested(shards, &mut work)?; - let mut xorbs = requested(xorbs, &mut work)?; + let xorbs = requested(xorbs, &mut work)?; if hashes.is_empty() && shards.is_empty() && xorbs.is_empty() { return Ok(PointerCatalog::new()); } @@ -320,13 +320,43 @@ impl PointerCatalogLookup { catalog.insert_file(hash.clone(), entry)?; } } + let selected = self.selected_shards(store, &shards, &mut work).await?; + for (hash, entry) in selected.shards { + catalog.insert_shard(hash, entry)?; + } + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + catalog.validate(true, true)?; + Ok(catalog) + } + + /// Select requested shard roots and their complete authenticated xorb closures. + pub async fn shards(&self, store: &Store, hashes: &[String]) -> Result { + let mut work = self.work(); + let hashes = requested(hashes, &mut work)?; + self.selected_shards(store, &hashes, &mut work).await + } + + async fn selected_shards( + &self, + store: &Store, + shards: &BTreeSet, + work: &mut PageWork, + ) -> Result { + let mut catalog = PointerCatalog::new(); let root = self .checkpoint .as_ref() .and_then(|s| s.descriptor.shards.as_ref()); let rows = self - .rows(store, root, Kind::CatalogShards, &keys(&shards)?, &mut work) + .rows(store, root, Kind::CatalogShards, &keys(shards)?, work) .await?; + let mut xorbs = BTreeSet::new(); for (hash, row) in shards.iter().zip(rows) { for value in row { let row: ShardRow = from_json(&value.bytes)?; @@ -336,7 +366,7 @@ impl PointerCatalogLookup { &value.origin, row.xorbs.as_ref(), Kind::CatalogShardXorbs, - &mut work, + work, ) .await?; let closure = closure @@ -352,20 +382,12 @@ impl PointerCatalogLookup { verify_identity(&value, Kind::CatalogShards, &json(&entry)?)?; catalog.insert_shard(hash.clone(), entry)?; } + let Some(entry) = catalog.shards.get(hash) else { + continue; + }; + xorbs.extend(entry.xorb_hashes.iter().cloned()); } - for file in catalog.files.values() { - catalog - .shards - .get(&file.shard_hash) - .ok_or_else(|| corrupt("selected file references an absent shard"))?; - } - xorbs.extend( - catalog - .shards - .values() - .flat_map(|entry| entry.xorb_hashes.iter().cloned()), - ); - let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + let selected = self.selected_xorbs(store, &xorbs, work).await?; for (hash, entry) in selected.xorbs { catalog.insert_xorb(hash, entry)?; } diff --git a/crates/crab-metadata/src/key_codec.rs b/crates/crab-metadata/src/key_codec.rs index 19301a0a6..b0436ec5a 100644 --- a/crates/crab-metadata/src/key_codec.rs +++ b/crates/crab-metadata/src/key_codec.rs @@ -11,6 +11,8 @@ //! - `0x03 || hash[32]` — latest committed chunk-receipt candidate. //! - `0x04 || proof_id[32]` — immutable xorb origin proof. //! - `0x05 || anchor_id[32]` — immutable source-shard anchor. +//! - `0x06 || hash[32]` — point-readable capsule chunk candidate. +//! - `0x07 || anchor_id[32]` — immutable capsule source/origin anchor. //! - `0xFF || b"sys:" || name` — system key (format_version, epoch, //! created_at, gc_generation). //! @@ -22,6 +24,9 @@ use crab_xet::xorb::format::MerkleHash; use crate::error::{MetadataError, Result}; +/// Canonical metadata database format shared by all reader and writer adapters. +pub const METADB_FORMAT_VERSION: u32 = 1; + /// Prefix byte for content-addressed keys (file_hash or chunk_hash). pub const PREFIX_CONTENT: u8 = 0x01; @@ -41,6 +46,23 @@ pub const PREFIX_ORIGIN_PROOF: u8 = 0x04; /// Prefix byte for immutable repository-shard source anchors. pub const PREFIX_SOURCE_ANCHOR: u8 = 0x05; +/// Capsule candidates use distinct keys so released manifest readers cannot +/// mistake capsule provenance for a manifest-generation assertion. +const PREFIX_CAPSULE_CHUNK_HEAD: u8 = 0x06; +const PREFIX_CAPSULE_CHUNK_ANCHOR: u8 = 0x07; + +/// Encode a point-readable capsule candidate key. +#[must_use] +pub fn capsule_chunk_head_key(hash: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_HEAD, hash) +} + +/// Encode an immutable capsule provenance key. +#[must_use] +pub fn capsule_chunk_anchor_key(hash: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_ANCHOR, hash) +} + /// Prefix byte for the `sys:*` system-key namespace. Chosen as `0xFF` /// so it sorts after every content key and is trivial to filter out of /// content-only scans. diff --git a/crates/crab-metadata/src/remote_index.rs b/crates/crab-metadata/src/remote_index.rs index e33763806..4c94dc6a8 100644 --- a/crates/crab-metadata/src/remote_index.rs +++ b/crates/crab-metadata/src/remote_index.rs @@ -100,6 +100,73 @@ pub async fn write_index_entries( close_result } +/// Publish rebuildable capsule chunk hints using origin receipts captured during byte verification. +/// +/// Call after ref visibility. Every opened database is closed on success or error; +/// callers must await this future through completion. Readers still revalidate source +/// visibility and exact origin identity, so partial batches never confer authority. +pub async fn write_capsule_chunk_candidates( + store: Arc, + config: &RemoteIndexConfig, + repo_prefix: &str, + catalog: &crate::capsule_protocol::PointerCatalog, + origins: &HashMap, +) -> Result<()> { + let mut source_shards = HashMap::new(); + for (shard_hash, shard) in catalog.shards() { + for xorb in shard.xorb_hashes() { + source_shards + .entry(xorb.as_str()) + .or_insert(shard_hash.as_str()); + } + } + let mut selected = Vec::new(); + for (hash, entry) in catalog.xorbs().iter().rev() { + let parsed = MerkleHash::from_hex(hash).map_err(|_| MetadataError::CorruptObject { + path: CHUNK_INDEX_DB_LABEL.to_owned(), + reason: "invalid capsule xorb hash".to_owned(), + })?; + if let Some(origin) = origins.get(&parsed) { + selected.push((hash, parsed, entry, origin)); + } + } + if selected.is_empty() { + return Ok(()); + } + for (hash, _, _, _) in &selected { + if !source_shards.contains_key(hash.as_str()) { + return Err(MetadataError::CorruptObject { + path: CHUNK_INDEX_DB_LABEL.to_owned(), + reason: "capsule xorb candidate has no source shard root".to_owned(), + }); + } + } + let writer = RemoteIndexWriter::open(store, config, false, true).await?; + let result = async { + // Last-write heads choose the lowest canonical xorb and first occurrence. + for (hash, parsed, entry, origin) in selected { + let source_shard = source_shards.get(hash.as_str()).ok_or_else(|| { + MetadataError::Internal( + "admitted capsule candidate lost its source shard".to_owned(), + ) + })?; + let rows = crate::capsule_protocol::capsule_chunk_candidate_rows( + repo_prefix, + source_shard, + parsed, + entry, + origin, + )?; + writer.write_capsule_candidates(&rows).await?; + } + Ok::<_, MetadataError>(()) + } + .await; + let close = writer.close().await; + result?; + close +} + /// Long-lived writer for caller-bounded remote index batches. /// /// Writes buffer data without waiting for durability. Always await [`Self::close`] @@ -164,6 +231,33 @@ impl RemoteIndexWriter { .await } + /// Buffer a bounded set of capsule candidate rows; close remains the durability barrier. + async fn write_capsule_candidates(&self, rows: &[(Vec, Vec)]) -> Result<()> { + let db = self + .chunk_db + .as_ref() + .ok_or_else(|| MetadataError::Internal("chunk index was not opened".to_owned()))?; + for rows in rows.chunks(4096) { + let mut batch = slatedb::WriteBatch::new(); + for (key, value) in rows { + batch.put(key.as_slice(), value.as_slice()); + } + db.write_with_options( + batch, + &slatedb::config::WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .map_err(|source| MetadataError::SlateDbWrite { + db: CHUNK_INDEX_DB_LABEL.to_owned(), + source, + })?; + } + Ok(()) + } + /// Flush and close every database opened by this writer. /// /// Both closes are attempted. If both fail, return the file-index error. @@ -474,6 +568,34 @@ async fn open_writer( path: &str, db: &'static str, ) -> Result { + let fresh = match open_reader(Arc::clone(&store), path, db).await? { + None => true, + Some(reader) => { + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let validation = async { + let value = + reader + .get(&key) + .await + .map_err(|source| MetadataError::SlateDbRead { + db: db.to_owned(), + source, + })?; + validate_writer_format(value.as_deref(), db) + } + .await; + let close = reader + .close() + .await + .map_err(|source| MetadataError::SlateDbClose { + db: db.to_owned(), + source, + }); + validation?; + close?; + false + } + }; let l0_sst_size_bytes = usize::try_from(DEFAULT_CHUNK_INDEX_L0_SST_SIZE_BYTES) .map_err(|error| MetadataError::Internal(format!("L0 SST size unsupported: {error}")))?; let settings = slatedb::config::Settings { @@ -484,7 +606,7 @@ async fn open_writer( l0_sst_size_bytes, ..slatedb::config::Settings::default() }; - slatedb::Db::builder(ObjectPath::from(path), store) + let writer = slatedb::Db::builder(ObjectPath::from(path), store) .with_settings(settings) .build() .await @@ -492,7 +614,61 @@ async fn open_writer( db: db.to_owned(), path: path.to_owned(), source, - }) + })?; + if fresh { + let initialized = async { + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = writer + .get(&key) + .await + .map_err(|source| MetadataError::SlateDbRead { + db: db.to_owned(), + source, + })?; + if let Some(value) = value { + return validate_writer_format(Some(&value), db); + } + let mut batch = slatedb::WriteBatch::new(); + batch.put(key, crate::key_codec::METADB_FORMAT_VERSION.to_le_bytes()); + writer + .write_with_options( + batch, + &slatedb::config::WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .map(|_| ()) + .map_err(|source| MetadataError::SlateDbWrite { + db: db.to_owned(), + source, + }) + } + .await; + if let Err(error) = initialized { + let _ = close_writer(Some(writer), db).await; + return Err(error); + } + } + Ok(writer) +} + +fn validate_writer_format(value: Option<&[u8]>, db: &str) -> Result<()> { + if value + != Some( + crate::key_codec::METADB_FORMAT_VERSION + .to_le_bytes() + .as_slice(), + ) + { + return Err(MetadataError::CorruptObject { + path: db.to_owned(), + reason: "metadata database is missing or has an unsupported canonical format marker" + .to_owned(), + }); + } + Ok(()) } async fn open_reader( @@ -790,3 +966,6 @@ mod tests { assert_eq!(config.chunk_index_path, ".crab/chunk_index_db/"); } } + +#[cfg(test)] +mod capsule_tests; diff --git a/crates/crab-metadata/src/remote_index/capsule_tests.rs b/crates/crab-metadata/src/remote_index/capsule_tests.rs new file mode 100644 index 000000000..a6e396a85 --- /dev/null +++ b/crates/crab-metadata/src/remote_index/capsule_tests.rs @@ -0,0 +1,86 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use crate::capsule_protocol::{PointerCatalog, XorbCatalogEntry, XorbChunkEntry}; +use crate::receipts::OriginReceipt; +use crab_xet::hash::MerkleHash; +use object_store::memory::InMemory; + +use super::{RemoteIndexConfig, write_capsule_chunk_candidates}; + +#[tokio::test] +async fn capsule_candidates_reject_xorbs_without_a_source_shard_root() { + let hash = MerkleHash::from([3; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + hash.hex(), + XorbCatalogEntry::new( + 100, + "4".repeat(64), + vec![XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9)], + ), + ) + .unwrap(); + let origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", hash.hex()), + hash.into(), + [5; 32], + 100, + Some("etag".to_owned()), + None, + ); + let result = write_capsule_chunk_candidates( + Arc::new(InMemory::new()), + &RemoteIndexConfig::for_repo("repo"), + "repo", + &catalog, + &HashMap::from([(hash, origin)]), + ) + .await; + assert!(result.is_err(), "an xorb descriptor alone is not a GC root"); +} + +#[tokio::test] +async fn capsule_candidates_writer_initializes_canonical_format() { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let writer = super::RemoteIndexWriter::open(Arc::clone(&store), &config, false, true) + .await + .unwrap(); + writer.close().await.unwrap(); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = reader.get(&key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!(value.as_deref(), Some(1u32.to_le_bytes().as_slice())); +} + +#[tokio::test] +async fn capsule_candidates_writer_rejects_noncanonical_existing_databases() { + for marker in [None, Some(2u32.to_le_bytes().to_vec()), Some(vec![1])] { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let seed = slatedb::Db::open(config.chunk_index_path.clone(), Arc::clone(&store)) + .await + .unwrap(); + if let Some(marker) = marker { + seed.put( + crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION), + marker, + ) + .await + .unwrap(); + } + seed.close().await.unwrap(); + let result = super::RemoteIndexWriter::open(store, &config, false, true).await; + if let Ok(writer) = result { + writer.close().await.unwrap(); + panic!("invalid existing format was accepted"); + } + } +} From a5e267023025b0c966106e88617943e4d7f98dcf Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:36:12 -0700 Subject: [PATCH 080/127] fix(push): bind capsule reuse to staged proofs --- crab/docs/design/add.md | 20 + crab/src/git/capsule_push.rs | 1 + crab/src/git/push.rs | 397 +++++++++------ .../src/git/push/tests/capsule_chunk_tests.rs | 480 ++++++++++++++++++ crab/src/git/xet_publication.rs | 101 +++- .../src/metadata/metadb/stores/chunk_index.rs | 43 +- .../src/capsule_protocol/chunk_candidates.rs | 17 +- .../chunk_candidates/tests.rs | 2 +- .../crab-metadata/src/capsule_protocol/mod.rs | 5 +- .../src/capsule_protocol/pointer.rs | 1 + .../src/capsule_protocol/pointer/paged.rs | 3 +- .../crab-metadata/src/capsule_protocol/run.rs | 4 + .../src/capsule_protocol/visibility/paged.rs | 3 + crates/crab-metadata/src/key_codec.rs | 8 + .../src/remote_index/capsule_tests.rs | 79 +++ 15 files changed, 973 insertions(+), 191 deletions(-) create mode 100644 crab/src/git/push/tests/capsule_chunk_tests.rs diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index 94426f02c..16fbe9e6e 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -212,6 +212,26 @@ source repository's current paged catalog, check each exact chunk position and size, then verify canonical origin identity. An absent source or changed physical descriptor cannot turn an index hint into authority. +Each capsule placement is also stored under its immutable placement digest. +Staged recipes retain that digest and the origin proof ID, so replacing a shared +chunk head does not erase an add's chosen assertion. Capsule push re-reads those +exact records, revalidates the source shard, and requires the recorded physical +origin identity before adopting remote-only chunks. Warm payload/index caches +cannot substitute for that identity check. The proof session is read-only and +closed before publication opens the shared index writer. + +The revalidator returns caller-owned source descriptors and original receipts +with the accepted candidates. Capsule preparation adopts those descriptors +without a full xorb GET or decompression. Retained metadata shares one catalog +byte allowance across source repositories; conflicting physical assertions fail +explicitly. Released manifest receipts still use full-body verification when +constructing a capsule descriptor that their older record does not contain. + +Canonical origin receipts belong to the trusted `global_prefix` of the same +authorization scope as the repository. Managed and ACL-scoped views use their +own supplied global root. Local staging and advisory caches only name candidates; +they cannot issue canonical receipts or certify an origin identity. + Direct publication and protected receive use the same bounded candidate encoder/writer. Protected promotion carries the canonical operation identity; client staging-prefix writes do not confer canonical origin proof. Every index diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index e0fbb8cc6..fc58d03c3 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -418,6 +418,7 @@ async fn run_inner( staging, caching_store, metrics, + &config.metadb, config.upload_concurrency, config.protected_push.is_none(), cancel, diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index 571615cfa..5e46f30aa 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -9970,7 +9970,7 @@ impl PushPipeline { &self, refs: &HashMap, ) -> Result> { - self.verify_xorb_refs_with_committed_receipts(refs, &HashMap::new()) + self.verify_xorb_refs_with_committed_receipts(refs, &HashMap::new(), false) .await } @@ -9978,6 +9978,7 @@ impl PushPipeline { &self, refs: &HashMap, committed_receipts: &HashMap, + require_committed_receipt: bool, ) -> Result> { use futures_util::StreamExt; @@ -10007,7 +10008,7 @@ impl PushPipeline { let cache = self.remote_full_xorb_ref_cache.lock().await; refs.values() .map(|xorb_ref| xorb_ref.xorb_hash) - .filter(|xorb_hash| cache.contains(xorb_hash)) + .filter(|xorb_hash| !require_committed_receipt && cache.contains(xorb_hash)) .collect::>() }; { @@ -10018,7 +10019,8 @@ impl PushPipeline { .entry(xorb_ref.xorb_hash) .or_default() .push((*chunk_hash, *xorb_ref)); - } else if chunk_ref_cache.contains(&RemoteChunkRefKey::new(*chunk_hash, *xorb_ref)) + } else if !require_committed_receipt + && chunk_ref_cache.contains(&RemoteChunkRefKey::new(*chunk_hash, *xorb_ref)) { verified.insert(*chunk_hash, *xorb_ref); cached_xorb_hashes.insert(xorb_ref.xorb_hash); @@ -10057,7 +10059,7 @@ impl PushPipeline { { let index_cache = self.remote_xorb_index_cache.lock().await; for xorb_hash in refs_by_xorb.keys().copied() { - if let Some(index) = index_cache.get(&xorb_hash) { + if !require_committed_receipt && let Some(index) = index_cache.get(&xorb_hash) { verified_xorbs.insert(xorb_hash, index.clone()); } else { planned_hashes.push(xorb_hash); @@ -10185,6 +10187,14 @@ impl PushPipeline { return None; } Some(async move { + // Capsule assertions bind the exact origin selected at add. + // Cached payload validity cannot certify a replaced object. + if require_committed_receipt && !committed_receipt.as_ref().zip(object_meta.as_ref()) + .is_some_and(|(receipt, meta)| committed_origin_receipt_matches_remote( + receipt, &xorb_hash, &path, &index, meta, + )) { + return Ok((Vec::new(), None, None, Vec::new())); + } if is_full_cached { let xorb_info = build_xorb_info_from_remote_index(&index)?; return Ok((valid_refs, Some((xorb_hash, xorb_info)), None, Vec::new())); @@ -10361,19 +10371,19 @@ impl PushPipeline { async fn revalidate_add_plan_existing_candidates( &self, planned: &HashMap, - ) -> Result> { + ) -> Result { if planned.is_empty() { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); } let chunk_store = { let guard = self.metadb.lock().await; let Some(guard) = guard.as_ref() else { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); }; match guard.chunk_index().await { Ok(store) => store, Err(error) if error.is_metadb_read_only_uninitialized() => { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); } Err(error) => return Err(error), } @@ -10382,8 +10392,47 @@ impl PushPipeline { .iter() .map(|(chunk_hash, candidate)| (*chunk_hash, candidate.placement_id)) .collect::>(); - let remote_candidates = tokio::time::timeout( - GLOBAL_CHUNK_LOOKUP_BUDGET, + let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; + let (capsule, capsule_indexed, capsule_matched) = + tokio::time::timeout_at(deadline, async { + let mut candidates = chunk_store + .capsule_candidates_by_id(&exact_candidates) + .await?; + let indexed = candidates.placements.len(); + candidates.placements.retain(|hash, placement| { + planned + .get(&MerkleHash::from(*hash)) + .is_some_and(|candidate| { + candidate.placement_id == placement.placement_id() + && candidate.origin_proof_id == placement.origin_proof_id + && candidate.xorb_ref + == XorbRef { + xorb_hash: MerkleHash::from(placement.xorb_hash), + chunk_index: placement.chunk_index, + uncompressed_size: placement.uncompressed_size, + } + }) + }); + let matched = candidates.placements.len(); + let verified = self.verify_capsule_chunk_candidates(candidates).await?; + Ok::<_, CrabError>((verified, indexed, matched)) + }) + .await + .map_err(|_| { + CrabError::NetworkTransient(object_store::Error::Generic { + store: "chunk_index_db", + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "staged capsule-proof revalidation budget exhausted", + )), + }) + })??; + let exact_candidates = exact_candidates + .into_iter() + .filter(|(hash, _)| !capsule.candidates.contains_key(hash)) + .collect::>(); + let remote_candidates = tokio::time::timeout_at( + deadline, chunk_store.get_committed_candidates_by_id_batch(&exact_candidates), ) .await @@ -10432,26 +10481,30 @@ impl PushPipeline { let committed_receipts = self.validate_committed_chunk_receipts(&refs).await; let validated = self.verified_committed_chunk_receipts.lock().await.clone(); let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &committed_receipts) + .verify_xorb_refs_with_committed_receipts(&refs, &committed_receipts, false) .await?; debug!( planned = planned.len(), - indexed = indexed_candidates, - matched = refs.len(), + indexed = indexed_candidates + capsule_indexed, + matched = refs.len() + capsule_matched, generation_verified = validated.len(), - payload_verified = verified.len(), + capsule_source_verified = capsule.candidates.len(), + payload_verified = verified.len() + capsule.candidates.len(), placement_mismatches, origin_proof_mismatches, "revalidated staged remote placement proofs" ); - Ok(verified - .into_iter() - .filter(|(chunk_hash, _)| { - placement_ids - .get(chunk_hash) - .is_some_and(|placement_id| validated.get(chunk_hash) == Some(placement_id)) - }) - .collect()) + let mut output = capsule; + output + .candidates + .extend(verified.into_iter().filter_map(|(hash, _)| { + let placement = placement_ids.get(&hash)?; + (validated.get(&hash) == Some(placement)) + .then(|| planned.get(&hash).copied()) + .flatten() + .map(|candidate| (hash, candidate)) + })); + Ok(output) } async fn prove_all_origin_xorbs_for_publish(&self) -> Result<()> { @@ -10960,7 +11013,7 @@ impl PushPipeline { // executor stack. This path owns several repository-sized maps; // heap-pinning the future avoids inflating the default test stack. match Box::pin(self.revalidate_add_plan_existing_candidates(&candidate_refs)).await { - Ok(refs) => refs, + Ok(verified) => verified.into_refs(), Err(CrabError::Cancelled) => return Err(CrabError::Cancelled), Err(error) => return Err(error), } @@ -15021,8 +15074,17 @@ impl PushPipeline { chunk_store: &crate::metadata::ChunkIndexStore, ) -> Result> { let candidates = chunk_store.capsule_candidates(hashes).await?; + self.verify_capsule_chunk_candidates(candidates) + .await + .map(|verified| verified.candidates) + } + + async fn verify_capsule_chunk_candidates( + &self, + candidates: crab_metadata::capsule_protocol::CapsuleChunkCandidates, + ) -> Result { let Some(store) = &self.store else { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); }; #[derive(Default)] struct SourceCandidates<'a> { @@ -15040,6 +15102,14 @@ impl PushPipeline { let mut refs = HashMap::new(); let mut origins = HashMap::new(); let mut proofs = HashMap::new(); + let mut bindings: HashMap< + MerkleHash, + ( + crab_metadata::capsule_protocol::XorbCatalogEntry, + crab_metadata::receipts::OriginReceipt, + ), + > = HashMap::new(); + let mut retained_bytes = 0_u64; for (repo, source) in by_repo { check_cancelled(&self.cancel)?; let layout = crab_storage::StoreLayout::with_global_prefix( @@ -15078,6 +15148,49 @@ impl PushPipeline { { continue; } + let entry = catalog.xorbs().get(&xorb_hash.hex()).ok_or_else(|| { + CrabError::Internal( + "validated capsule placement lost its descriptor".to_owned(), + ) + })?; + if let Some((prior, origin)) = bindings.get(&xorb_hash) { + // Observation time may differ; physical assertions for the + // same immutable xorb cannot disagree or certify one another. + if prior != entry + || origin.etag != anchor.origin.etag + || origin.object_version != anchor.origin.object_version + || origin.payload_digest != anchor.origin.payload_digest + || origin.size != anchor.origin.size + { + return Err(CrabError::CorruptObject { + path: path.to_string(), + reason: "conflicting canonical xorb assertions".to_owned(), + }); + } + } else { + let metadata_bytes = std::mem::size_of_val(entry.chunks()) as u64 + + entry + .chunks() + .iter() + .map(|chunk| chunk.hash().len() as u64) + .sum::() + + entry.body_digest().len() as u64 + + serde_json::to_vec(&anchor.origin) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))? + .len() as u64; + retained_bytes = + retained_bytes.checked_add(metadata_bytes).ok_or_else(|| { + CrabError::Internal("selected xorb metadata size overflows".to_owned()) + })?; + if retained_bytes > crab_metadata::capsule_protocol::MAX_CATALOG_BYTES { + return Err(CrabError::CorruptObject { + path: "capsule chunk candidates".to_owned(), + reason: "selected xorb metadata exceeds the catalog byte limit" + .to_owned(), + }); + } + bindings.insert(xorb_hash, (entry.clone(), anchor.origin.clone())); + } let hash = MerkleHash::from(placement.chunk_hash); refs.insert( hash, @@ -15091,12 +15204,18 @@ impl PushPipeline { proofs.insert(hash, (placement.placement_id(), placement.origin_proof_id)); } } - // Source catalogs establish commitment; the existing origin verifier - // binds payload/index checks to the exact object version in the receipt. + // Source catalogs establish commitment. Receipts belong to the trusted + // scope's canonical global index; fresh provider identity binds their + // verified bytes, while cache or staged IDs alone cannot certify reuse. let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &origins) + .verify_xorb_refs_with_committed_receipts(&refs, &origins, true) .await?; - Ok(verified + let accepted_xorbs = verified + .values() + .map(|reference| reference.xorb_hash) + .collect::>(); + bindings.retain(|hash, _| accepted_xorbs.contains(hash)); + let candidates = verified .into_iter() .filter_map(|(hash, xorb_ref)| { let (placement_id, origin_proof_id) = proofs.get(&hash).copied()?; @@ -15109,7 +15228,11 @@ impl PushPipeline { }, )) }) - .collect()) + .collect(); + Ok(VerifiedRemoteCandidates { + candidates, + xorbs: bindings, + }) } async fn lookup_proven_remote_chunks_for_add( @@ -15177,7 +15300,7 @@ impl PushPipeline { .validate_committed_chunk_candidates(&refs, candidates) .await; let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins) + .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins, false) .await?; capsule.extend(verified.into_iter().filter_map(|(chunk_hash, xorb_ref)| { let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; @@ -15285,7 +15408,7 @@ impl PushPipeline { let committed_receipts = Box::pin(self.validate_committed_chunk_receipts(&hits)).await; let refs = self - .verify_xorb_refs_with_committed_receipts(&hits, &committed_receipts) + .verify_xorb_refs_with_committed_receipts(&hits, &committed_receipts, false) .await?; let stale_hashes = hits .keys() @@ -17057,6 +17180,77 @@ impl PushPipeline { }) } } +#[derive(Debug, Default)] +pub(crate) struct VerifiedRemoteCandidates { + pub candidates: HashMap, + pub xorbs: HashMap< + MerkleHash, + ( + crab_metadata::capsule_protocol::XorbCatalogEntry, + crab_metadata::receipts::OriginReceipt, + ), + >, +} + +impl VerifiedRemoteCandidates { + fn into_refs(self) -> HashMap { + self.candidates + .into_iter() + .map(|(hash, candidate)| (hash, candidate.xorb_ref)) + .collect() + } +} + +// Capsule preparation owns this read-only session through revalidation and close. +// Reusing the planner's verifier keeps released manifest and capsule proofs on +// the same origin checks without opening a writer during payload preparation. +pub(crate) async fn revalidate_staged_remote_chunks( + metadb: &crate::core::config::MetaDbTomlConfig, + layout: &crab_storage::StoreLayout, + planned: &HashMap, + cancel: &CancellationToken, +) -> Result { + if planned.is_empty() { + return Ok(VerifiedRemoteCandidates::default()); + } + check_cancelled(cancel)?; + let store = Store::from_storage(layout.store().clone()); + let router = StoreLayout::with_global_prefix( + store.clone(), + layout.repo_prefix().to_owned(), + layout.global_prefix().to_owned(), + ); + let config = PushConfig { + metadb: metadb.clone(), + ..Default::default() + }; + let pipeline = PushPipeline::new( + config, + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + cancel.clone(), + None, + ); + pipeline.install_metadb(build_push_metadb_guard_with_object_store( + &store, None, &router, None, metadb, true, + )); + let result = Box::pin(pipeline.revalidate_add_plan_existing_candidates(planned)).await; + let guard = { pipeline.metadb.lock().await.take() }; + let closed = match guard { + Some(guard) => guard.close().await, + None => Ok(()), + }; + let verified = result?; + closed?; + check_cancelled(cancel)?; + Ok(verified) +} + /// Run the push pipeline for a batch of push specs. /// /// This is the main entry point called by the remote helper when Git @@ -20705,6 +20899,7 @@ mod tests { struct RangeCountingStore { inner: object_store::memory::InMemory, range_gets: Arc, + full_xorb_gets: AtomicUsize, head_e_tag: Option>>, } @@ -20713,6 +20908,7 @@ mod tests { Self { inner: object_store::memory::InMemory::new(), range_gets, + full_xorb_gets: AtomicUsize::new(0), head_e_tag: None, } } @@ -20724,6 +20920,7 @@ mod tests { Self { inner: object_store::memory::InMemory::new(), range_gets, + full_xorb_gets: AtomicUsize::new(0), head_e_tag: Some(head_e_tag), } } @@ -20760,6 +20957,9 @@ mod tests { options: GetOptions, ) -> object_store::Result { let is_head = options.head; + if !is_head && options.range.is_none() && location.as_ref().contains("/xorbs/") { + self.full_xorb_gets.fetch_add(1, Ordering::Relaxed); + } if options.range.is_some() { self.range_gets.fetch_add(1, Ordering::Relaxed); } @@ -22162,134 +22362,7 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { - use crab_staging::push_plan::ExistingChunkLookup; - use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; - - let _git_env = CleanGitEnvGuard::new(); - let source = tempfile::tempdir().unwrap(); - let git = |args: &[&str]| { - let output = Command::new("git") - .args(args) - .current_dir(source.path()) - .env("GIT_AUTHOR_NAME", "Test") - .env("GIT_AUTHOR_EMAIL", "test@example.invalid") - .env("GIT_COMMITTER_NAME", "Test") - .env("GIT_COMMITTER_EMAIL", "test@example.invalid") - .output() - .unwrap(); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - }; - git(&["init", "-b", "main"]); - let staging_root = source.path().join("staging"); - let staging = crab_staging::StagingArea::open(staging_root.clone()) - .await - .unwrap(); - let staged = stage_reader_streaming( - &mut std::io::Cursor::new(b"shared capsule payload"), - std::path::Path::new("model.bin"), - &staging, - StreamStageProgress::default(), - &CancellationToken::new(), - ) - .await - .unwrap(); - let chunks = staging - .recipe_page(&staged.recipe, 0) - .unwrap() - .chunks - .into_iter() - .map(|chunk| (chunk.chunk_hash, chunk.len)) - .collect::>(); - staging - .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) - .unwrap(); - let pointer = crab_types::pointer::Pointer { - file_hash: staged.file_hash, - size: staged.size, - shard_hint: None, - }; - std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); - git(&["add", "model.bin"]); - git(&["commit", "-m", "shared payload"]); - staging.close().await.unwrap(); - let staging = Arc::new( - crab_staging::StagingAreaReadOnly::open(staging_root) - .await - .unwrap(), - ); - let (store, source_router) = test_store_router("repos/source"); - let layout = - crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); - crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") - .await - .unwrap(); - let config = PushConfig { - git_dir: Some(source.path().join(".git")), - ..Default::default() - }; - let (result, _) = super::super::capsule_push::run( - &config, - &[crate::git::remote_helper::PushSpec { - force: false, - src: "refs/heads/main".to_owned(), - dst: "refs/heads/main".to_owned(), - }], - &store, - &source_router, - None, - &[], - Some(&staging), - None, - None, - &CancellationToken::new(), - ) - .await - .unwrap(); - assert!(result.all_ok(), "source publication failed: {result:?}"); - drop(staging); - - let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); - let pipeline = PushPipeline::new( - PushConfig::default(), - Vec::new(), - Some(store.clone()), - None, - None, - router.repo_prefix().to_owned(), - router.clone(), - None, - CancellationToken::new(), - None, - ); - let cache_root = tempfile::tempdir().unwrap(); - let mut metadata = crate::core::config::MetaDbTomlConfig::default(); - metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); - pipeline.install_metadb(build_push_metadb_guard( - &store, &router, None, &metadata, true, - )); - // No advisory caches or source-repository state can manufacture a hit. - // This exercises the real publisher-to-classifier authority boundary. - let classifier = AddRemoteChunkClassifier { - pipeline, - remote_chunk_index: tokio::sync::OnceCell::new(), - candidate_cache: None, - candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), - candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), - }; - let candidates = classifier.lookup_existing_candidates(&chunks).await; - classifier.close().await; - let candidates = candidates.unwrap(); - assert!( - !candidates.is_empty() && candidates.iter().all(Option::is_some), - "committed source chunks must be reusable from a cold consumer: {candidates:?}" - ); - } + mod capsule_chunk_tests; fn add_classifier_with_private_cache( store: &Store, @@ -30830,7 +30903,11 @@ mod tests { range_gets.store(0, Ordering::Relaxed); let verified = pipeline - .verify_xorb_refs_with_committed_receipts(&refs, &HashMap::from([(xorb_hash, receipt)])) + .verify_xorb_refs_with_committed_receipts( + &refs, + &HashMap::from([(xorb_hash, receipt)]), + false, + ) .await .expect("verify refs from committed origin receipt"); diff --git a/crab/src/git/push/tests/capsule_chunk_tests.rs b/crab/src/git/push/tests/capsule_chunk_tests.rs new file mode 100644 index 000000000..0f60b530e --- /dev/null +++ b/crab/src/git/push/tests/capsule_chunk_tests.rs @@ -0,0 +1,480 @@ +use super::*; + +struct FirstCandidateOnly(Arc); + +#[async_trait::async_trait] +impl crab_staging::push_plan::ExistingChunkLookup for FirstCandidateOnly { + async fn lookup_existing_candidates( + &self, + chunks: &[(MerkleHash, u64)], + ) -> crab_staging::Result>> { + let candidates = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( + self.0.as_ref(), + chunks, + ) + .await?; + Ok(candidates + .into_iter() + .enumerate() + .map( + |(index, candidate)| { + if index == 0 { candidate } else { None } + }, + ) + .collect()) + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { + use crab_staging::push_plan::ExistingChunkLookup; + use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; + + let _git_env = CleanGitEnvGuard::new(); + let content = test_deterministic_bytes(0x4cab, 2 * 1024 * 1024); + let source = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .args(args) + .current_dir(source.path()) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + git(&["init", "-b", "main"]); + let staging_root = source.path().join("staging"); + let staging = crab_staging::StagingArea::open(staging_root.clone()) + .await + .unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let chunks = staging + .recipe_page(&staged.recipe, 0) + .unwrap() + .chunks + .into_iter() + .map(|chunk| (chunk.chunk_hash, chunk.len)) + .collect::>(); + staging + .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + let pointer = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); + git(&["add", "model.bin"]); + git(&["commit", "-m", "shared payload"]); + staging.close().await.unwrap(); + let staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(staging_root) + .await + .unwrap(), + ); + let reads = Arc::new(RangeCountingStore::new(Arc::new(AtomicUsize::new(0)))); + let store = Store::new(reads.clone()); + let source_router = StoreLayout::new(store.clone(), "repos/source".to_owned()); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..Default::default() + }; + let (result, _) = crate::git::capsule_push::run( + &config, + &[crate::git::remote_helper::PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &source_router, + None, + &[], + Some(&staging), + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(result.all_ok(), "source publication failed: {result:?}"); + drop(staging); + + let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); + let pipeline = PushPipeline::new( + PushConfig::default(), + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + CancellationToken::new(), + None, + ); + let cache_root = tempfile::tempdir().unwrap(); + let mut metadata = crate::core::config::MetaDbTomlConfig::default(); + metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard( + &store, &router, None, &metadata, true, + )); + // No advisory caches or source-repository state can manufacture a hit. + // This exercises the real publisher-to-classifier authority boundary. + let classifier = Arc::new(AddRemoteChunkClassifier { + pipeline, + remote_chunk_index: tokio::sync::OnceCell::new(), + candidate_cache: None, + candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), + candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), + }); + let candidates = classifier.lookup_existing_candidates(&chunks).await; + let consumer_staging_root = cache_root.path().join("consumer-staging"); + let consumer_staging = crab_staging::StagingArea::open(consumer_staging_root.clone()) + .await + .unwrap(); + let remote_staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &consumer_staging, + StreamStageProgress { + existing_lookup: Some(classifier.clone()), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await; + let partial_root = cache_root.path().join("partial-staging"); + let partial_staging = crab_staging::StagingArea::open(partial_root.clone()) + .await + .unwrap(); + let partial_staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &partial_staging, + StreamStageProgress { + existing_lookup: Some(Arc::new(FirstCandidateOnly(classifier.clone()))), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await; + classifier.close().await; + let partial_staged = partial_staged.unwrap(); + partial_staging + .delegate_publication(&[(&partial_staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + partial_staging.close().await.unwrap(); + let remote_staged = remote_staged.unwrap(); + consumer_staging + .delegate_publication(&[(&remote_staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + consumer_staging.close().await.unwrap(); + let candidates = candidates.unwrap(); + assert!( + !candidates.is_empty() && candidates.iter().all(Option::is_some), + "committed source chunks must be reusable from a cold consumer: {candidates:?}" + ); + + let planned = chunks + .iter() + .zip(&candidates) + .filter_map(|((hash, _), candidate)| candidate.map(|candidate| (*hash, candidate))) + .collect::>(); + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + let catalog = crab_metadata::capsule_protocol::load_pointer_catalog_from_root(&layout, &root) + .await + .unwrap(); + let mut origins = HashMap::new(); + for candidate in planned.values() { + let path = layout.xorb_path(&candidate.xorb_ref.xorb_hash); + let (bytes, identity) = layout.store().get_with_etag(&path).await.unwrap(); + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).unwrap(); + let mut origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + candidate.xorb_ref.xorb_hash.into(), + parser.payload_digest(), + bytes.len() as u64, + identity.e_tag, + identity.version, + ); + // Change the head's proof identity while retaining the actual committed source. + origin.proven_at_unix_secs += 10; + origins.insert(candidate.xorb_ref.xorb_hash, origin); + } + crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), + "repos/source", + &catalog, + &origins, + ) + .await + .unwrap(); + let consumer_layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/consumer".to_owned()); + let verified = revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + verified.candidates.len(), + planned.len(), + "old exact proof survives a newer shared head" + ); + + for field in ["placement", "origin", "position"] { + let mut forged = planned.clone(); + for candidate in forged.values_mut() { + match field { + "placement" => candidate.placement_id = [9; 32], + "origin" => candidate.origin_proof_id = [9; 32], + "position" => candidate.xorb_ref.chunk_index += 1, + _ => unreachable!(), + } + } + assert!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &forged, + &CancellationToken::new() + ) + .await + .unwrap() + .candidates + .is_empty(), + "accepted forged {field}" + ); + } + + let consumer_base = crab_write::capsule_protocol::initialize( + &consumer_layout, + &"2".repeat(64), + "refs/heads/main", + ) + .await + .unwrap(); + let consumer_staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(consumer_staging_root) + .await + .unwrap(), + ); + reads.full_xorb_gets.store(0, Ordering::Relaxed); + let (delta, stats) = crate::git::xet_publication::prepare_delta( + &consumer_layout, + &consumer_base, + &[crab_types::pointer::Pointer { + file_hash: remote_staged.file_hash, + size: remote_staged.size, + shard_hint: None, + }], + Some(&consumer_staging), + None, + None, + &metadata, + 1, + true, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + (delta.files().len(), stats.xorbs_uploaded), + (1, 0), + "remote-only staging must publish without repacking" + ); + assert_eq!( + reads.full_xorb_gets.load(Ordering::Relaxed), + 0, + "an exact, current source/origin proof must not download the reused xorb body" + ); + drop(consumer_staging); + let partial_staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(partial_root) + .await + .unwrap(), + ); + reads.full_xorb_gets.store(0, Ordering::Relaxed); + let (_, partial_stats) = crate::git::xet_publication::prepare_delta( + &consumer_layout, + &consumer_base, + &[crab_types::pointer::Pointer { + file_hash: partial_staged.file_hash, + size: partial_staged.size, + shard_hint: None, + }], + Some(&partial_staging), + None, + None, + &metadata, + 1, + true, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + ( + partial_stats.xorbs_uploaded, + reads.full_xorb_gets.load(Ordering::Relaxed) + ), + (0, 0), + "partial add-time hints must reuse the authenticated requested closure without packing or downloading" + ); + drop(partial_staging); + + let source_key = layout.capsule_root_path(); + let (source_bytes, _) = layout.store().get_with_etag(&source_key).await.unwrap(); + layout.store().delete(&source_key).await.unwrap(); + let unrooted = revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new(), + ) + .await + .unwrap(); + layout.store().put(&source_key, source_bytes).await.unwrap(); + assert!( + unrooted.candidates.is_empty(), + "physical bytes alone cannot replace a source root" + ); + + for candidate in planned.values() { + let key = layout.xorb_path(&candidate.xorb_ref.xorb_hash); + let (bytes, previous) = layout.store().get_with_etag(&key).await.unwrap(); + // Bypass the immutable adapter only to inject a provider-side replacement. + let replaced = layout + .store() + .inner() + .put(&key, bytes.into()) + .await + .unwrap(); + assert_ne!( + (previous.e_tag, previous.version), + (replaced.e_tag, replaced.version), + "the fault must actually change the provider identity" + ); + } + assert!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new() + ) + .await + .unwrap() + .candidates + .is_empty(), + "a replacement identity cannot reuse the original proof" + ); + let verified_refs = verified.into_refs(); + let cached = classifier + .pipeline + .verify_xorb_refs_with_committed_receipts(&verified_refs, &origins, true) + .await + .unwrap(); + assert!( + cached.is_empty(), + "warm proofs cannot certify the replacement identity" + ); + + // A fresh attestation for one occurrence must not certify another + // occurrence's stale provider identity in the same immutable xorb. + let mut fresh_origins = HashMap::new(); + for hash in planned + .values() + .map(|candidate| candidate.xorb_ref.xorb_hash) + .collect::>() + { + let key = layout.xorb_path(&hash); + let (bytes, identity) = layout.store().get_with_etag(&key).await.unwrap(); + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).unwrap(); + parser.verify_payload_digest().unwrap(); + parser.verify_all_chunks().unwrap(); + fresh_origins.insert( + hash, + crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + key.to_string(), + hash.into(), + parser.payload_digest(), + bytes.len() as u64, + identity.e_tag, + identity.version, + ), + ); + } + crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), + "repos/source", + &catalog, + &fresh_origins, + ) + .await + .unwrap(); + let guard = build_push_metadb_guard(&store, &router, None, &metadata, true); + let chunk_store = guard.chunk_index().await.unwrap(); + let hashes = planned.keys().copied().collect::>(); + let fresh = classifier + .pipeline + .lookup_capsule_chunks_for_add(&hashes, &chunk_store) + .await; + guard.close().await.unwrap(); + let fresh = fresh.unwrap(); + assert_eq!( + fresh.len(), + planned.len(), + "replacement proof fixture must be independently valid" + ); + assert!( + fresh.len() > 1, + "mixed-origin coverage needs distinct chunks in one xorb" + ); + let mut mixed = planned.clone(); + let (hash, candidate) = fresh.into_iter().next().unwrap(); + mixed.insert(hash, candidate); + assert!( + matches!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &mixed, + &CancellationToken::new(), + ) + .await, + Err(CrabError::CorruptObject { .. }) + ), + "conflicting immutable-xorb identities must not certify one another" + ); +} diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index b9b2c57d1..266fa1d6f 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -42,6 +42,7 @@ pub(crate) async fn prepare_delta( staging: Option<&Arc>, caching_store: Option<&crab_cache_store::CachingStore>, metrics: Option<&Metrics>, + metadb: &crate::core::config::MetaDbTomlConfig, upload_concurrency: usize, publish_gc_roots: bool, cancel: &CancellationToken, @@ -66,7 +67,8 @@ pub(crate) async fn prepare_delta( }, ) .await?; - let lookup = view.pointer_catalog_lookup(layout, 2 * 1024 * 1024 * 1024)?; + let lookup = + view.pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; drop(view); let mut pointer_sizes = BTreeMap::::new(); @@ -139,7 +141,7 @@ pub(crate) async fn prepare_delta( let mut prepared_files = Vec::with_capacity(unresolved.len()); let mut prepared_candidates = Vec::::new(); let mut candidate_indices = HashMap::::new(); - let mut external_candidates = HashMap::>::new(); + let mut staged_candidates = HashMap::new(); for (file_hash, file_size) in unresolved { check_cancelled(cancel)?; @@ -208,11 +210,8 @@ pub(crate) async fn prepare_delta( expected_size ))); } - let refs = external_candidates - .entry(candidate.xorb_ref.xorb_hash) - .or_default(); - match refs.insert(chunk_hash, candidate.xorb_ref) { - Some(existing) if existing != candidate.xorb_ref => { + match staged_candidates.insert(chunk_hash, candidate) { + Some(existing) if existing != candidate => { return Err(CrabError::StagingCorrupt(format!( "remote xorb authority disagrees for chunk {}", chunk_hash.hex() @@ -265,15 +264,78 @@ pub(crate) async fn prepare_delta( drop(base_chunks); drop(needed_chunks); - let mut external_candidates = external_candidates - .into_iter() - .filter(|(hash, refs)| { - !xorb_entries.contains_key(hash) - && refs - .keys() - .any(|chunk_hash| !placements.contains_key(chunk_hash)) - }) - .collect::>(); + // The recipe retains the exact add-time IDs. Re-read their immutable + // records and current source roots before trusting any remote-only bytes. + staged_candidates.retain(|hash, _| !placements.contains_key(hash)); + let planned_existing = staged_candidates.len(); + let verified_staged = + super::push::revalidate_staged_remote_chunks(metadb, layout, &staged_candidates, cancel) + .await?; + tracing::debug!( + planned_existing, + stale_existing = planned_existing.saturating_sub(verified_staged.candidates.len()), + global_existing = 0, + "capsule staged remote proof adoption complete", + ); + let mut reused_xorbs = verified_staged.xorbs.len(); + let requested = requested_chunks + .iter() + .map(String::as_str) + .collect::>(); + for (hash, (entry, origin)) in verified_staged.xorbs { + if xorb_entries + .get(&hash) + .is_some_and(|existing| existing != &entry) + { + return Err(CrabError::CorruptObject { + path: layout.xorb_path(&hash).to_string(), + reason: "verified xorb differs from the captured destination catalog".to_owned(), + }); + } + // The authenticated closure also proves requested chunks whose rebuildable + // heads were absent during add. Expand only outgoing occurrences, so an + // incomplete global index cannot force repacking the same verified bytes. + for (index, chunk) in entry.chunks().iter().enumerate() { + if requested.contains(chunk.hash()) { + let chunk_hash = parse_merkle_hash(chunk.hash(), "verified catalog chunk")?; + placements.entry(chunk_hash).or_insert(ChunkPlacement { + chunk_hash, + xorb_hash: hash, + chunk_index: u32::try_from(index).map_err(|_| { + CrabError::Internal("verified xorb has too many chunks".to_owned()) + })?, + uncompressed_size: chunk.uncompressed_size(), + }); + } + } + xorb_entries.insert(hash, entry); + origins.insert(hash, origin); + } + drop(requested); + let mut external_candidates = HashMap::>::new(); + for (hash, candidate) in verified_staged.candidates { + let xorb_ref = candidate.xorb_ref; + if xorb_entries.contains_key(&xorb_ref.xorb_hash) { + // Exact source membership and current provider identity already + // certify this descriptor. Re-reading payload would download a + // whole reused file despite the committed origin receipt. + placements.entry(hash).or_insert(ChunkPlacement { + chunk_hash: hash, + xorb_hash: xorb_ref.xorb_hash, + chunk_index: xorb_ref.chunk_index, + uncompressed_size: xorb_ref.uncompressed_size, + }); + } else { + // Manifest receipts shipped in v1.2.4 lack capsule body descriptors. + // Full-body verification supplies that value when importing them + // into capsule publication. + external_candidates + .entry(xorb_ref.xorb_hash) + .or_default() + .insert(hash, xorb_ref); + } + } + let mut external_candidates = external_candidates.into_iter().collect::>(); external_candidates.sort_by_key(|(hash, _)| hash.hex()); let external_reads = futures_util::stream::iter(external_candidates.into_iter().enumerate().map( @@ -303,7 +365,6 @@ pub(crate) async fn prepare_delta( verified_external.push(result?); } verified_external.sort_by_key(|(ordinal, _)| *ordinal); - let mut reused_xorbs = 0_usize; for (_, verified) in verified_external { let Some((xorb_hash, entry, xorb_placements, origin)) = verified else { continue; @@ -1237,6 +1298,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1316,6 +1378,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1397,6 +1460,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), 1, false, &CancellationToken::new(), @@ -1514,6 +1578,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), concurrency, false, &CancellationToken::new(), @@ -1557,6 +1622,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1620,6 +1686,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), diff --git a/crab/src/metadata/metadb/stores/chunk_index.rs b/crab/src/metadata/metadb/stores/chunk_index.rs index 89dd9597a..0b1d91563 100644 --- a/crab/src/metadata/metadb/stores/chunk_index.rs +++ b/crab/src/metadata/metadb/stores/chunk_index.rs @@ -108,27 +108,58 @@ impl ChunkIndexStore { pub(crate) async fn capsule_candidates( &self, hashes: &[MerkleHash], + ) -> Result { + use crab_metadata::capsule_protocol::capsule_chunk_head_key; + let requests = hashes + .iter() + .map(|hash| (*hash, capsule_chunk_head_key(&(*hash).into()), None)) + .collect::>(); + self.capsule_candidate_records(&requests).await + } + + /// Read immutable capsule placements selected during add, independent of newer heads. + pub(crate) async fn capsule_candidates_by_id( + &self, + candidates: &[(MerkleHash, [u8; 32])], + ) -> Result { + let requests = candidates + .iter() + .map(|(hash, id)| { + ( + *hash, + crab_metadata::key_codec::capsule_chunk_placement_key(id), + Some(*id), + ) + }) + .collect::>(); + self.capsule_candidate_records(&requests).await + } + + async fn capsule_candidate_records( + &self, + requests: &[(MerkleHash, [u8; 33], Option<[u8; 32]>)], ) -> Result { use crab_metadata::capsule_protocol::{ CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, - capsule_chunk_head_key, }; let mut output = CapsuleChunkCandidates::default(); // Stay below Db's dense range-scan threshold: unrelated chunk history // must not turn an add-time point batch into a scan of the global index. - for hashes in hashes.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { - let keys = hashes + for requests in requests.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { + let keys = requests .iter() - .map(|hash| Bytes::copy_from_slice(&capsule_chunk_head_key(&(*hash).into()))) + .map(|(_, key, _)| Bytes::copy_from_slice(key)) .collect::>(); let values = self.db.get_batch(&keys).await?; let mut anchor_ids = HashSet::new(); - for (hash, value) in hashes.iter().zip(values) { + for ((hash, _, expected_id), value) in requests.iter().zip(values) { let Some(value) = value else { continue; }; let placement = CommittedChunkPlacement::decode(&value)?; - if placement.chunk_hash != <[u8; 32]>::from(*hash) { + if placement.chunk_hash != <[u8; 32]>::from(*hash) + || expected_id.is_some_and(|id| placement.placement_id() != id) + { return Err(CrabError::StagingCorrupt( "capsule candidate hash differs from index key".to_owned(), )); diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs index 9553a0c83..7ba9ebd52 100644 --- a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs @@ -1,11 +1,15 @@ //! Rebuildable chunk hints whose source authority is a capsule catalog. -use std::collections::{BTreeMap, HashMap}; +#[cfg(any(test, feature = "remote-index"))] +use std::collections::BTreeMap; +use std::collections::HashMap; use serde::{Deserialize, Serialize}; use super::XorbCatalogEntry; use crate::error::{MetadataError, Result}; -use crate::receipts::{CommittedChunkPlacement, OriginReceipt, RECEIPT_SCHEMA_VERSION}; +#[cfg(any(test, feature = "remote-index"))] +use crate::receipts::RECEIPT_SCHEMA_VERSION; +use crate::receipts::{CommittedChunkPlacement, OriginReceipt}; use crab_xet::hash::MerkleHash; pub use crate::key_codec::{capsule_chunk_anchor_key, capsule_chunk_head_key}; @@ -110,6 +114,7 @@ pub struct CapsuleChunkCandidates { pub anchors: HashMap<[u8; 32], CapsuleChunkAnchor>, } +#[cfg(any(test, feature = "remote-index"))] /// Encode one bounded xorb's candidates after full verification and ref publication. pub(crate) fn capsule_chunk_candidate_rows( repo_prefix: &str, @@ -152,10 +157,14 @@ pub(crate) fn capsule_chunk_candidate_rows( }); } for (hash, placement) in placements { + let encoded = placement.encode()?.to_vec(); + // Preserve the exact add-time assertion when another publication replaces + // the shared head. The placement digest also commits to the chunk key. rows.push(( - capsule_chunk_head_key(&hash).to_vec(), - placement.encode()?.to_vec(), + crate::key_codec::capsule_chunk_placement_key(&placement.placement_id()).to_vec(), + encoded.clone(), )); + rows.push((capsule_chunk_head_key(&hash).to_vec(), encoded)); } Ok(rows) } diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs index aff1436c4..c0f82c629 100644 --- a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs @@ -38,7 +38,7 @@ fn capsule_candidates_preserve_distinct_digests_and_first_occurrence() { anchor.body_digest, anchor.origin.payload_digest ), - (2, 0, entry.body_digest().to_owned(), origin.payload_digest) + (3, 0, entry.body_digest().to_owned(), origin.payload_digest) ); } diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 07c1359c9..de338fc22 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -70,8 +70,8 @@ pub use plan::{ #[cfg(feature = "storage")] pub use pointer::PointerCatalogLookup; pub use pointer::{ - FileCatalogEntry, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, XorbCatalogEntry, - XorbChunkEntry, + FileCatalogEntry, MAX_CATALOG_BYTES, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, @@ -129,4 +129,5 @@ pub use chunk_candidates::{ CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, capsule_chunk_head_key, }; +#[cfg(feature = "remote-index")] pub(crate) use chunk_candidates::capsule_chunk_candidate_rows; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 2b359154a..85255d410 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -10,6 +10,7 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; pub(super) mod paged; +pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] pub use paged::PointerCatalogLookup; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index b7e3e9518..66e273430 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -10,7 +10,8 @@ pub use reader::PointerCatalogLookup; const MAGIC: &[u8; 8] = b"CRBPCP01"; const TRAILER_BYTES: usize = 12; const MAX_DESCRIPTOR_BYTES: usize = 4096; -pub(in crate::capsule_protocol) const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; +/// Maximum byte length of one encoded pointer catalog. +pub const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index e20d5eef9..c07b638c0 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -144,6 +144,7 @@ pub struct CapsuleControl { #[derive(Debug, Clone)] enum ControlVisibility { + #[cfg(any(feature = "storage", test))] Complete(std::sync::Arc), #[cfg(feature = "storage")] Paged(std::sync::Arc), @@ -768,6 +769,9 @@ impl CapsuleControl { pub fn visibility_delta(&self) -> Result> { match &self.visibility { None => Ok(None), + #[cfg(not(any(feature = "storage", test)))] + Some(never) => match *never {}, + #[cfg(any(feature = "storage", test))] Some(ControlVisibility::Complete(delta)) => Ok(Some(delta)), #[cfg(feature = "storage")] Some(ControlVisibility::Paged(_)) => Err(corrupt( diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs index 3ae87ee33..105e8cad5 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs @@ -224,6 +224,7 @@ impl Descriptor { } } +#[cfg(feature = "storage")] impl VisibilityEditHeader { /// Return the prior tip used by the edit. pub fn old_oid(&self) -> Option<&str> { @@ -260,7 +261,9 @@ impl VisibilityEditHeader { _ => false, } } +} +impl VisibilityEditHeader { fn decode(key: &[u8], bytes: &[u8], body_bytes: u64) -> Result { let row: Self = serde_json::from_slice(bytes) .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; diff --git a/crates/crab-metadata/src/key_codec.rs b/crates/crab-metadata/src/key_codec.rs index b0436ec5a..e95d2d79f 100644 --- a/crates/crab-metadata/src/key_codec.rs +++ b/crates/crab-metadata/src/key_codec.rs @@ -13,6 +13,7 @@ //! - `0x05 || anchor_id[32]` — immutable source-shard anchor. //! - `0x06 || hash[32]` — point-readable capsule chunk candidate. //! - `0x07 || anchor_id[32]` — immutable capsule source/origin anchor. +//! - `0x08 || placement_id[32]` — immutable capsule chunk placement. //! - `0xFF || b"sys:" || name` — system key (format_version, epoch, //! created_at, gc_generation). //! @@ -50,6 +51,7 @@ pub const PREFIX_SOURCE_ANCHOR: u8 = 0x05; /// mistake capsule provenance for a manifest-generation assertion. const PREFIX_CAPSULE_CHUNK_HEAD: u8 = 0x06; const PREFIX_CAPSULE_CHUNK_ANCHOR: u8 = 0x07; +const PREFIX_CAPSULE_CHUNK_PLACEMENT: u8 = 0x08; /// Encode a point-readable capsule candidate key. #[must_use] @@ -63,6 +65,12 @@ pub fn capsule_chunk_anchor_key(hash: &[u8; 32]) -> [u8; 33] { encode_proof_record_key(PREFIX_CAPSULE_CHUNK_ANCHOR, hash) } +/// Encode an immutable capsule placement independently of its replaceable head. +#[must_use] +pub fn capsule_chunk_placement_key(id: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_PLACEMENT, id) +} + /// Prefix byte for the `sys:*` system-key namespace. Chosen as `0xFF` /// so it sorts after every content key and is trivial to filter out of /// content-only scans. diff --git a/crates/crab-metadata/src/remote_index/capsule_tests.rs b/crates/crab-metadata/src/remote_index/capsule_tests.rs index a6e396a85..269567268 100644 --- a/crates/crab-metadata/src/remote_index/capsule_tests.rs +++ b/crates/crab-metadata/src/remote_index/capsule_tests.rs @@ -84,3 +84,82 @@ async fn capsule_candidates_writer_rejects_noncanonical_existing_databases() { } } } + +#[tokio::test] +async fn capsule_exact_placement_survives_head_replacement() { + use crate::capsule_protocol::ShardCatalogEntry; + use crate::receipts::CommittedChunkPlacement; + + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("source"); + let xorb = MerkleHash::from([3; 32]); + let chunk = MerkleHash::from([1; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 100, + "4".repeat(64), + vec![XorbChunkEntry::new(chunk.hex(), 9)], + ), + ) + .unwrap(); + catalog + .insert_shard("2".repeat(64), ShardCatalogEntry::new(20, vec![xorb.hex()])) + .unwrap(); + let mut origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", xorb.hex()), + xorb.into(), + [5; 32], + 100, + Some("first".to_owned()), + None, + ); + let mut placements = Vec::new(); + for repo in ["source", "newer-source"] { + write_capsule_chunk_candidates( + Arc::clone(&store), + &config, + repo, + &catalog, + &HashMap::from([(xorb, origin.clone())]), + ) + .await + .unwrap(); + let reader = super::open_reader( + Arc::clone(&store), + &config.chunk_index_path, + "chunk_index_db", + ) + .await + .unwrap() + .unwrap(); + let head = reader + .get(crate::key_codec::capsule_chunk_head_key(&chunk.into())) + .await + .unwrap() + .unwrap(); + reader.close().await.unwrap(); + placements.push(CommittedChunkPlacement::decode(&head).unwrap()); + origin.etag = Some("second".to_owned()); + } + let first = &placements[0]; + let mut key = vec![0x08]; + key.extend_from_slice(&first.placement_id()); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let exact = reader.get(key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!( + ( + placements[0] != placements[1], + exact.map(|value| CommittedChunkPlacement::decode(&value).unwrap()) + ), + (true, Some(first.clone())), + "a changed shared head must not erase the proof selected by an earlier add", + ); +} From 193fdcf86840177b42da5fa89c9258aef6c30994 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:36:37 -0700 Subject: [PATCH 081/127] test(git): verify capsule reuse events exactly --- .../docs/architecture/indexed-remote-reads.md | 40 ++++++++++++ .../e2e/run_add_commit_push_rustfs_smoke.py | 55 +++++++++++++--- .../scripts/e2e/test_add_commit_push_proof.py | 65 +++++++++++++++++++ 3 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 crab/scripts/e2e/test_add_commit_push_proof.py diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index a13bda5eb..52411228a 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2266,3 +2266,43 @@ remote-helper boundary passes date text through, while wire fetch uses a normalized timestamp. Fixing that adapter and qualifying dated boundaries against native Git are required; changing the expected message alone would hide the missing behavior. No full CLI E2E pass is claimed from this receipt. + + +## Capsule chunk authority and exact staged reuse in progress + +PR #593 publishes rebuildable global chunk candidates after capsule ref +publication. Each candidate names a source shard in the source repository's +current authenticated catalog and a canonical origin receipt in the caller's +trusted global scope. A physical xorb alone is insufficient: its catalog shard +must retain the payload through the existing GC contract. Direct CLI publication +and protected receive share the metadata encoder and writer. Protected receive's +80 native-Git/in-memory tests pass; this is not real-S3 qualification. + +The follow-up stores each placement under its immutable digest as well as the +replaceable chunk head. Push re-reads the exact add-time placement and origin +IDs, validates source membership, and requires current provider identity even +with warm payload caches. Its read-only proof session closes before publication +opens the global writer. Verified source descriptors and original receipts stay +caller-owned, with one retained-metadata byte allowance across source repositories. +Conflicting descriptors or physical assertions fail explicitly. + +A regression measured one redundant full xorb GET after otherwise successful +source/origin proof. Adopting the authenticated descriptor removes that GET: +the first before/after pair fails at one versus zero body reads, then passes the +unchanged zero-read bound. Final-source validation passes all 10 focused CLI/Xet tests, including partial +add-time hints and mixed stale/fresh origin assertions. The cold source fixture +uses native Git and real staging with an in-memory object store. +Released v1.2.4 manifest receipts lack these capsule descriptors and still require +full-body verification when imported into capsule publication. + +The dedicated cross-repository smoke oracle now requires one complete capsule +source-proof event and its matching adoption event. It checks exact planned, +indexed, matched, source-verified and payload-verified counts, with zero stale, +mismatched or globally reclassified chunks. Five oracle tests pass; the original +substring oracle rejected valid capsule proofs and accepted counters assembled +from different events. This representation correction is not product E2E proof. +Payload-only run tests pass (28), and strict minimal metadata Clippy passes. + +Current-source S3 cross-repository add/push/cold reconstruction, the shared +`crab-remote` preparation consumer, final 10-GiB qualification, history scaling, +provider latency, lifecycle and full blame semantics remain open gates. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 5e9f70b77..fd9cf3c17 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -102,6 +102,44 @@ class SmokeReport: updated_at: str = "" +def capsule_staged_proof_complete(stderr: str, count: int) -> bool: + """Require complete direct reuse evidence for the capsule duplicate fixture.""" + if type(count) is not int or count <= 0: + return False + proof_message = "revalidated staged remote placement proofs" + adoption_message = "capsule staged remote proof adoption complete" + events = [] + for line in stderr.splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + fields = record.get("fields") if isinstance(record, dict) else None + if isinstance(fields, dict) and fields.get("message") in ( + proof_message, adoption_message + ): + events.append(fields) + # This fixture makes one push with one direct-proof batch. Counts from + # different batches cannot jointly prove that every planned chunk survived. + if len(events) != 2: + return False + proof, adoption = events + if proof.get("message") != proof_message or adoption.get("message") != adoption_message: + return False + expected = { + "planned": count, "indexed": count, "matched": count, + "capsule_source_verified": count, "generation_verified": 0, + "payload_verified": count, "placement_mismatches": 0, + "origin_proof_mismatches": 0, + } + adopted = {"planned_existing": count, "stale_existing": 0, "global_existing": 0} + return all( + type(event.get(key)) is int and event[key] == value + for event, counters in [(proof, expected), (adoption, adopted)] + for key, value in counters.items() + ) + + def utc_now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -2031,13 +2069,7 @@ def run_cross_repository_remote_duplicate_case(self) -> None: remote_chunk_count = inventory["recipe_remote_chunks"] self.check( f"{case_name}-push-revalidates-staged-proof-directly", - "revalidated staged remote placement proofs" in push_stderr - and f'"planned":{remote_chunk_count}' in push_stderr - and f'"matched":{remote_chunk_count}' in push_stderr - and f'"generation_verified":{remote_chunk_count}' in push_stderr - and f'"payload_verified":{remote_chunk_count}' in push_stderr - and '"stale_existing":0' in push_stderr - and '"global_existing":0' in push_stderr, + capsule_staged_proof_complete(push_stderr, remote_chunk_count), {"remote_chunks": remote_chunk_count}, ) self.head_repository_state(consumer_prefix) @@ -2052,12 +2084,19 @@ def run_cross_repository_remote_duplicate_case(self) -> None: ) clone_dir = self.run_root / case_name / "consumer-clone" + clone_cache = self.run_root / case_name / "consumer-clone-cache" + self.check(f"{case_name}-clone-cache-starts-empty", not clone_cache.exists()) + clone_env = {"CRAB_CACHE_DIR": str(clone_cache)} self.run_cmd( f"{case_name} consumer clone", [self.crab_bin, "clone", consumer_url, str(clone_dir), "--jsonl"], self.run_root, + extra_env=clone_env, + ) + self.run_crab( + clone_dir, ["hydrate", "--all"], name=f"{case_name} hydrate", + extra_env=clone_env, ) - self.run_crab(clone_dir, ["hydrate", "--all"], name=f"{case_name} hydrate") self.check( f"{case_name}-fresh-hydrate-is-byte-identical", sha256_file(clone_dir / "model.bin") == expected_sha256, diff --git a/crab/scripts/e2e/test_add_commit_push_proof.py b/crab/scripts/e2e/test_add_commit_push_proof.py new file mode 100644 index 000000000..e9ced8bb2 --- /dev/null +++ b/crab/scripts/e2e/test_add_commit_push_proof.py @@ -0,0 +1,65 @@ +"""Regression tests for the capsule smoke's direct-proof oracle.""" + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from run_add_commit_push_rustfs_smoke import capsule_staged_proof_complete + + +class CapsuleProofTest(unittest.TestCase): + def proof(self): + return { + "message": "revalidated staged remote placement proofs", + "planned": 7, "indexed": 7, "matched": 7, + "generation_verified": 0, "capsule_source_verified": 7, + "payload_verified": 7, "placement_mismatches": 0, + "origin_proof_mismatches": 0, + } + + def adoption(self): + return { + "message": "capsule staged remote proof adoption complete", + "planned_existing": 7, "stale_existing": 0, "global_existing": 0, + } + + def log(self, *events): + return "\n".join(json.dumps({"fields": e}, separators=(",", ":")) for e in events) + + def test_accepts_complete_capsule_evidence(self): + self.assertTrue(capsule_staged_proof_complete(self.log(self.proof(), self.adoption()), 7)) + + def test_rejects_incomplete_or_invalid_counters(self): + for key in self.proof(): + if key == "message": + continue + for value in [None, -1, True, "7", 8]: + with self.subTest(key=key, value=value): + event = {**self.proof(), key: value} + self.assertFalse(capsule_staged_proof_complete(self.log(event, self.adoption()), 7)) + for key in ["planned_existing", "stale_existing", "global_existing"]: + with self.subTest(key=key): + event = {**self.adoption(), key: 1} + self.assertFalse(capsule_staged_proof_complete(self.log(self.proof(), event), 7)) + + def test_rejects_counts_spread_across_events(self): + first = {**self.proof(), "generation_verified": 7, "matched": 0} + second = {**first, "matched": 7, "payload_verified": 0} + self.assertFalse(capsule_staged_proof_complete(self.log(first, second, self.adoption()), 7)) + + def test_rejects_missing_reordered_or_ambiguous_events(self): + for events in [[], [self.proof()], [self.adoption()], + [self.adoption(), self.proof()], + [self.proof(), self.proof(), self.adoption()]]: + with self.subTest(events=events): + self.assertFalse(capsule_staged_proof_complete(self.log(*events), 7)) + + def test_rejects_non_positive_expected_count(self): + for count in [0, -1, True]: + self.assertFalse(capsule_staged_proof_complete(self.log(self.proof(), self.adoption()), count)) + + +if __name__ == "__main__": + unittest.main() From da6fa6599aa4f3e33bc12c5f297b668ea60da856 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:46:57 -0700 Subject: [PATCH 082/127] docs: record exact reuse qualification --- .../docs/architecture/indexed-remote-reads.md | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 52411228a..8a50df839 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2301,8 +2301,26 @@ indexed, matched, source-verified and payload-verified counts, with zero stale, mismatched or globally reclassified chunks. Five oracle tests pass; the original substring oracle rejected valid capsule proofs and accepted counters assembled from different events. This representation correction is not product E2E proof. -Payload-only run tests pass (28), and strict minimal metadata Clippy passes. - -Current-source S3 cross-repository add/push/cold reconstruction, the shared -`crab-remote` preparation consumer, final 10-GiB qualification, history scaling, -provider latency, lifecycle and full blame semantics remain open gates. +Payload-only run tests pass (28), candidate/key-codec/writer tests pass (19), +existing origin/range siblings pass (12), and strict minimal and remote-index +metadata Clippy pass. The production CLI binary build also passes. + +The current-source RustFS cross-repository fixture passes 34 commands and 32 +checks at 4 MiB (`capsule-exact-proof-r2-c6a202ae`, code head `098859c02f1`). +The source worktree is clean, and its recorded source hashes remain unchanged +throughout the run. Consumer add retains neither segment nor prepared payload; +push revalidates all 58 exact placements with zero stale, mismatched or globally +reclassified chunks and uploads no new xorb. Add and push each read two xorb +ranges totaling 2,658 bytes (footer and chunk metadata), with no full xorb GET. +A separate empty clone cache then performs a real remote payload GET during +hydration and produces the original SHA-256. This is one local debug-profile +correctness fixture, not a latency or history-scaling qualification. + +The binary SHA-256 is +`5002c1c59f8e2d6d3fa82abed1c5686c67d46feb93f975c0c1d76c1930f3dc70`; +the report SHA-256 is +`36d6ec1b744a2f5545a9a998710271bc2fa092ceadf756dd56c95acf75eeff2f`. + +The shared `crab-remote` preparation consumer, final 10-GiB qualification, +history scaling, provider latency, lifecycle and full blame semantics remain +open gates. From e4066e19347f013987d0c92a040a2216fc230759 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 06:21:44 -0700 Subject: [PATCH 083/127] fix(remote): bind capsule preparation to proofs --- Cargo.lock | 1 + .../docs/architecture/indexed-remote-reads.md | 44 ++ crates/crab-remote/Cargo.toml | 1 + crates/crab-remote/README.md | 13 + crates/crab-remote/src/prepare.rs | 481 ++------------ crates/crab-remote/src/prepare/capsule.rs | 495 ++++++++++++++ .../crab-remote/tests/capsule_preparation.rs | 621 ++++++++++++++++++ 7 files changed, 1226 insertions(+), 430 deletions(-) create mode 100644 crates/crab-remote/src/prepare/capsule.rs create mode 100644 crates/crab-remote/tests/capsule_preparation.rs diff --git a/Cargo.lock b/Cargo.lock index aa7fddfa3..dc6ffd938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2539,6 +2539,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.10.6", + "slatedb", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 8a50df839..eb032baae 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2324,3 +2324,47 @@ the report SHA-256 is The shared `crab-remote` preparation consumer, final 10-GiB qualification, history scaling, provider latency, lifecycle and full blame semantics remain open gates. + + +## Shared capsule preparation in progress + +The shared `Prepared::upload_capsule` caller in HTTP receive now uses captured +file, shard and xorb lookups instead of materializing unrelated historical +catalogs. An ordinary Git upload with no new Crab pointer performs zero catalog +reads. Selected pointer and prepared-content closures preserve size, membership +and descriptor conflict checks. Capsule-specific implementation lives in +`crates/crab-remote/src/prepare/capsule.rs`; its public API remains under +`crab_remote::prepare`. + +Four regressions failed before the change and pass afterward: one redundant +historical-catalog read for a Git-only upload; Xet-word-order formatting of a raw +body digest; acceptance of rewritten xorb/shard scratch bodies after inspection; +and no shared chunk candidate index after a successful native-content commit. +The initial Git-only fixture sent an all-history pack containing an old pointer; +its corrected incremental pack excludes the advertised tip before measuring. +No weakened assertion or production failure is inferred from that fixture issue. + +Six public-API integration tests and strict all-target Clippy pass. The tests use +native Git packs, real Xet artifacts, explicit publication leases, exact origin +and source-anchor checks, complete file reconstruction, and a noncanonical index +fault that must leave committed refs successful. The SlateDB test dependency uses +the same pinned 0.15.0 release already used by metadata; the lockfile adds only +that test dependency edge. Production dependency versions do not change. + +The dedicated RustFS public-API test passes for both newly created and occupied +xorb keys. It uses native Git preparation, real S3 publication, cold source and +origin-anchor checks, and complete file-hash reconstruction. The historical +pointer integration also accepts the correct declared size and rejects a +conflicting one. The final source passes strict all-target Clippy and the +no-default-feature check. + +Eight HTTP receive/fault tests pass. Three dedicated real-RustFS cells also pass: +the public native-content API, native Git push/fetch and browser editing over the +HTTP listener, and the six receive-fault/restart scenarios. Source hashes remain +unchanged throughout these runs. The API test-binary SHA-256 is +`fe8a75004fd4ef7ec0946b999d32a829b83b28f94b51317a9c34e9990e9ebf5d`; the HTTP test-binary SHA-256 is +`92318648278160dd57f96f35dbcffc01f44799a75a43b84442251965c07fef51`. These are small correctness fixtures, separate from the required +large-file, growth and latency cells. + +The final combined 10-GiB matrix, unrelated-history/ref/frontier scaling, broader +lifecycle/provider proof and fuller blame semantics still prevent completion. diff --git a/crates/crab-remote/Cargo.toml b/crates/crab-remote/Cargo.toml index 67659fc1a..bf30ec429 100644 --- a/crates/crab-remote/Cargo.toml +++ b/crates/crab-remote/Cargo.toml @@ -41,6 +41,7 @@ futures-util = { workspace = true, optional = true } object_store = { workspace = true, optional = true } [dev-dependencies] +slatedb = "=0.15.0" sha1 = "0.10" tracing-subscriber = "0.3" async-trait = { workspace = true } diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 1ba312482..5c391f6d2 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -58,6 +58,19 @@ The pinned-view outcome also reports the last inventory published by that pass, or the input inventory if neither phase published. It deliberately excludes later ref-head updates instead of rescanning them for CLI statistics. +Capsule preparation resolves outgoing pointer files and only the prepared +shard/xorb identities against a captured paged catalog. Ordinary Git and LFS-only +uploads do not read historical Xet catalog pages. Selected dependencies retain +complete closure, size and immutable-descriptor conflict checks. + +Scratch ownership does not freeze artifact bytes. Bounded reads revalidate the +inspected raw Blake3 body identity before upload; Xet protocol identities retain +their distinct word-ordered representation. Capsule commitment retains exact +verified create/read identities with their source catalog, publishes rebuildable +chunk hints only after proven ref visibility, and preserves a committed outcome +if hint-index maintenance fails. Uncertain publication does not publish hints. +Caller-owned ref leases and GC fences cover this complete operation. + `objects` owns bounded tree reads, deterministic tree edits and encoding, commit encoding, and Git object identities. `prepare` owns normalized packs and hydrated content artifacts used by remote commits and recovery. File bodies and diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 4d8ba63bf..7210a1e20 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, fs::File, io::{BufRead, BufReader}, path::PathBuf, @@ -20,7 +20,8 @@ use crab_git::{ }; use crab_metadata::{ capsule_protocol::{ - FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + FileCatalogEntry, PointerCatalog, PointerCatalogLookup, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }, git_object_locator::GitObjectKind, git_visibility::{GitCatalogVisibilityIndex, GitVisibilityEdit, GitVisibilityIndex}, @@ -28,8 +29,11 @@ use crab_metadata::{ use crab_remote_git::{OperationContext, OperationKind, RemoteGitRepository}; use gix_hash::ObjectId; use gix_object::Kind; +use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; +mod capsule; + /// Failure while validating and preparing incoming Git objects. #[derive(Debug, thiserror::Error)] pub enum Error { @@ -125,6 +129,41 @@ impl ContentArtifact { pub const fn size(&self) -> u64 { self.size } + + // Scratch ownership does not freeze file bytes. Revalidate the inspected + // identity with a bounded read before publishing content or its receipt. + async fn read_verified(&self, maximum: u64, cancel: &CancellationToken) -> Result { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + if self.size > maximum { + return Err(Error::Content( + "prepared artifact exceeds its byte limit".to_owned(), + )); + } + let mut bytes = Vec::new(); + tokio::fs::File::open(&self.path) + .await? + .take(self.size + 1) + .read_to_end(&mut bytes) + .await?; + let bytes = Bytes::from(bytes); + let size = self.size; + let hash = self.body_hash; + let bytes = tokio::task::spawn_blocking(move || { + if bytes.len() as u64 != size || blake3::hash(&bytes).as_bytes() != &hash { + return Err(Error::Content( + "prepared artifact changed after inspection".to_owned(), + )); + } + Ok(bytes) + }) + .await??; + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + Ok(bytes) + } } /// One Crab pointer supplied by a prepared reconstruction shard. @@ -236,15 +275,14 @@ impl PreparedContent { { return Err(Error::Content("shard identity is invalid".to_owned())); } - let bytes = tokio::fs::read(&artifact.path).await?; - if bytes.len() as u64 != artifact.size - || blake3::hash(&bytes).as_bytes() != &artifact.body_hash - || <[u8; 32]>::from(crab_xet::hash::compute_data_hash(&bytes)) - != artifact.protocol_hash + let bytes = artifact + .read_verified(crab_xet::shard::MAX_PUSH_SHARD_SIZE_BYTES, cancel) + .await?; + if <[u8; 32]>::from(crab_xet::hash::compute_data_hash(&bytes)) != artifact.protocol_hash { return Err(Error::Content("shard binding changed".to_owned())); } - shard_bodies.insert(artifact.protocol_hash, Bytes::from(bytes)); + shard_bodies.insert(artifact.protocol_hash, bytes); } let mut seen_files = std::collections::BTreeSet::new(); let mut used_shards = std::collections::BTreeSet::new(); @@ -330,7 +368,9 @@ impl PreparedContent { crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), XorbCatalogEntry::new( artifact.size, - crab_xet::hash::MerkleHash::from(artifact.body_hash).hex(), + blake3::Hash::from_bytes(artifact.body_hash) + .to_hex() + .to_string(), inspected .chunks .iter() @@ -428,6 +468,8 @@ pub struct CapsuleArtifacts { edits: Vec, packs: Vec, sections: Vec, + pointer_delta: PointerCatalog, + origins: HashMap, changes_namespace: bool, } @@ -809,427 +851,6 @@ impl Prepared { shards, }) } - - /// Upload artifacts for one captured capsule-protocol repository view. - /// - /// The caller must hold ref leases and GC fences from before opening - /// `view` through commitment. This validates all historical dependencies - /// against that view, uploads same-publication Xet content before Git - /// visibility, and retains the exact root snapshot for per-ref CAS. - pub async fn upload_capsule( - &self, - view: &crab_read::capsule_protocol::CapsuleRepositoryView, - limits: crab_read::dependency_proof::DependencyProofLimits, - cancel: &CancellationToken, - ) -> Result { - let started = Instant::now(); - let repository_refs = self - .repository - .refs() - .entries - .iter() - .map(|reference| (reference.name.clone(), reference.target.to_string())) - .collect::>(); - if repository_refs != *view.refs() - || self.repository.generation() != view.root().root().generation() - { - return Err(Error::Request( - "Capsule view differs from the prepared repository version", - )); - } - let lookup = view.pointer_catalog_lookup(&self.layout, 2 * 1024 * 1024 * 1024)?; - let local = self.content.as_ref().map(PreparedContent::pointer_catalog); - let mut files = self - .plan - .pointers() - .iter() - .filter_map(|dependency| match &dependency.pointer { - crab_git::pointer_detect::PointerKind::Crab(pointer) => { - Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) - } - _ => None, - }) - .collect::>(); - files.extend( - local - .into_iter() - .flat_map(|catalog| catalog.files().keys().cloned()), - ); - let files = files.into_iter().collect::>(); - let shards = local - .into_iter() - .flat_map(|catalog| catalog.shards().keys().cloned()) - .collect::>(); - let xorbs = local - .into_iter() - .flat_map(|catalog| catalog.xorbs().keys().cloned()) - .collect::>(); - // Local artifacts can collide with historical identities even when - // none of their files are present. Select those keys and the outgoing - // file closure together; unrelated history must stay cold. - let base_catalog = lookup - .dependencies(self.layout.store(), &files, &shards, &xorbs) - .await?; - let prepared_files = self - .content - .as_ref() - .map(|content| { - content - .files - .iter() - .map(|file| file.file_hash) - .collect::>() - }) - .unwrap_or_default(); - crab_read::dependency_proof::verify_capsule_dependencies_except_crab( - &self.layout, - &base_catalog, - self.plan.pointers(), - &prepared_files, - limits, - cancel, - ) - .await - .map_err(|error| Error::Dependency(Box::new(error)))?; - - let pointer_delta = match &self.content { - Some(content) => { - upload_capsule_content(&self.layout, &base_catalog, content, cancel).await? - } - None => PointerCatalog::new(), - }; - let mut packs = Vec::new(); - if let Some(pack) = &self.pack { - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let (pack_bytes, index, reverse_index, locator) = tokio::try_join!( - tokio::fs::read(pack.pack_path()), - tokio::fs::read(pack.index_path()), - tokio::fs::read(pack.reverse_path()), - tokio::fs::read(pack.kinds_path()), - )?; - let external_delta_bases = pack - .external_delta_bases() - .iter() - .map(ToString::to_string) - .collect(); - packs.push( - crab_metadata::capsule_protocol::CapsuleGitPack::new_with_external_delta_bases( - Bytes::from(pack_bytes), - Bytes::from(index), - Bytes::from(reverse_index), - Bytes::from(locator), - pack.git_sha1().to_string(), - u64::from(pack.object_count()), - external_delta_bases, - )?, - ); - } - let mut sections = Vec::with_capacity(2); - if !pointer_delta.is_empty() { - sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( - crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta, - pointer_delta.encode_delta()?, - )); - } - if !self.visibility.is_empty() { - let visibility = crab_metadata::capsule_protocol::CapsuleVisibilityDelta::new( - self.visibility.clone(), - )?; - sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( - crab_metadata::capsule_protocol::CapsuleSectionKind::VisibilityDelta, - visibility.encode()?, - )); - } - let edits = self - .updates - .iter() - .map(|update| { - crab_metadata::capsule_protocol::CapsuleRefEdit::new( - update.name.clone(), - update.old.map(|oid| oid.to_string()), - update.new.map(|oid| oid.to_string()), - self.plan - .peeled() - .get(&update.name) - .map(ToString::to_string), - ) - }) - .collect::>(); - let changes_namespace = edits - .iter() - .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); - tracing::info!( - phase = "capsule_input", - packs = packs.len(), - elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, - "Git receive phase completed" - ); - Ok(CapsuleArtifacts { - layout: self.layout.clone(), - base: view.root_snapshot().clone(), - edits, - packs, - sections, - changes_namespace, - }) - } -} - -impl CapsuleArtifacts { - /// Commit through independently mutable ref heads and one capsule authority. - /// - /// Revalidate authorization and repository policy immediately before this - /// call. The caller must retain its original ref leases and GC fences until - /// the returned outcome is known and all lease cleanup has completed. - pub async fn commit( - self, - plan_id: Option<&str>, - namespace_ttl: Duration, - cancel: &CancellationToken, - ) -> Result { - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let transaction = match plan_id { - Some(plan_id) => crab_metadata::capsule_protocol::CapsuleTransaction::for_plan( - self.base.record().digest(), - plan_id, - self.edits, - )?, - None => crab_metadata::capsule_protocol::CapsuleTransaction::new( - self.base.record().digest(), - self.edits, - )?, - }; - let transaction_id = transaction.id()?; - let started = Instant::now(); - let capsule = crab_metadata::capsule_protocol::Capsule::build( - &transaction, - self.packs, - self.sections, - )?; - tracing::info!( - phase = "capsule_encode", - %transaction_id, - bytes = capsule.bytes().len(), - elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, - "Git receive phase completed" - ); - let publish = if self.changes_namespace { - let ref_names = transaction - .edits() - .iter() - .map(|edit| edit.ref_name().to_owned()) - .collect::>(); - let layout = self.layout.clone(); - let commit_layout = layout.clone(); - let base = self.base; - crab_write::with_ref_namespaces( - layout.store(), - &layout, - &ref_names, - namespace_ttl, - cancel, - move |scoped| async move { - if scoped.is_cancelled() { - return Err(crab_write::WriteError::Cancelled); - } - crab_write::capsule_protocol::validate_ref_namespace( - &commit_layout, - base.record().root(), - transaction.edits(), - ) - .await?; - crab_write::capsule_protocol::publish( - &commit_layout, - base, - &transaction, - &capsule, - ) - .await - }, - ) - .await - } else { - crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) - .await - }; - match publish { - Ok(_) => Ok(CapsuleCommitOutcome::Committed { transaction_id }), - Err(error @ crab_write::WriteError::CapsuleCommitUncertain { .. }) => { - Ok(CapsuleCommitOutcome::Indeterminate { - transaction_id, - source: Box::new(error), - }) - } - Err(error) => Err(error.into()), - } - } -} - -async fn upload_capsule_content( - layout: &crab_storage::StoreLayout, - base: &PointerCatalog, - content: &PreparedContent, - cancel: &CancellationToken, -) -> Result { - let local = content.pointer_catalog(); - let mut delta = PointerCatalog::new(); - let mut needed_shards = std::collections::BTreeSet::new(); - for (file_hash, file) in local.files() { - if let Some(existing) = base.files().get(file_hash) { - if existing.size() != file.size() { - return Err(Error::Content(format!( - "file {file_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - needed_shards.insert(file.shard_hash().to_owned()); - delta.insert_file(file_hash.clone(), file.clone())?; - } - let mut needed_xorbs = std::collections::BTreeSet::new(); - for shard_hash in &needed_shards { - let shard = local - .shards() - .get(shard_hash) - .ok_or_else(|| Error::Content("prepared shard catalog entry is missing".to_owned()))?; - if let Some(existing) = base.shards().get(shard_hash) { - if existing != shard { - return Err(Error::Content(format!( - "shard {shard_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - needed_xorbs.extend(shard.xorb_hashes().iter().cloned()); - delta.insert_shard(shard_hash.clone(), shard.clone())?; - } - let xorb_artifacts = content - .xorbs - .iter() - .map(|artifact| { - ( - crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), - artifact, - ) - }) - .collect::>(); - for xorb_hash in needed_xorbs { - let local_entry = local - .xorbs() - .get(&xorb_hash) - .ok_or_else(|| Error::Content("prepared xorb catalog entry is missing".to_owned()))?; - if let Some(existing) = base.xorbs().get(&xorb_hash) { - if existing.chunks() != local_entry.chunks() { - return Err(Error::Content(format!( - "xorb {xorb_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let artifact = xorb_artifacts - .get(&xorb_hash) - .ok_or_else(|| Error::Content("prepared xorb artifact is missing".to_owned()))?; - let bytes = Bytes::from(tokio::fs::read(&artifact.path).await?); - let hash = crab_xet::hash::MerkleHash::from(artifact.protocol_hash); - let entry = match layout - .store() - .create_or_read_immutable( - &layout.xorb_path(&hash), - bytes, - crab_xet::xorb::format::MAX_XORB_SIZE as u64, - ) - .await? - { - crab_storage::ImmutableCreateOutcome::Created(_) => local_entry.clone(), - crab_storage::ImmutableCreateOutcome::Existing { - bytes: existing, .. - } => catalog_xorb_from_existing(&xorb_hash, local_entry, existing)?, - }; - delta.insert_xorb(xorb_hash, entry)?; - } - let shard_artifacts = content - .shards - .iter() - .map(|artifact| { - ( - crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), - artifact, - ) - }) - .collect::>(); - for shard_hash in needed_shards { - if base.shards().contains_key(&shard_hash) { - continue; - } - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let artifact = shard_artifacts - .get(&shard_hash) - .ok_or_else(|| Error::Content("prepared shard artifact is missing".to_owned()))?; - let bytes = Bytes::from(tokio::fs::read(&artifact.path).await?); - layout - .store() - .put_if_absent_verified( - &layout.shard_path(&crab_xet::hash::MerkleHash::from(artifact.protocol_hash)), - bytes, - ) - .await?; - } - if !delta.shards().is_empty() { - crab_metadata::ref_registry::union_register_repo_shards( - layout.store(), - layout, - delta.shards().keys().cloned().collect(), - ) - .await?; - } - let mut complete = base.clone(); - complete.apply(&delta)?; - Ok(delta) -} - -fn catalog_xorb_from_existing( - xorb_hash: &str, - local: &XorbCatalogEntry, - bytes: Bytes, -) -> Result { - let parser = - crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).map_err(Error::ContentFormat)?; - parser - .verify_payload_digest() - .map_err(Error::ContentFormat)?; - parser.verify_all_chunks().map_err(Error::ContentFormat)?; - if parser.hash().hex() != xorb_hash { - return Err(Error::Content( - "existing xorb has the wrong logical identity".to_owned(), - )); - } - let chunks = (0..parser.num_chunks()) - .map(|index| { - parser - .chunk_meta(index) - .map(|chunk| XorbChunkEntry::new(chunk.hash.hex(), chunk.uncompressed_len)) - }) - .collect::, _>>() - .map_err(Error::ContentFormat)?; - if chunks != local.chunks() { - return Err(Error::Content( - "existing xorb has a conflicting chunk layout".to_owned(), - )); - } - Ok(XorbCatalogEntry::new( - bytes.len() as u64, - blake3::hash(&bytes).to_hex().to_string(), - chunks, - )) } impl Artifacts<'_> { diff --git a/crates/crab-remote/src/prepare/capsule.rs b/crates/crab-remote/src/prepare/capsule.rs new file mode 100644 index 000000000..dcf26b988 --- /dev/null +++ b/crates/crab-remote/src/prepare/capsule.rs @@ -0,0 +1,495 @@ +use super::*; + +impl Prepared { + /// Upload artifacts for one captured capsule-protocol repository view. + /// + /// The caller must hold ref leases and GC fences from before opening + /// `view` through commitment. This validates all historical dependencies + /// against that view, uploads same-publication Xet content before Git + /// visibility, and retains the exact root snapshot for per-ref CAS. + pub async fn upload_capsule( + &self, + view: &crab_read::capsule_protocol::CapsuleRepositoryView, + limits: crab_read::dependency_proof::DependencyProofLimits, + cancel: &CancellationToken, + ) -> Result { + let started = Instant::now(); + let repository_refs = self + .repository + .refs() + .entries + .iter() + .map(|reference| (reference.name.clone(), reference.target.to_string())) + .collect::>(); + if repository_refs != *view.refs() + || self.repository.generation() != view.root().root().generation() + { + return Err(Error::Request( + "Capsule view differs from the prepared repository version", + )); + } + if self.plan.pointers().len() > limits.max_dependencies { + return Err(Error::Dependency(Box::new( + crab_read::dependency_proof::DependencyProofError::Limit("pointer count"), + ))); + } + let lookup = view.pointer_catalog_lookup( + &self.layout, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + )?; + let hashes = self + .plan + .pointers() + .iter() + .filter_map(|dependency| match &dependency.pointer { + crab_git::pointer_detect::PointerKind::Crab(pointer) => { + Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) + } + _ => None, + }) + .collect::>(); + // Ordinary Git and LFS-only changes need no Xet catalog pages. Selected + // Crab pointers retain complete authenticated shard/xorb closure checks. + let base_catalog = if hashes.is_empty() { + PointerCatalog::new() + } else { + lookup.files(self.layout.store(), &hashes).await? + }; + let prepared_files = self + .content + .as_ref() + .map(|content| { + content + .files + .iter() + .map(|file| file.file_hash) + .collect::>() + }) + .unwrap_or_default(); + crab_read::dependency_proof::verify_capsule_dependencies_except_crab( + &self.layout, + &base_catalog, + self.plan.pointers(), + &prepared_files, + limits, + cancel, + ) + .await + .map_err(|error| Error::Dependency(Box::new(error)))?; + + let (pointer_delta, origins) = match &self.content { + Some(content) => { + upload_capsule_content(&self.layout, &lookup, base_catalog, content, cancel).await? + } + None => (PointerCatalog::new(), HashMap::new()), + }; + let mut packs = Vec::new(); + if let Some(pack) = &self.pack { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let (pack_bytes, index, reverse_index, locator) = tokio::try_join!( + tokio::fs::read(pack.pack_path()), + tokio::fs::read(pack.index_path()), + tokio::fs::read(pack.reverse_path()), + tokio::fs::read(pack.kinds_path()), + )?; + let external_delta_bases = pack + .external_delta_bases() + .iter() + .map(ToString::to_string) + .collect(); + packs.push( + crab_metadata::capsule_protocol::CapsuleGitPack::new_with_external_delta_bases( + Bytes::from(pack_bytes), + Bytes::from(index), + Bytes::from(reverse_index), + Bytes::from(locator), + pack.git_sha1().to_string(), + u64::from(pack.object_count()), + external_delta_bases, + )?, + ); + } + let mut sections = Vec::with_capacity(1); + if !self.visibility.is_empty() { + let visibility = crab_metadata::capsule_protocol::CapsuleVisibilityDelta::new( + self.visibility.clone(), + )?; + sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( + crab_metadata::capsule_protocol::CapsuleSectionKind::VisibilityDelta, + visibility.encode()?, + )); + } + let edits = self + .updates + .iter() + .map(|update| { + crab_metadata::capsule_protocol::CapsuleRefEdit::new( + update.name.clone(), + update.old.map(|oid| oid.to_string()), + update.new.map(|oid| oid.to_string()), + self.plan + .peeled() + .get(&update.name) + .map(ToString::to_string), + ) + }) + .collect::>(); + let changes_namespace = edits + .iter() + .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); + tracing::info!( + phase = "capsule_input", + packs = packs.len(), + elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, + "Git receive phase completed" + ); + Ok(CapsuleArtifacts { + layout: self.layout.clone(), + base: view.root_snapshot().clone(), + edits, + packs, + sections, + pointer_delta, + origins, + changes_namespace, + }) + } +} + +impl CapsuleArtifacts { + /// Commit through independently mutable ref heads and one capsule authority. + /// + /// Revalidate authorization and repository policy immediately before this + /// call. The caller must retain its original ref leases and GC fences until + /// the returned outcome is known and all lease cleanup has completed. + pub async fn commit( + self, + plan_id: Option<&str>, + namespace_ttl: Duration, + cancel: &CancellationToken, + ) -> Result { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let transaction = match plan_id { + Some(plan_id) => crab_metadata::capsule_protocol::CapsuleTransaction::for_plan( + self.base.record().digest(), + plan_id, + self.edits, + )?, + None => crab_metadata::capsule_protocol::CapsuleTransaction::new( + self.base.record().digest(), + self.edits, + )?, + }; + let transaction_id = transaction.id()?; + let started = Instant::now(); + let mut sections = self.sections; + if !self.pointer_delta.is_empty() { + sections.insert( + 0, + crab_metadata::capsule_protocol::CapsuleSection::new( + crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta, + self.pointer_delta.encode_delta()?, + ), + ); + } + let capsule = + crab_metadata::capsule_protocol::Capsule::build(&transaction, self.packs, sections)?; + tracing::info!( + phase = "capsule_encode", + %transaction_id, + bytes = capsule.bytes().len(), + elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, + "Git receive phase completed" + ); + let publish = if self.changes_namespace { + let ref_names = transaction + .edits() + .iter() + .map(|edit| edit.ref_name().to_owned()) + .collect::>(); + let layout = self.layout.clone(); + let commit_layout = layout.clone(); + let base = self.base; + crab_write::with_ref_namespaces( + layout.store(), + &layout, + &ref_names, + namespace_ttl, + cancel, + move |scoped| async move { + if scoped.is_cancelled() { + return Err(crab_write::WriteError::Cancelled); + } + crab_write::capsule_protocol::validate_ref_namespace( + &commit_layout, + base.record().root(), + transaction.edits(), + ) + .await?; + crab_write::capsule_protocol::publish( + &commit_layout, + base, + &transaction, + &capsule, + ) + .await + }, + ) + .await + } else { + crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) + .await + }; + match publish { + Ok(_) => { + // Ref visibility owns the payload before hints become discoverable. + // Failed hint maintenance cannot reject an already committed Git push. + let config = + crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + self.layout.repo_prefix(), + self.layout.global_prefix(), + ); + if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(self.layout.store().inner()), + &config, + self.layout.repo_prefix(), + &self.pointer_delta, + &self.origins, + ) + .await + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + } + Ok(CapsuleCommitOutcome::Committed { transaction_id }) + } + Err(error @ crab_write::WriteError::CapsuleCommitUncertain { .. }) => { + Ok(CapsuleCommitOutcome::Indeterminate { + transaction_id, + source: Box::new(error), + }) + } + Err(error) => Err(error.into()), + } + } +} + +async fn upload_capsule_content( + layout: &crab_storage::StoreLayout, + lookup: &PointerCatalogLookup, + mut base: PointerCatalog, + content: &PreparedContent, + cancel: &CancellationToken, +) -> Result<( + PointerCatalog, + HashMap, +)> { + let local = content.pointer_catalog(); + let mut delta = PointerCatalog::new(); + let mut origins = HashMap::new(); + let mut needed_shards = std::collections::BTreeSet::new(); + for (file_hash, file) in local.files() { + if let Some(existing) = base.files().get(file_hash) { + if existing.size() != file.size() { + return Err(Error::Content(format!( + "file {file_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + needed_shards.insert(file.shard_hash().to_owned()); + delta.insert_file(file_hash.clone(), file.clone())?; + } + if !needed_shards.is_empty() { + base.apply( + &lookup + .shards( + layout.store(), + &needed_shards.iter().cloned().collect::>(), + ) + .await?, + )?; + } + let mut needed_xorbs = std::collections::BTreeSet::new(); + for shard_hash in &needed_shards { + let shard = local + .shards() + .get(shard_hash) + .ok_or_else(|| Error::Content("prepared shard catalog entry is missing".to_owned()))?; + if let Some(existing) = base.shards().get(shard_hash) { + if existing != shard { + return Err(Error::Content(format!( + "shard {shard_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + needed_xorbs.extend(shard.xorb_hashes().iter().cloned()); + delta.insert_shard(shard_hash.clone(), shard.clone())?; + } + if !needed_xorbs.is_empty() { + base.apply( + &lookup + .xorbs( + layout.store(), + &needed_xorbs.iter().cloned().collect::>(), + ) + .await?, + )?; + } + let xorb_artifacts = content + .xorbs + .iter() + .map(|artifact| { + ( + crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), + artifact, + ) + }) + .collect::>(); + for xorb_hash in needed_xorbs { + let local_entry = local + .xorbs() + .get(&xorb_hash) + .ok_or_else(|| Error::Content("prepared xorb catalog entry is missing".to_owned()))?; + if let Some(existing) = base.xorbs().get(&xorb_hash) { + if existing.chunks() != local_entry.chunks() { + return Err(Error::Content(format!( + "xorb {xorb_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let artifact = xorb_artifacts + .get(&xorb_hash) + .ok_or_else(|| Error::Content("prepared xorb artifact is missing".to_owned()))?; + let bytes = artifact + .read_verified(crab_xet::xorb::format::MAX_XORB_SIZE as u64, cancel) + .await?; + let hash = crab_xet::hash::MerkleHash::from(artifact.protocol_hash); + let path = layout.xorb_path(&hash); + let payload_digest = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()) + .map_err(Error::ContentFormat)? + .payload_digest(); + let (entry, payload_digest, identity) = match layout + .store() + .create_or_read_immutable(&path, bytes, crab_xet::xorb::format::MAX_XORB_SIZE as u64) + .await? + { + crab_storage::ImmutableCreateOutcome::Created(identity) => { + (local_entry.clone(), payload_digest, identity) + } + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, + identity, + } => { + let expected = local_entry.clone(); + let expected_hash = xorb_hash.clone(); + let (entry, payload_digest) = tokio::task::spawn_blocking(move || { + catalog_xorb_from_existing(&expected_hash, &expected, existing) + }) + .await??; + (entry, payload_digest, identity) + } + }; + origins.insert( + hash, + crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + hash.into(), + payload_digest, + entry.encoded_size(), + identity.e_tag, + identity.version, + ), + ); + delta.insert_xorb(xorb_hash, entry)?; + } + let shard_artifacts = content + .shards + .iter() + .map(|artifact| { + ( + crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), + artifact, + ) + }) + .collect::>(); + for shard_hash in needed_shards { + if base.shards().contains_key(&shard_hash) { + continue; + } + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let artifact = shard_artifacts + .get(&shard_hash) + .ok_or_else(|| Error::Content("prepared shard artifact is missing".to_owned()))?; + let bytes = artifact + .read_verified(crab_xet::shard::MAX_PUSH_SHARD_SIZE_BYTES, cancel) + .await?; + layout + .store() + .put_if_absent_verified( + &layout.shard_path(&crab_xet::hash::MerkleHash::from(artifact.protocol_hash)), + bytes, + ) + .await?; + } + if !delta.shards().is_empty() { + crab_metadata::ref_registry::union_register_repo_shards( + layout.store(), + layout, + delta.shards().keys().cloned().collect(), + ) + .await?; + } + base.apply(&delta)?; + Ok((delta, origins)) +} + +fn catalog_xorb_from_existing( + xorb_hash: &str, + local: &XorbCatalogEntry, + bytes: Bytes, +) -> Result<(XorbCatalogEntry, [u8; 32])> { + let parser = + crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).map_err(Error::ContentFormat)?; + parser + .verify_payload_digest() + .map_err(Error::ContentFormat)?; + parser.verify_all_chunks().map_err(Error::ContentFormat)?; + if parser.hash().hex() != xorb_hash { + return Err(Error::Content( + "existing xorb has the wrong logical identity".to_owned(), + )); + } + let chunks = (0..parser.num_chunks()) + .map(|index| { + parser + .chunk_meta(index) + .map(|chunk| XorbChunkEntry::new(chunk.hash.hex(), chunk.uncompressed_len)) + }) + .collect::, _>>() + .map_err(Error::ContentFormat)?; + if chunks != local.chunks() { + return Err(Error::Content( + "existing xorb has a conflicting chunk layout".to_owned(), + )); + } + Ok(( + XorbCatalogEntry::new( + bytes.len() as u64, + blake3::hash(&bytes).to_hex().to_string(), + chunks, + ), + parser.payload_digest(), + )) +} diff --git a/crates/crab-remote/tests/capsule_preparation.rs b/crates/crab-remote/tests/capsule_preparation.rs new file mode 100644 index 000000000..52d9db6fe --- /dev/null +++ b/crates/crab-remote/tests/capsule_preparation.rs @@ -0,0 +1,621 @@ +#![cfg(feature = "publication")] + +use std::sync::Arc; + +use bytes::Bytes; +use crab_remote::prepare::{ContentArtifact, ContentFile, PreparedContent}; +use crab_xet::hash::{MerkleHash, compute_data_hash}; +use crab_xet::shard::{PushShardSession, file_info_from_placements, xorb_info_from_placements}; +use crab_xet::xorb::builder::{RunId, XorbBuilder}; +use crab_xet::xorb::format::Chunk; +use tokio_util::sync::CancellationToken; + +const LIMIT: u64 = 8 * 1024 * 1024; + +async fn content_fixture() -> (tempfile::TempDir, PreparedContent) { + let directory = tempfile::tempdir().unwrap(); + let data = Bytes::from_static(b"prepared capsule content must remain reconstructable"); + let chunk_hash = compute_data_hash(&data); + let file_hash = MerkleHash::from(*blake3::hash(&data).as_bytes()); + let mut builder = XorbBuilder::new(); + builder + .push( + &Chunk { + hash: chunk_hash, + data: data.clone(), + }, + RunId(0), + ) + .unwrap(); + let xorb = builder.finalize().unwrap().pop().unwrap(); + let placements = xorb + .placements + .iter() + .map(|placement| (placement.chunk_hash, placement.clone())) + .collect(); + let file = file_info_from_placements(file_hash, &[chunk_hash], &placements).unwrap(); + let xorb_info = Arc::new(xorb_info_from_placements(xorb.hash, &xorb.placements).unwrap()); + let mut shards = PushShardSession::new(); + shards.add_file_bundle(file, &[xorb_info]).unwrap(); + let (shard, shard_hash) = shards.finalize().unwrap().pop().unwrap(); + let xorb_path = directory.path().join("content.xorb"); + let shard_path = directory.path().join("content.shard"); + std::fs::write(&xorb_path, &xorb.bytes).unwrap(); + std::fs::write(&shard_path, &shard).unwrap(); + let content = PreparedContent::new( + vec![ContentArtifact::new( + xorb.hash.into(), + *blake3::hash(&xorb.bytes).as_bytes(), + xorb.bytes.len() as u64, + xorb_path, + )], + vec![ContentArtifact::new( + shard_hash.into(), + *blake3::hash(&shard).as_bytes(), + shard.len() as u64, + shard_path, + )], + vec![ContentFile::new( + file_hash.into(), + data.len() as u64, + shard_hash.into(), + )], + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + (directory, content) +} + +#[tokio::test] +async fn prepared_catalog_preserves_raw_blake3_body_identity() { + let (directory, content) = content_fixture().await; + let expected = blake3::hash(&std::fs::read(directory.path().join("content.xorb")).unwrap()) + .to_hex() + .to_string(); + let actual = content + .pointer_catalog() + .xorbs() + .values() + .next() + .unwrap() + .body_digest(); + assert_eq!( + actual, expected, + "body digests use raw Blake3 hex; only protocol identities use Xet word order" + ); +} + +#[derive(Default)] +struct Reads(std::sync::atomic::AtomicUsize); +impl crab_storage::StorageObserver for Reads { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, observation: crab_storage::StorageObservation) { + if matches!( + observation.operation, + crab_storage::StorageOperation::Get | crab_storage::StorageOperation::Range + ) { + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + } +} + +fn git(directory: &std::path::Path, args: &[&str]) -> Vec { + git_input(directory, args, b"") +} + +fn git_input(directory: &std::path::Path, args: &[&str], input: &[u8]) -> Vec { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgSign=false", + "-c", + "core.hooksPath=/dev/null", + ]) + .args(args) + .current_dir(directory) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(input).unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + +fn source_commit(directory: &std::path::Path, content: &PreparedContent) { + git(directory, &["init", "-b", "main"]); + let file = &content.files()[0]; + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\n", + blake3::Hash::from_bytes(*file.file_hash()).to_hex(), + file.size() + ); + std::fs::write(directory.join("model.bin"), pointer).unwrap(); + git(directory, &["add", "model.bin"]); + git(directory, &["commit", "-m", "publish pointer"]); +} + +async fn layout() -> (crab_storage::StoreLayout, Arc) { + let reads = Arc::new(Reads::default()); + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(reads.clone()); + let layout = crab_storage::StoreLayout::new(store, "repositories/prepared".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + (layout, reads) +} + +fn proof_limits() -> crab_read::dependency_proof::DependencyProofLimits { + crab_read::dependency_proof::DependencyProofLimits { + max_dependencies: 16, + max_total_file_bytes: LIMIT, + max_duration: std::time::Duration::from_secs(60), + lookup: crab_metadata::file_index_lookup::FileIndexLookupLimits { + max_files: 16, + max_shard_visits: 16, + max_shard_bytes: LIMIT, + max_recipe_entries: 1024, + }, + content: crab_read::pointer_proof::PointerProofLimits { + max_file_bytes: LIMIT, + max_shard_bytes: LIMIT, + max_xorb_bytes: LIMIT, + max_read_bytes: LIMIT, + max_chunks: 1024, + max_duration: std::time::Duration::from_secs(60), + }, + } +} + +async fn prepare( + layout: &crab_storage::StoreLayout, + directory: &std::path::Path, + content: Option, +) -> ( + crab_remote::prepare::Prepared, + crab_read::capsule_protocol::CapsuleRepositoryView, + Arc, + crab_remote::publication::PublicationLeases, +) { + let cancel = CancellationToken::new(); + let leases = crab_remote::publication::acquire_leases( + layout.store(), + layout, + ["refs/heads/main".to_owned()], + crab_remote::publication::LeaseOptions::renewing(std::time::Duration::from_secs(60)), + &cancel, + ) + .await + .unwrap(); + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + let view = crab_read::capsule_protocol::open_git_view_from_root( + layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let repository = view + .git_repository_from_store( + layout.clone(), + crab_remote_git::RepositoryIdentity::new("memory", "prepared", 1).unwrap(), + runtime.clone(), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap(); + let oid = gix_hash::ObjectId::from_hex( + String::from_utf8(git(directory, &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .as_bytes(), + ) + .unwrap(); + let mut revisions = String::from("HEAD\n"); + if let Some(old) = view.refs().get("refs/heads/main") { + revisions.push_str(&format!("^{old}\n")); + } + let pack = git_input( + directory, + &["pack-objects", "--stdout", "--revs"], + revisions.as_bytes(), + ); + let incoming = directory.join("incoming.pack"); + std::fs::write(&incoming, pack).unwrap(); + let limits = crab_git::incoming_pack::ReceiveLimits { + max_pack_bytes: LIMIT, + max_objects: 1024, + max_object_bytes: LIMIT as usize, + max_inflated_bytes: LIMIT, + max_delta_depth: 128, + }; + let prepared = crab_remote::prepare::prepare_capsule( + repository, + view.git_visibility_index().await.unwrap(), + crab_git::incoming_pack::ScratchSpace::new(directory, limits), + Some(std::io::BufReader::new( + std::fs::File::open(incoming).unwrap(), + )), + vec![crab_git::receive_plan::RefUpdate { + name: "refs/heads/main".to_owned(), + old: view + .refs() + .get("refs/heads/main") + .map(|oid| gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap()), + new: Some(oid), + }], + std::collections::BTreeMap::new(), + &cancel, + crab_remote::prepare::Options { + layout: layout.clone(), + graph: crab_git::receive_plan::GraphLimits { + max_ref_updates: 16, + max_graph_steps: 1024, + max_object_bytes: LIMIT as usize, + max_read_bytes: LIMIT, + }, + pack: limits, + policy: |_: &str| crab_git::receive_plan::RefPolicy { + allow_delete: true, + allow_non_fast_forward: false, + }, + }, + ) + .await + .unwrap(); + let prepared = match content { + Some(content) => prepared.attach_content(content).unwrap(), + None => prepared, + }; + (prepared, view, runtime, leases) +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_upload_rejects_changed_prepared_bytes() { + let mut rejected = Vec::new(); + for artifact in ["content.xorb", "content.shard"] { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + std::fs::write(directory.path().join(artifact), b"changed after inspection").unwrap(); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + leases.release().await; + runtime.shutdown().await; + rejected.push((artifact, result.is_err())); + } + assert!( + rejected.iter().all(|(_, failed)| *failed), + "changed artifacts must fail before ref publication: {rejected:?}" + ); +} + +async fn verify_indexed_publication( + layout: crab_storage::StoreLayout, + occupied: bool, +) { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let chunk = content + .pointer_catalog() + .xorbs() + .values() + .next() + .unwrap() + .chunks()[0] + .hash() + .to_owned(); + if occupied { + let artifact = &content.xorbs()[0]; + layout + .store() + .put( + &layout.xorb_path(&MerkleHash::from(*artifact.protocol_hash())), + Bytes::from(std::fs::read(directory.path().join("content.xorb")).unwrap()), + ) + .await + .unwrap(); + } + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let result = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!(matches!( + result.unwrap(), + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + let reader = + slatedb::DbReader::builder(config.chunk_index_path, Arc::clone(layout.store().inner())) + .build() + .await + .expect("shared publication must populate cold native-add candidates"); + let hash = MerkleHash::from_hex(&chunk).unwrap(); + let candidate = reader + .get(crab_metadata::key_codec::capsule_chunk_head_key( + &hash.into(), + )) + .await + .unwrap() + .expect("published payload candidate"); + let placement = crab_metadata::receipts::CommittedChunkPlacement::decode(&candidate).unwrap(); + let anchor = reader + .get(crab_metadata::key_codec::capsule_chunk_anchor_key( + &placement.source_anchor_id, + )) + .await + .unwrap() + .unwrap(); + let anchor = crab_metadata::capsule_protocol::CapsuleChunkAnchor::decode( + &placement.source_anchor_id, + &anchor, + ) + .unwrap(); + reader.close().await.unwrap(); + let path = layout.xorb_path(&MerkleHash::from(placement.xorb_hash)); + let (body, identity) = layout.store().get_with_etag(&path).await.unwrap(); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let current = crab_read::capsule_protocol::open_git_view_from_root( + &layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); + let catalog = current.pointer_catalog().await.unwrap(); + anchor + .validate_placement(&placement, &catalog, path.as_ref()) + .unwrap(); + assert_eq!( + ( + anchor.origin.etag, + anchor.origin.object_version, + anchor.body_digest + ), + ( + identity.e_tag, + identity.version, + blake3::hash(&body).to_hex().to_string() + ) + ); + let crab_git::pointer_detect::PointerKind::Crab(pointer) = crab_git::pointer_detect::classify( + &std::fs::read(directory.path().join("model.bin")).unwrap(), + ) else { + panic!("fixture pointer"); + }; + crab_read::pointer_proof::verify_crab_pointer( + &layout, + &pointer, + MerkleHash::from_hex(catalog.files().values().next().unwrap().shard_hash()).unwrap(), + proof_limits().content, + &CancellationToken::new(), + ) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_commit_indexes_verified_native_payload() { + for occupied in [false, true] { + let (layout, _) = layout().await; + verify_indexed_publication(layout, occupied).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] +async fn live_capsule_preparation_publishes_and_reconstructs_native_content() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").expect("explicit scratch bucket required"); + let store = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + for occupied in [false, true] { + let prefix = format!("shared-preparation/{nonce}/{occupied}"); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.clone(), + format!("{prefix}/repo"), + format!("{prefix}/global"), + ); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + verify_indexed_publication(layout, occupied).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn git_only_capsule_upload_does_not_read_historical_catalog() { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, reads) = layout().await; + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let committed = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + committed, + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + leases.release_after_commit().await; + runtime.shutdown().await; + git(directory.path(), &["rm", "model.bin"]); + std::fs::write(directory.path().join("README.md"), b"ordinary Git update").unwrap(); + git(directory.path(), &["add", "README.md"]); + git( + directory.path(), + &["commit", "-m", "replace pointer with ordinary Git file"], + ); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), None).await; + assert!(prepared.plan().pointers().is_empty()); + reads.0.store(0, std::sync::atomic::Ordering::Relaxed); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + let count = reads.0.load(std::sync::atomic::Ordering::Relaxed); + leases.release().await; + runtime.shutdown().await; + result.unwrap(); + assert_eq!( + count, 0, + "ordinary Git publication must not read unrelated Xet history" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn committed_capsule_survives_candidate_index_failure() { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + let db = slatedb::Db::open(config.chunk_index_path, Arc::clone(layout.store().inner())) + .await + .unwrap(); + db.put( + crab_metadata::key_codec::encode_system_key(crab_metadata::key_codec::SYS_FORMAT_VERSION), + 2u32.to_le_bytes(), + ) + .await + .unwrap(); + db.close().await.unwrap(); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let result = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!( + matches!( + result.unwrap(), + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + ), + "repairable hint-index failure must not reject committed refs" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn historical_pointer_lookup_preserves_declared_size_checks() { + for size_delta in [0, 1] { + let (directory, content) = content_fixture().await; + let file = content.files()[0].clone(); + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let committed = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + committed, + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + leases.release_after_commit().await; + runtime.shutdown().await; + // A hint changes the Git blob identity while retaining the existing file. + // Native negotiation therefore sends a real historical-pointer dependency. + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\nshard-hint {}\n", + blake3::Hash::from_bytes(*file.file_hash()).to_hex(), + file.size() + size_delta, + blake3::Hash::from_bytes(*file.shard_hash()).to_hex() + ); + std::fs::write(directory.path().join("model.bin"), pointer).unwrap(); + git(directory.path(), &["add", "model.bin"]); + git( + directory.path(), + &["commit", "-m", "update pointer binding"], + ); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), None).await; + assert_eq!(prepared.plan().pointers().len(), 1); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + leases.release().await; + runtime.shutdown().await; + assert_eq!( + result.is_ok(), + size_delta == 0, + "historical pointer size delta {size_delta}" + ); + } +} From 12df8905061e13a420820b4e4e82dbbb35f076d1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 06:42:53 -0700 Subject: [PATCH 084/127] test(gc): adapt captured catalog sweep call --- crab/src/cmd/gc/capsule_cleanup_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crab/src/cmd/gc/capsule_cleanup_tests.rs b/crab/src/cmd/gc/capsule_cleanup_tests.rs index 9a3a7f618..d1e0311e1 100644 --- a/crab/src/cmd/gc/capsule_cleanup_tests.rs +++ b/crab/src/cmd/gc/capsule_cleanup_tests.rs @@ -187,7 +187,6 @@ async fn captured_catalog_pages_survive_checkpoint_replacement_and_gc() { &layout, root.record().root(), current.capsule_run_pointers(), - ¤t.state_digest(), &snapshot.manifest, &HashSet::new(), &CancellationToken::new(), From 33eaa289ef9e440af9bf36698b0ed76e0fe294a1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 07:27:30 -0700 Subject: [PATCH 085/127] test(http): serialize corruption fault injection Drain the preceding repair worker and bind each injected fault to current readiness. A published ready record can precede worker completion, so overlapping mutations can legitimately observe graph-only readiness. --- .../docs/architecture/indexed-remote-reads.md | 31 +++++++++++++++++++ crates/crab-http-server/src/receive_tests.rs | 18 ++++++++--- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index eb032baae..866cc3b3d 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2368,3 +2368,34 @@ large-file, growth and latency cells. The final combined 10-GiB matrix, unrelated-history/ref/frontier scaling, broader lifecycle/provider proof and fuller blame semantics still prevent completion. + + +## Combined catalog/staging qualification and HTTP fault boundaries + +The combined revision carries selected publication, verified catalog bindings, +source/browse retention, retained-source fsck, exact capsule chunk reuse and +shared native-content preparation. Focused checks passed: 558 metadata tests, +81 protected-receive tests, six shared-preparation tests, 242 staging tests, +26 CLI integration tests, 57 CLI siblings, 20 origin/lease tests and eight HTTP +receive/fault tests. Explicit live tests remain separately qualified; these +counts do not establish the full release gate. + +The first combined RustFS native HTTP fixture returned 202 during its +corrupt-descriptor case, where the corruption oracle required 503. Readiness +can become visible before the publishing repair worker finishes. A diagnostic +assertion with a one-second delayed generation-owner release response proved +that the fixture could inject its next fault while that worker was active. +The shared native HTTP fixture now awaits maintenance completion before each +fault, reloads the current descriptor and checks its snapshot identity. All +original status, error-code, recovery and Git assertions remain intact. The +controlled probe then passed across ten delayed lease releases, and all eight +HTTP unit/fault checks passed. This changes test synchronization, not the valid +production distinction between incomplete readiness and corrupt metadata. + +Raw before/after evidence remains outside the checkout in +`final-integration-http-metadata-probe-r4.json`, +`final-integration-http-metadata-probe-r5.json` and +`http-fault-boundary-after-r1.log`. Combined-source 10 GiB qualification, +representative S3 latency, additional history/ref scaling and fuller Git blame +semantics remain open gates. Earlier component receipts retain their original +source identities and are not final combined-source proof. diff --git a/crates/crab-http-server/src/receive_tests.rs b/crates/crab-http-server/src/receive_tests.rs index 498dcb05e..60240fca3 100644 --- a/crates/crab-http-server/src/receive_tests.rs +++ b/crates/crab-http-server/src/receive_tests.rs @@ -415,17 +415,25 @@ async fn exercise(server: Arc, branch: &str) { .await .unwrap(); - let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&repo.layout) - .await - .unwrap() - .unwrap(); - let path_state_hash = indexes.path_state_hash().unwrap(); for corruption in [ "missing descriptor", "corrupt descriptor", "malformed record", "oversized record", ] { + // Readiness can precede its repair worker's completion. Drain it before + // injecting another fault, or that worker can replace the tested record + // with legitimate graph-only readiness before the corruption probe. + server.finish_maintenance().await.unwrap(); + let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&repo.layout) + .await + .unwrap() + .unwrap(); + assert_eq!( + indexes.state_digest(), + repo.open_view().await.unwrap().state_digest() + ); + let path_state_hash = indexes.path_state_hash().unwrap(); let descriptor = repo .layout .bulk_manifest_path("path-state", path_state_hash); From 348fe3d24630e46ceaf9a2ac55f322babdf1db54 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 07:49:46 -0700 Subject: [PATCH 086/127] fix(fsck): pin source lifetime through verification Retain decoded head CAS tokens and the pruning floor through source, data and Git verification. Reject concurrent authority movement or corrupt control bytes without rescanning immutable history. --- crab/src/cmd/fsck_store.rs | 31 +++- .../cmd/fsck_store/capsule_history_tests.rs | 153 +++++++++++++++++- crates/crab-metadata/README.md | 3 + .../src/capsule_protocol/source_retention.rs | 44 ++++- .../source_retention/tests.rs | 50 ++++++ 5 files changed, 272 insertions(+), 9 deletions(-) diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 8dc5aae48..af740c9f0 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -95,6 +95,7 @@ pub struct StoreChecker { struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, + source_lifetime: Arc, } impl StoreChecker { @@ -134,13 +135,17 @@ impl StoreChecker { ) .await?; let catalog = view.pointer_catalog().await?; - verify_capsule_history(&layout, view.root().root()).await?; + let source_lifetime = verify_capsule_history(&layout, view.root().root()).await?; Ok(Self { store, prefix, router, multipart_journal: None, - capsule: Some(CapsuleFsckState { view, catalog }), + capsule: Some(CapsuleFsckState { + view, + catalog, + source_lifetime: Arc::new(source_lifetime), + }), }) } @@ -233,6 +238,7 @@ impl StoreChecker { } } + self.check_capsule_root_stability(state).await?; Ok(issues) } @@ -258,7 +264,7 @@ impl StoreChecker { .to_owned(), )); } - Ok(()) + require_current_source_lifetime(&layout, &state.source_lifetime).await } /// Prove source pointers against the exact catalog captured by a capsule view. @@ -888,7 +894,7 @@ impl StoreChecker { async fn verify_capsule_history( layout: &crab_storage::StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, -) -> Result<()> { +) -> Result { let segments = match root.history() { Some(history) => { crab_metadata::capsule_protocol::load_history_chain( @@ -981,7 +987,22 @@ async fn verify_capsule_history( .buffer_unordered(16) .try_collect::>() .await?; - Ok(()) + // Per-ref retirement and pruning can move without changing visible refs. + // Keep the capture for the later data/Git checks instead of rescanning history. + require_current_source_lifetime(layout, &retained).await?; + Ok(retained) +} + +async fn require_current_source_lifetime( + layout: &crab_storage::StoreLayout, + capture: &crab_metadata::capsule_protocol::RetainedCapsuleSources, +) -> Result<()> { + if capture.is_current(layout).await? { + return Ok(()); + } + Err(CrabError::Protocol( + "source lifetime changed during fsck; retry against one stable capture".to_owned(), + )) } fn insert_history_pointer( diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index c8b0cb691..364dbc243 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -5,8 +5,91 @@ use crab_metadata::capsule_protocol::{ LayeredCheckpoint, PackLayer, PackSourceDescriptor, PointerCatalog, RootSnapshot, }; use crab_storage::{StorageObservation, StorageObserver, StorageOperation}; -use object_store::{ObjectStoreExt, memory::InMemory}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, memory::InMemory, + path::Path as ObjectPath, +}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CorruptLifetimeDuringRead { + inner: Arc, + source: ObjectPath, + head: ObjectPath, + fired: AtomicBool, +} + +impl std::fmt::Display for CorruptLifetimeDuringRead { + fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + output.write_str("source lifetime corruption fixture") + } +} + +#[async_trait::async_trait] +impl ObjectStore for CorruptLifetimeDuringRead { + async fn put_opts( + &self, + path: &ObjectPath, + payload: PutPayload, + options: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + + async fn put_multipart_opts( + &self, + path: &ObjectPath, + options: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + + async fn get_opts( + &self, + path: &ObjectPath, + options: GetOptions, + ) -> object_store::Result { + let full_body = !options.head && options.range.is_none(); + let response = self.inner.get_opts(path, options).await?; + if full_body && *path == self.source && !self.fired.swap(true, Ordering::SeqCst) { + self.inner + .put(&self.head, Bytes::from_static(b"corrupt").into()) + .await?; + } + Ok(response) + } + + fn delete_stream( + &self, + paths: futures_util::stream::BoxStream<'static, object_store::Result>, + ) -> futures_util::stream::BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + + fn list( + &self, + prefix: Option<&ObjectPath>, + ) -> futures_util::stream::BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&ObjectPath>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &ObjectPath, + to: &ObjectPath, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} async fn history_fixture() -> ( crab_storage::StoreLayout, @@ -104,6 +187,72 @@ struct SourceReads { count: AtomicUsize, } +#[tokio::test] +async fn fsck_rejects_lifetime_corruption_during_original_source_verification() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + let head = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await + .unwrap() + .remove(0) + .location; + let backend = Arc::new(CorruptLifetimeDuringRead { + inner: Arc::clone(layout.store().inner()), + source: layout.capsule_path(run.hash()), + head, + fired: AtomicBool::new(false), + }); + let observed = crab_storage::StoreLayout::new( + crab_storage::Store::new(backend.clone()), + layout.repo_prefix().to_owned(), + ); + let result = verify_capsule_history(&observed, root.record().root()).await; + assert!(backend.fired.load(Ordering::SeqCst)); + assert!( + result.is_err(), + "fsck accepted corrupt lifetime metadata after its original source read: {result:?}" + ); +} + +#[tokio::test] +async fn data_checks_reject_lifetime_movement_after_checker_construction() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + let checker = StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(layout.store().clone()), + layout.repo_prefix().to_owned(), + root, + ) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + crab_metadata::capsule_protocol::prune_retained_capsule_sources(&layout, 0) + .await + .unwrap(); + assert!(matches!( + checker.check_data_chain().await, + Err(CrabError::Protocol(reason)) if reason.contains("source lifetime changed") + )); +} + #[tokio::test] async fn retired_sources_are_checked_without_checkpoint_history() { let (layout, root, run) = history_fixture().await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 69c34c83d..2754bab09 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -150,6 +150,9 @@ a per-ref head under `v2/history/source-heads/` and authenticated source-pointer segments under `v2/history/source-roots/`. Foreground catalog sessions stay read-only and keep their original source ranges. GC discovers these roots in its existing history enumeration; fsck authenticates their original run proofs. +Fsck retains the decoded heads' CAS tokens and the pruning floor through later +data and Git checks. It rechecks mutable authority without decoding history again; +concurrent movement requests a retry, while invalid control bytes fail validation. Explicit history pruning advances a persistent generation floor after history CAS, trims older segments, and prevents expired publishers from restoring them. These new lifetime records do not change capsule, checkpoint, or catalog formats. diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention.rs b/crates/crab-metadata/src/capsule_protocol/source_retention.rs index 65bf1df2e..1053e3a64 100644 --- a/crates/crab-metadata/src/capsule_protocol/source_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/source_retention.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crab_storage::{ETag, Store, StoreLayout}; +use futures_util::{StreamExt, TryStreamExt}; use object_store::path::Path; use serde::{Deserialize, Serialize}; @@ -48,9 +49,12 @@ struct Segment { } /// Authenticated lifetime dependencies; they never authorize ref or object visibility. +#[derive(Debug)] pub struct RetainedCapsuleSources { paths: Vec, runs: Vec, + head_versions: BTreeMap, + floor: Option<(u64, ETag)>, } impl RetainedCapsuleSources { @@ -65,6 +69,38 @@ impl RetainedCapsuleSources { pub fn runs(&self) -> &[CapsulePointer] { &self.runs } + + /// Check whether a complete capture still names the current lifetime authority. + /// + /// The loader must have received every source head. Invalid control records + /// return an error; valid concurrent movement returns false for caller retry. + pub async fn is_current(&self, layout: &StoreLayout) -> Result { + let heads = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await?; + if heads.len() != self.head_versions.len() + || heads + .iter() + .any(|meta| !self.head_versions.contains_key(&meta.location)) + { + return Ok(false); + } + // LIST can omit version IDs. Compare the GET CAS tokens that bound + // decoded heads so versioned buckets keep the same stability proof. + let mut versions = futures_util::stream::iter(self.head_versions.clone()) + .map(|(path, version)| async move { + let current = load_head(layout, &path).await?; + Ok::<_, MetadataError>(current.is_some_and(|(_, current)| current == version)) + }) + .buffer_unordered(16); + while let Some(current) = versions.try_next().await? { + if !current { + return Ok(false); + } + } + Ok(load_floor(layout).await?.as_ref() == self.floor.as_ref()) + } } impl Tip { @@ -347,20 +383,22 @@ pub async fn load_retained_capsule_sources( ) -> Result { let mut paths = BTreeSet::new(); let mut runs = BTreeMap::new(); + let mut head_versions = BTreeMap::new(); let mut roots = 0usize; let mut bytes = 0u64; let floor = load_floor(layout).await?; if floor.is_some() { paths.insert(layout.capsule_source_floor_path()); } - let minimum_generation = floor.map_or(0, |(floor, _)| floor); + let minimum_generation = floor.as_ref().map_or(0, |(floor, _)| *floor); for path in heads { if !paths.insert(path.clone()) { continue; } - let (head, _) = load_head(layout, path) + let (head, version) = load_head(layout, path) .await? .ok_or_else(|| corrupt("source lifetime head disappeared"))?; + head_versions.insert(path.clone(), version); if head.minimum_generation > minimum_generation { return Err(corrupt("source head exceeds the repository pruning floor")); } @@ -392,6 +430,8 @@ pub async fn load_retained_capsule_sources( Ok(RetainedCapsuleSources { paths: paths.into_iter().collect(), runs: runs.into_values().collect(), + head_versions, + floor, }) } diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs index 62d06c3d0..e12965453 100644 --- a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs @@ -75,6 +75,56 @@ async fn repeated_retirement_does_not_append_or_rewrite_lifetime() { assert_eq!(layout.store().get_with_etag(&path).await.unwrap(), before); } +#[tokio::test] +async fn source_snapshot_detects_appends_and_new_ref_heads() { + let layout = fixture(); + let first = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[first]) + .await + .unwrap(); + let captured = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + assert!(captured.is_current(&layout).await.unwrap()); + let second = source(&layout, 2).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, std::slice::from_ref(&second)) + .await + .unwrap(); + assert!(!captured.is_current(&layout).await.unwrap()); + let captured = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + retain_capsule_sources(&layout, "refs/heads/other", 0, &[second]) + .await + .unwrap(); + assert!(!captured.is_current(&layout).await.unwrap()); +} + +#[tokio::test] +async fn source_snapshot_tracks_floor_without_rewriting_ref_heads() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[pointer]) + .await + .unwrap(); + let paths = heads(&layout).await; + let before = layout.store().get_with_etag(&paths[0]).await.unwrap(); + let captured = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 0).await.unwrap(); + assert_eq!( + layout.store().get_with_etag(&paths[0]).await.unwrap(), + before + ); + assert!(!captured.is_current(&layout).await.unwrap()); + let captured = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 0).await.unwrap(); + assert!(captured.is_current(&layout).await.unwrap()); +} + #[tokio::test] async fn independent_ref_heads_share_source_identity_without_sharing_cas() { let layout = fixture(); From 95fcf5e8edf6af43c861cf2d930444fcdd330fda Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 08:49:35 -0700 Subject: [PATCH 087/127] fix(fsck): retain browse capture through checks --- crab/src/cmd/fsck_store.rs | 34 ++++-- .../cmd/fsck_store/capsule_history_tests.rs | 111 ++++++++++++++++++ crates/crab-metadata/README.md | 6 +- .../src/capsule_protocol/browse_retention.rs | 29 ++++- .../browse_retention/tests.rs | 33 ++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 2 +- 6 files changed, 200 insertions(+), 15 deletions(-) diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index af740c9f0..98e9005d3 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -96,6 +96,8 @@ struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, source_lifetime: Arc, + browse_lifetime: + Arc>, } impl StoreChecker { @@ -145,6 +147,7 @@ impl StoreChecker { view, catalog, source_lifetime: Arc::new(source_lifetime), + browse_lifetime: Arc::new(tokio::sync::OnceCell::new()), }), }) } @@ -163,11 +166,15 @@ impl StoreChecker { ); // Current catalog health cannot prove cold captures still have their // original sources. Full fsck checks lifetime roots; mirror recipe probes stay narrow. - crab_metadata::capsule_protocol::verify_retained_browse_indexes( - &layout, - MAX_FSCK_FRONTIER_BYTES, - ) - .await?; + state + .browse_lifetime + .get_or_try_init(|| { + crab_metadata::capsule_protocol::verify_retained_browse_indexes( + &layout, + MAX_FSCK_FRONTIER_BYTES, + ) + }) + .await?; let mut issues = Vec::new(); for (hash, entry) in state.catalog.xorbs() { @@ -264,7 +271,17 @@ impl StoreChecker { .to_owned(), )); } - require_current_source_lifetime(&layout, &state.source_lifetime).await + require_current_source_lifetime(&layout, &state.source_lifetime).await?; + // Full fsck keeps browse authority through payload and final checks. + // Mirror recipe probes never initialize this expensive maintenance proof. + if let Some(capture) = state.browse_lifetime.get() + && !capture.is_current(&layout).await? + { + return Err(CrabError::Protocol( + "browse retention changed during fsck; retry against one stable capture".to_owned(), + )); + } + Ok(()) } /// Prove source pointers against the exact catalog captured by a capsule view. @@ -1902,7 +1919,7 @@ mod tests { create_manifest(store, &router, &manifest).await.unwrap(); } - async fn capsule_checker_fixture() -> ( + pub(super) async fn capsule_checker_fixture() -> ( Store, String, crab_metadata::capsule_protocol::RootSnapshot, @@ -1930,7 +1947,8 @@ mod tests { let parser = XorbParser::parse(xorb_bytes.clone()).unwrap(); assert_eq!(parser.num_chunks(), 1); let chunk = parser.chunk_meta(0).unwrap(); - let file_hash = chunk.hash; + // File identity uses ordinary Blake3; Xet's chunk hash is keyed. + let file_hash = MerkleHash::from(*blake3::hash(&content).as_bytes()); let placement = crab_xet::xorb::format::ChunkPlacement { chunk_hash: chunk.hash, xorb_hash, diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index 364dbc243..bf3fa7de8 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -224,6 +224,117 @@ async fn fsck_rejects_lifetime_corruption_during_original_source_verification() ); } +#[tokio::test] +async fn data_checks_reject_browse_corruption_during_xorb_verification() { + for readiness in [false, true] { + let (store, prefix, root, xorb_hash, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let backend = Arc::new(CorruptLifetimeDuringRead { + inner: Arc::clone(store.inner()), + source: router.xorb_path(&xorb_hash), + head: if readiness { + layout.capsule_browse_indexes_path() + } else { + layout.browse_retained_head_path() + }, + fired: AtomicBool::new(false), + }); + let store = crab_storage::Store::new(backend.clone()); + let checker = StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(store), + prefix, + root, + ) + .await + .unwrap(); + let result = checker.check_data_chain().await; + assert!(backend.fired.load(Ordering::SeqCst)); + assert!( + result.is_err(), + "fsck accepted corrupt browse metadata after its xorb read: readiness={readiness}, {result:?}" + ); + } +} + +#[tokio::test] +async fn final_snapshot_check_rejects_browse_movement_after_payload_verification() { + for readiness in [false, true] { + let (store, prefix, root, _, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let checker = StoreChecker::for_capsule_repository(store.clone(), prefix, root) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + let indexes = crab_metadata::capsule_protocol::BrowseIndexes::new( + "a".repeat(64), + "b".repeat(64), + None, + ) + .unwrap(); + if readiness { + store + .put( + &layout.capsule_browse_indexes_path(), + indexes.encode().unwrap(), + ) + .await + .unwrap(); + } else { + crab_metadata::capsule_protocol::retain_browse_indexes(&layout, &indexes, 0, None, &[]) + .await + .unwrap(); + } + assert!(matches!( + checker.check_orphan_file_index().await, + Err(CrabError::Protocol(reason)) if reason.contains("browse retention changed") + )); + } +} + +#[tokio::test] +async fn capsule_recipe_probe_does_not_require_browse_maintenance() { + let (store, prefix, root, _, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + store + .put( + &router.repo_path("v2/browse-retention"), + Bytes::from_static(b"corrupt"), + ) + .await + .unwrap(); + let checker = StoreChecker::for_capsule_repository(store, prefix, root) + .await + .unwrap(); + let (hash, file) = checker + .capsule + .as_ref() + .unwrap() + .catalog + .files() + .first_key_value() + .unwrap(); + let pointer = Pointer { + file_hash: MerkleHash::from_hex(hash).unwrap().into(), + size: file.size(), + shard_hint: None, + }; + let proof = checker + .verify_capsule_pointer_data(&[pointer], &CancellationToken::new()) + .await + .unwrap(); + assert!(proof.verified == 1 && proof.issues.is_empty(), "{proof:?}"); +} + #[tokio::test] async fn data_checks_reject_lifetime_movement_after_checker_construction() { let (layout, root, run) = history_fixture().await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 2754bab09..993fdbcbb 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -133,8 +133,10 @@ requires background browse maintenance before GC; there is no scan fallback. Full capsule fsck also authenticates the retained chain, decodes complete graph/path-state indexes, and verifies physical-index and original source bytes, including superseded ref-only runs. Shared captures stream each source body once -during this phase. Lifetime/readiness movement during verification requires a retry; -mirror pointer-recipe probes do not acquire this full maintenance workload. +during this phase. Fsck holds the verified lifetime/readiness capture through +payload verification and the final snapshot check. Revalidation reads only bounded +control records; valid movement requires a retry and corruption fails validation. +Mirror pointer-recipe probes do not initialize this full maintenance workload. Explicit history pruning trims superseded captures below the oldest retained checkpoint generation only after the authoritative history CAS succeeds. Current diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 48b26afb3..407d15424 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -288,12 +288,13 @@ pub async fn load_retained_browse_indexes( /// Verify retained index/source integrity against stable lifetime and readiness heads. /// /// Maintenance decodes complete graph/path indexes and streams original source -/// and physical-index bodies. This proves retained bytes, never ref visibility. +/// and physical-index bodies. Retain the returned capture and recheck it after +/// later phases. This proves retained bytes, never ref visibility. #[cfg(feature = "remote-index")] pub async fn verify_retained_browse_indexes( layout: &StoreLayout, maximum_bytes: u64, -) -> Result<()> { +) -> Result { let head = load_head(layout).await?; let current = super::load_browse_indexes(layout).await?; let records = match &head { @@ -363,12 +364,32 @@ pub async fn verify_retained_browse_indexes( } // Browse maintenance can change lifetime without moving a Git ref. A // concurrent prune/rebuild must yield a retry, never a clean mixed capture. - if load_head(layout).await? != head || super::load_browse_indexes(layout).await? != current { + let capture = BrowseRetentionCapture { head, current }; + if !capture.is_current(layout).await? { return Err(corrupt( "browse retention changed during verification; retry fsck", )); } - Ok(()) + Ok(capture) +} + +/// Verified browse lifetime and readiness authority retained across maintenance phases. +#[cfg(feature = "remote-index")] +#[derive(Debug)] +pub struct BrowseRetentionCapture { + head: Option<(Head, ETag)>, + current: Option, +} + +#[cfg(feature = "remote-index")] +impl BrowseRetentionCapture { + /// Revalidate control identity without decoding indexes or original sources again. + /// + /// Invalid records return an error; valid concurrent movement returns false. + pub async fn is_current(&self, layout: &StoreLayout) -> Result { + Ok(load_head(layout).await? == self.head + && super::load_browse_indexes(layout).await? == self.current) + } } /// Trim captured generations after authoritative history pruning commits. diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index e5fb4d855..4fb9f5491 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -248,6 +248,39 @@ async fn integrity_checks_every_retained_body_after_readiness_replacement() { .unwrap(); } +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn verified_capture_distinguishes_control_movement_from_corruption() { + for readiness in [false, true] { + let layout = fixture(); + let capture = verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); + assert!(capture.is_current(&layout).await.unwrap()); + let path = if readiness { + let path = layout.capsule_browse_indexes_path(); + layout + .store() + .put(&path, indexes('a').encode().unwrap()) + .await + .unwrap(); + path + } else { + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + layout.browse_retained_head_path() + }; + assert!(!capture.is_current(&layout).await.unwrap()); + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + assert!(capture.is_current(&layout).await.is_err()); + } +} + #[tokio::test] #[cfg(feature = "remote-index")] async fn advertised_readiness_requires_authenticated_lifetime_ownership() { diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index de338fc22..bbecfb74f 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -33,7 +33,7 @@ mod visibility; pub use browse::load_browse_indexes; pub use browse::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES}; #[cfg(feature = "remote-index")] -pub use browse_retention::verify_retained_browse_indexes; +pub use browse_retention::{BrowseRetentionCapture, verify_retained_browse_indexes}; #[cfg(feature = "storage")] pub use browse_retention::{ RetainedBrowseIndexes, load_retained_browse_indexes, prune_retained_browse_indexes, From 150448cf701d1727d46597bcee4961faf4e77e77 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 10:40:20 -0700 Subject: [PATCH 088/127] fix(fsck): verify reachable pointer payloads --- crab/docs/guides/fsck.md | 8 +- crab/src/cmd/fsck_store.rs | 46 +++- crab/src/cmd/fsck_store/pointer_data_tests.rs | 226 ++++++++++++++++++ 3 files changed, 269 insertions(+), 11 deletions(-) create mode 100644 crab/src/cmd/fsck_store/pointer_data_tests.rs diff --git a/crab/docs/guides/fsck.md b/crab/docs/guides/fsck.md index e07f692c3..0312f9136 100644 --- a/crab/docs/guides/fsck.md +++ b/crab/docs/guides/fsck.md @@ -13,6 +13,12 @@ crab fsck [OPTIONS] `crab fsck` cross-checks Crab manifests, pack/index presence, the shard/xorb data chain, and coordination state against the object store. +For protocol-v2 repositories, the check pins an authenticated root, verifies +Git connectivity, and follows every reachable Crab or Git LFS pointer to its +origin payload. Crab files must reconstruct to the pointer's exact Blake3 hash +and size; LFS objects must match their SHA-256 identity and size. Full checking +reads the payload bytes, even when a local cache already contains them. + With `--repair`, it can automatically fix certain categories of issues. ## Options @@ -104,7 +110,7 @@ crab fsck: 1 error(s), 0 info, 1 repaired, 0 repair failure(s) ## Verification Boundaries -- The Git-object check currently confirms that ref-journal records are +- For legacy layouts, the Git-object check confirms that ref-journal records are readable; it does not reconstruct the current repository and run full Git reachability for missing commits, trees, or blobs. Use `crab recover history verify ` for strict Git fsck of a selected historical root, and diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 98e9005d3..a3845ee03 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -91,13 +91,13 @@ pub struct StoreChecker { capsule: Option, } -#[derive(Clone)] struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, source_lifetime: Arc, browse_lifetime: Arc>, + git_workspace: tokio::sync::OnceCell, } impl StoreChecker { @@ -148,6 +148,7 @@ impl StoreChecker { catalog, source_lifetime: Arc::new(source_lifetime), browse_lifetime: Arc::new(tokio::sync::OnceCell::new()), + git_workspace: tokio::sync::OnceCell::new(), }), }) } @@ -245,6 +246,25 @@ impl StoreChecker { } } + if issues.is_empty() { + // Keep the authenticated Git database across phases. Catalog bodies + // alone cannot prove reachable pointer coverage or whole-file/LFS bytes. + let workspace = state + .git_workspace + .get_or_try_init(|| { + check_capsule_git_connectivity(&self.store, &self.router, state) + }) + .await?; + crab_read::capsule_protocol::verify_installed_dependencies( + &layout, + workspace.path(), + state.view.refs(), + &state.catalog, + CAPSULE_GIT_SCAN_LIMITS, + &CancellationToken::new(), + ) + .await?; + } self.check_capsule_root_stability(state).await?; Ok(issues) } @@ -1214,10 +1234,10 @@ async fn install_capsule_git_packs( } async fn check_capsule_git_connectivity( - store: Store, - router: StoreLayout, - state: CapsuleFsckState, -) -> Result<()> { + store: &Store, + router: &StoreLayout, + state: &CapsuleFsckState, +) -> Result { let workspace = tempfile::tempdir()?; crab_git::initialize_bare_git_dir(workspace.path())?; let storage = store.as_storage().clone(); @@ -1233,7 +1253,7 @@ async fn check_capsule_git_connectivity( .map(|(name, oid)| (name.clone(), oid.clone())) .collect::>(); install_capsule_git_packs( - state.view, + state.view.clone(), layout, workspace.path().to_owned(), MAX_FSCK_FRONTIER_BYTES, @@ -1247,7 +1267,8 @@ async fn check_capsule_git_connectivity( .await .map_err(|error| CrabError::Internal(format!("Git connectivity scan failed: {error}")))??; crab_git::batch::verify_git_dir_blobs(workspace.path(), &scan.unchecked_blobs, &|| false) - .map_err(CrabError::Io) + .map_err(CrabError::Io)?; + Ok(workspace) } impl FsckChecker for StoreChecker { @@ -1257,13 +1278,15 @@ impl FsckChecker for StoreChecker { let store = self.store.clone(); let router = self.router.clone(); let prefix = self.prefix.clone(); - let capsule = self.capsule.clone(); Box::pin(async move { - if let Some(state) = capsule { + if let Some(state) = &self.capsule { // Pack installation validates every advertised pack. The reachable // walker then proves each ref's commit/tree/blob closure instead of // treating an authenticated ref OID as sufficient evidence. - check_capsule_git_connectivity(store, router, state).await?; + state + .git_workspace + .get_or_try_init(|| check_capsule_git_connectivity(&store, &router, state)) + .await?; return Ok(Vec::new()); } // Git-object connectivity requires a local git repo and gix-fsck. @@ -1757,6 +1780,9 @@ mod verification_tests; #[cfg(test)] mod capsule_history_tests; +#[cfg(test)] +mod pointer_data_tests; + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { diff --git a/crab/src/cmd/fsck_store/pointer_data_tests.rs b/crab/src/cmd/fsck_store/pointer_data_tests.rs new file mode 100644 index 000000000..051bc6562 --- /dev/null +++ b/crab/src/cmd/fsck_store/pointer_data_tests.rs @@ -0,0 +1,226 @@ +#![expect(clippy::unwrap_used, reason = "test assertions")] + +use super::*; +use bytes::Bytes; +use crab_metadata::capsule_protocol::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, + CapsuleTransaction, CapsuleVisibilityDelta, FileCatalogEntry, PointerCatalog, + ShardCatalogEntry, +}; +use crab_metadata::git_visibility::GitVisibilityEdit; +use sha2::{Digest, Sha256}; + +async fn publish_pointer( + layout: &crab_storage::StoreLayout, + root: crab_metadata::capsule_protocol::RootSnapshot, + body: &[u8], + delta: &PointerCatalog, +) -> crab_metadata::capsule_protocol::RootSnapshot { + let kind = gix_object::Kind::Blob; + let oid = crab_remote::objects::object_id(kind, body) + .unwrap() + .to_string(); + let mut bytes = Vec::new(); + crab_git::pack_writer::write_pack( + &mut bytes, + std::iter::once(Ok((kind, body.len() as u64, body))), + 1024 * 1024, + || false, + ) + .unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let path = scratch.path().join("source.pack"); + std::fs::write(&path, &bytes).unwrap(); + let indexed = crab_git::pack::install_pack_file_from_path( + &scratch.path().join("indexed"), + &path, + blake3::hash(&bytes).to_hex().as_ref(), + 1024 * 1024, + true, + ) + .unwrap(); + let checksum = gix_hash::ObjectId::from_hex(indexed.git_sha1.as_bytes()).unwrap(); + let kinds = crab_git::pack_locator::encode_pack_kind_metadata(checksum, &[kind]).unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from(bytes), + Bytes::from(std::fs::read(indexed.idx_path).unwrap()), + Bytes::from(std::fs::read(indexed.rev_path).unwrap()), + Bytes::from(kinds), + indexed.git_sha1, + 1, + ) + .unwrap(); + let name = "refs/tags/payload"; + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new(name, None, Some(oid.clone()), None)], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + name.to_owned(), + GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid]), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![pack], + vec![ + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + ), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + delta.encode_delta().unwrap(), + ), + ], + ) + .unwrap(); + crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + .await + .unwrap() +} + +async fn pointer_checker(case: &str) -> StoreChecker { + let (store, prefix, root, xorb_hash, shard_hash) = + super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let mut pointer = Pointer { + file_hash: blake3::hash(&[0x61; 1024]).into(), + size: 1024, + shard_hint: None, + }; + let mut delta = PointerCatalog::new(); + match case { + "missing" => pointer.file_hash = [42; 32], + "size" => pointer.size += 1, + "hash" => { + let (body, _) = store + .get_with_etag_bounded(&router.shard_path(&shard_hash), MAX_FSCK_SHARD_BYTES) + .await + .unwrap(); + let reader = ShardReader::from_bytes(body, shard_hash); + let mut recipe = reader + .get_file_info(&MerkleHash::from(pointer.file_hash)) + .unwrap() + .unwrap(); + pointer.file_hash = [42; 32]; + recipe.metadata.file_hash = MerkleHash::from(pointer.file_hash); + let mut shard = crab_xet::shard::ShardWriter::new(); + shard + .add_xorb(Arc::new(reader.get_xorb_info(&xorb_hash).unwrap().unwrap())) + .unwrap(); + shard.add_file(recipe).unwrap(); + let (body, hash) = shard.finalize().unwrap(); + delta + .insert_shard( + hash.hex(), + ShardCatalogEntry::new(body.len() as u64, vec![xorb_hash.hex()]), + ) + .unwrap(); + delta + .insert_file( + MerkleHash::from(pointer.file_hash).hex(), + FileCatalogEntry::new(pointer.size, hash.hex()), + ) + .unwrap(); + store + .put(&router.shard_path(&hash), Bytes::from(body)) + .await + .unwrap(); + } + _ => {} + } + let body = if case.starts_with("lfs-") { + let lfs = crab_git::lfs_pointer::LfsPointer { + oid: Sha256::digest(b"hello").into(), + size: 5, + extensions: Vec::new(), + }; + if case != "lfs-missing" { + let path = crab_lfs::LfsObjectStore::new(layout.store().clone(), layout.repo_prefix()) + .object_path_for(&lfs.oid); + let body = if case == "lfs-corrupt" { + b"wrong" + } else { + b"hello" + }; + layout + .store() + .put(&path, Bytes::copy_from_slice(body)) + .await + .unwrap(); + } + lfs.serialize() + } else { + pointer.serialize() + }; + let root = publish_pointer(&layout, root, &body, &delta).await; + StoreChecker::for_capsule_repository(store, prefix, root) + .await + .unwrap() +} + +#[tokio::test] +async fn full_fsck_proves_reachable_crab_and_lfs_payloads() { + let mut outcomes = Vec::new(); + for case in [ + "valid", + "missing", + "size", + "hash", + "lfs-valid", + "lfs-missing", + "lfs-corrupt", + ] { + let checker = Box::pin(pointer_checker(case)).await; + let (_, outcome) = crate::cmd::fsck::run_fsck( + &crate::cmd::fsck::FsckArgs::default(), + &checker, + &crate::cmd::fsck::NullRepairer, + &CancellationToken::new(), + Duration::from_hours(1), + None, + ) + .await + .unwrap(); + outcomes.push((case, outcome.errors == 0)); + } + assert_eq!( + outcomes, + vec![ + ("valid", true), + ("missing", false), + ("size", false), + ("hash", false), + ("lfs-valid", true), + ("lfs-missing", false), + ("lfs-corrupt", false) + ] + ); +} + +#[tokio::test] +async fn fsck_retains_one_git_database_through_data_checks_then_releases_it() { + let checker = Box::pin(pointer_checker("valid")).await; + checker.check_git_objects().await.unwrap(); + let state = checker.capsule.as_ref().unwrap(); + let original = state.git_workspace.get().unwrap().path().to_owned(); + let clean = checker.check_data_chain().await.unwrap().is_empty(); + let reused = state.git_workspace.get().unwrap().path() == original; + drop(checker); + assert!(clean && reused && !original.exists()); +} + +#[tokio::test] +async fn data_checks_prove_pointer_coverage_without_a_prior_git_phase() { + let checker = Box::pin(pointer_checker("missing")).await; + assert!(matches!(checker.check_data_chain().await, + Err(CrabError::CorruptObject { reason, .. }) if reason.contains("absent from the catalog") + )); +} From 38fce67aa4b92e2c673c437f9a5b7eacd73b7485 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:53:08 -0700 Subject: [PATCH 089/127] perf(staging): bound residual decoding --- crab/src/git/push.rs | 2 +- crates/crab-staging/README.md | 11 + crates/crab-staging/src/coalesced_read.rs | 332 +++++++++++++++++ crates/crab-staging/src/lib.rs | 411 ++++++++++++---------- 4 files changed, 560 insertions(+), 196 deletions(-) create mode 100644 crates/crab-staging/src/coalesced_read.rs diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index 5e46f30aa..2a2381029 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -11738,7 +11738,7 @@ impl PushPipeline { } while let Some(payloads) = read_plan - .read_next(batch_size, XORB_PACK_READ_PAYLOAD_LIMIT) + .read_next_bounded(batch_size, XORB_PACK_READ_PAYLOAD_LIMIT) .await? { check_cancelled(&self.cancel)?; diff --git a/crates/crab-staging/README.md b/crates/crab-staging/README.md index 7b954a0dc..4010dacae 100644 --- a/crates/crab-staging/README.md +++ b/crates/crab-staging/README.md @@ -101,6 +101,17 @@ Drop clears the reader marker and discards an uncommitted snapshot when no sibling reader prevents cleanup. Deferred journals remain for exclusive maintenance; native and uncaptured leases remain owned throughout. +The coalesced residual reader uses `CoalescedChunkReadPlan::read_next_bounded` after all +selected requests are appended. One prepared source is opened and fully verified +once, then retained as a format-bounded serialized body while decoded batches obey +the caller's count and byte limits. A single oversized chunk may exceed the byte +target to make progress; its format limit still applies. Returned chunks own their +allocations. Source descriptors are pinned before worker admission, and cancelling +a pending read preserves its result for the next call. The public `read_next` +method retains the complete-source behavior shipped in v1.2.4; it shares the same +source and chunk verification. Canonical callers should migrate to the bounded +method; removing the released method requires a breaking library release. + Native intent recovery may preserve caller-attested committed recipes under canonical history owners before rolling back an old Git index state. `preserve_publication_history` accepts only verified recipe roots belonging to diff --git a/crates/crab-staging/src/coalesced_read.rs b/crates/crab-staging/src/coalesced_read.rs new file mode 100644 index 000000000..18fdfffd9 --- /dev/null +++ b/crates/crab-staging/src/coalesced_read.rs @@ -0,0 +1,332 @@ +use std::fs::File; +use std::iter::Peekable; +use std::sync::Arc; +use std::vec::IntoIter; + +use super::{ + Bytes, IndexedCoalescedReadGroup, MerkleHash, Path, PreparedChunkLocator, ReaderPool, Result, + StagedChunkLocator, StagingAreaReadOnly, StagingError, StagingMetrics, XorbParser, + decode_prepared_chunk, load_prepared_source, lock_index, pin_segment_read_groups, push_plan, + read_located_staged_chunks_batch, read_prepared_staged_chunks_batch, +}; + +enum CoalescedChunkAuthority { + Segments(Vec<(u64, StagedChunkLocator)>), + Prepared(Vec<(u64, MerkleHash, PreparedChunkLocator)>), +} + +/// One bounded residual-read unit. Prepared units contain exactly one source xorb. +struct CoalescedChunkReadGroup { + authority: CoalescedChunkAuthority, +} + +/// Connection-local residual planner backed by SQLite temporary storage. +/// +/// Appends are bounded by the caller's recipe page. Reads remove one bounded +/// segment batch or one complete prepared source xorb at a time. +pub struct CoalescedChunkReadPlan<'a> { + staging: &'a StagingAreaReadOnly, + next_sequence: u64, + prepared: Option, + pending: Option>>, + buffered: Peekable>, +} + +impl<'a> CoalescedChunkReadPlan<'a> { + pub(super) fn new(staging: &'a StagingAreaReadOnly) -> Result { + lock_index(&staging.index)?.begin_coalesced_read_plan()?; + Ok(Self { + staging, + next_sequence: 0, + prepared: None, + pending: None, + buffered: Vec::new().into_iter().peekable(), + }) + } + + /// Append a bounded request page with an opaque caller context per chunk. + pub fn append(&mut self, requests: &[(MerkleHash, u64, u64)]) -> Result<()> { + let mut indexed = Vec::with_capacity(requests.len()); + for (hash, size, context) in requests { + indexed.push((self.next_sequence, <[u8; 32]>::from(*hash), *size, *context)); + self.next_sequence = self.next_sequence.checked_add(1).ok_or_else(|| { + StagingError::StagingCorrupt("residual request sequence overflow".to_owned()) + })?; + } + lock_index(&self.staging.index)?.append_coalesced_read_requests(&indexed) + } + + /// Read one segment batch or complete prepared source. + /// + /// This preserves the complete-source API shipped in v1.2.4. New callers + /// should use `read_next_bounded`; removing this method requires a breaking release. + pub async fn read_next( + &self, + max_segment_chunks: usize, + max_segment_bytes: u64, + ) -> Result>> { + // The complete-source method is the v1.2.4 public contract. Mixing it + // with a partially consumed bounded source would silently skip chunks. + if self.prepared.is_some() || self.pending.is_some() || self.buffered.len() > 0 { + return Err(StagingError::StagingCorrupt( + "bounded read is not drained".to_owned(), + )); + } + let group = lock_index(&self.staging.index)? + .take_coalesced_read_group(max_segment_chunks, max_segment_bytes)? + .map(|group| match group { + IndexedCoalescedReadGroup::Segments(chunks) => CoalescedChunkReadGroup { + authority: CoalescedChunkAuthority::Segments( + chunks + .into_iter() + .map(|(context, chunk)| (context, StagedChunkLocator::from(chunk))) + .collect(), + ), + }, + IndexedCoalescedReadGroup::Prepared(chunks) => CoalescedChunkReadGroup { + authority: CoalescedChunkAuthority::Prepared( + chunks + .into_iter() + .map(|(context, hash, locator)| { + (context, MerkleHash::from(hash), locator) + }) + .collect(), + ), + }, + }); + match group { + Some(group) => read_coalesced_chunk_group( + &self.staging.root, + &self.staging.readers, + self.staging.metrics.as_deref(), + group, + ) + .await + .map(Some), + None => Ok(None), + } + } + /// Read a bounded decoded batch while retaining at most one serialized source. + /// + /// One oversized chunk may exceed the byte target to ensure progress; the + /// xorb format bounds that chunk. Finish appending requests before reading + /// to coalesce every occurrence of one source into one verified open. + /// Retain selected recipe leases or a push snapshot until reading completes. + /// Cancelling a pending call keeps its worker result for the next call. + pub async fn read_next_bounded( + &mut self, + max_chunks: usize, + max_bytes: u64, + ) -> Result>> { + if self.buffered.len() == 0 { + if self.pending.is_none() { + let task = if let Some(cursor) = self.prepared.take() { + tokio::task::spawn_blocking(move || cursor.batch(max_chunks, max_bytes)) + } else { + let index = lock_index(&self.staging.index)?; + let Some(group) = index.take_coalesced_read_group(max_chunks, max_bytes)? + else { + return Ok(None); + }; + let metrics = self.staging.metrics.clone(); + match group { + IndexedCoalescedReadGroup::Segments(chunks) => { + let locators = chunks + .iter() + .map(|(_, chunk)| StagedChunkLocator::from(*chunk)) + .collect::>(); + let pinned = pin_segment_read_groups(&self.staging.readers, &locators)?; + tokio::spawn(async move { + let payloads = read_located_staged_chunks_batch( + pinned, + metrics.as_deref(), + &locators, + ) + .await?; + let chunks = chunks + .into_iter() + .zip(payloads) + .map(|((context, chunk), (_, data))| { + (context, MerkleHash::from(chunk.chunk_hash), data) + }) + .collect(); + Ok(DecodedBatch { next: None, chunks }) + }) + } + IndexedCoalescedReadGroup::Prepared(chunks) => { + let (_, _, first) = chunks.first().ok_or_else(|| { + StagingError::StagingCorrupt("empty prepared read group".to_owned()) + })?; + let identity = (first.xorb_hash, first.payload_hash, first.xorb_bytes); + // Pin before releasing locator ownership or queuing the + // worker; retirement cannot invalidate this descriptor. + let file = File::open(push_plan::prepared_xorb_path( + &self.staging.root, + &MerkleHash::from(identity.0), + ))?; + tokio::task::spawn_blocking(move || { + let activity = PreparedReadActivity::new(metrics); + let parser = load_prepared_source(file, identity)?; + if let Some(metrics) = activity.0.as_deref() { + metrics.add_staging_bytes_read(identity.2); + metrics.add_prepared_source_xorb_bytes_read(identity.2); + } + PreparedCursor { + parser, + chunks: chunks + .into_iter() + .map(|(context, hash, locator)| { + (context, MerkleHash::from(hash), locator) + }) + .collect::>() + .into_iter() + .peekable(), + } + .batch(max_chunks, max_bytes) + }) + } + } + }; + // Await the stored handle by reference. Cancelling this read keeps + // its owned worker and result available to the next call. + self.pending = Some(task); + } + let pending = self.pending.as_mut().ok_or_else(|| { + StagingError::Internal("bounded read worker disappeared".to_owned()) + })?; + let result = pending.await; + self.pending = None; + let batch = result.map_err(std::io::Error::other)??; + self.prepared = batch.next; + self.buffered = batch.chunks.into_iter().peekable(); + } + // A cancelled call may have queued a larger batch. Bound the returned + // prefix using this call's limits and retain the remaining owned bytes. + let mut out = Vec::new(); + let mut bytes = 0_u64; + while let Some(chunk) = self.buffered.next_if(|(_, _, data)| { + out.is_empty() + || (out.len() < max_chunks.max(1) + && bytes.saturating_add(data.len() as u64) <= max_bytes.max(1)) + }) { + bytes = bytes.saturating_add(chunk.2.len() as u64); + out.push(chunk); + } + Ok(Some(out)) + } +} + +impl Drop for CoalescedChunkReadPlan<'_> { + fn drop(&mut self) { + if let Ok(index) = lock_index(&self.staging.index) { + let _ = index.clear_coalesced_read_plan(); + } + } +} + +struct DecodedBatch { + next: Option, + chunks: Vec<(u64, MerkleHash, Bytes)>, +} + +// The worker owns activity accounting, so dropping an awaiting future cannot +// leave a reader permanently active after its pinned source is released. +struct PreparedReadActivity(Option>); + +impl PreparedReadActivity { + fn new(metrics: Option>) -> Self { + if let Some(metrics) = metrics.as_deref() { + metrics.inc_prepared_source_xorb_open(); + metrics.prepared_source_reader_started(); + } + Self(metrics) + } +} + +impl Drop for PreparedReadActivity { + fn drop(&mut self) { + if let Some(metrics) = self.0.as_deref() { + metrics.prepared_source_reader_finished(); + } + } +} + +struct PreparedCursor { + parser: XorbParser, + chunks: Peekable>, +} + +impl PreparedCursor { + fn batch(mut self, max_chunks: usize, max_bytes: u64) -> Result { + let mut out = Vec::new(); + let mut bytes = 0_u64; + while let Some((context, hash, locator)) = self.chunks.next_if(|(_, _, locator)| { + out.is_empty() + || (out.len() < max_chunks.max(1) + && bytes.saturating_add(u64::from(locator.size)) <= max_bytes.max(1)) + }) { + let data = + decode_prepared_chunk(&self.parser, hash, locator.chunk_index, locator.size)?; + bytes = bytes.saturating_add(u64::from(locator.size)); + out.push((context, hash, data)); + } + let next = self.chunks.peek().is_some().then_some(self); + Ok(DecodedBatch { next, chunks: out }) + } +} + +async fn read_coalesced_chunk_group( + root: &Path, + readers: &ReaderPool, + metrics: Option<&dyn StagingMetrics>, + group: CoalescedChunkReadGroup, +) -> Result> { + match group.authority { + CoalescedChunkAuthority::Segments(chunks) => { + let locators = chunks + .iter() + .map(|(_, locator)| *locator) + .collect::>(); + let payloads = read_located_staged_chunks_batch( + pin_segment_read_groups(readers, &locators)?, + metrics, + &locators, + ) + .await?; + Ok(chunks + .into_iter() + .zip(payloads) + .map(|((position, locator), (_, payload))| (position, locator.hash, payload)) + .collect()) + } + CoalescedChunkAuthority::Prepared(chunks) => { + let contexts = chunks + .iter() + .map(|(context, _, _)| *context) + .collect::>(); + let indexed = chunks + .into_iter() + .enumerate() + .map(|(position, (_, hash, locator))| (position, hash, locator)) + .collect::>(); + Ok(read_prepared_staged_chunks_batch(root, metrics, &indexed) + .await? + .into_iter() + .map(|(position, payload)| { + indexed + .get(position) + .and_then(|(_, hash, _)| { + contexts + .get(position) + .map(|context| (*context, *hash, payload)) + }) + .ok_or_else(|| { + StagingError::Internal( + "prepared read returned an unknown position".to_owned(), + ) + }) + }) + .collect::>>()?) + } + } +} diff --git a/crates/crab-staging/src/lib.rs b/crates/crab-staging/src/lib.rs index 025bbbc30..564ee280d 100644 --- a/crates/crab-staging/src/lib.rs +++ b/crates/crab-staging/src/lib.rs @@ -38,6 +38,7 @@ //! `HashMap` lookups, never across `.await`. mod add_push_plan; +mod coalesced_read; pub mod config; pub mod error; mod index; @@ -67,6 +68,7 @@ mod prop_orphan_sweep_idempotence; #[path = "../tests/unit/prop_torn_tail_recovery.rs"] mod prop_torn_tail_recovery; +pub use coalesced_read::CoalescedChunkReadPlan; pub use config::StagingConfig; pub use error::{Result, StagingError}; pub use metrics::StagingMetrics; @@ -1367,88 +1369,6 @@ pub(crate) struct StagedChunkLocator { pub locator: ChunkLocator, } -enum CoalescedChunkAuthority { - Segments(Vec<(u64, StagedChunkLocator)>), - Prepared(Vec<(u64, MerkleHash, PreparedChunkLocator)>), -} - -/// One bounded residual-read unit. Prepared units contain exactly one source xorb. -struct CoalescedChunkReadGroup { - authority: CoalescedChunkAuthority, -} - -/// Connection-local residual planner backed by SQLite temporary storage. -/// -/// Appends are bounded by the caller's recipe page. Reads remove one bounded -/// segment batch or one complete prepared source xorb at a time. -pub struct CoalescedChunkReadPlan<'a> { - staging: &'a StagingAreaReadOnly, - next_sequence: u64, -} - -impl CoalescedChunkReadPlan<'_> { - /// Append a bounded request page with an opaque caller context per chunk. - pub fn append(&mut self, requests: &[(MerkleHash, u64, u64)]) -> Result<()> { - let mut indexed = Vec::with_capacity(requests.len()); - for (hash, size, context) in requests { - indexed.push((self.next_sequence, <[u8; 32]>::from(*hash), *size, *context)); - self.next_sequence = self.next_sequence.checked_add(1).ok_or_else(|| { - StagingError::StagingCorrupt("residual request sequence overflow".to_owned()) - })?; - } - lock_index(&self.staging.index)?.append_coalesced_read_requests(&indexed) - } - - /// Read and remove the next bounded residual group. - pub async fn read_next( - &self, - max_segment_chunks: usize, - max_segment_bytes: u64, - ) -> Result>> { - let group = lock_index(&self.staging.index)? - .take_coalesced_read_group(max_segment_chunks, max_segment_bytes)? - .map(|group| match group { - IndexedCoalescedReadGroup::Segments(chunks) => CoalescedChunkReadGroup { - authority: CoalescedChunkAuthority::Segments( - chunks - .into_iter() - .map(|(context, chunk)| (context, StagedChunkLocator::from(chunk))) - .collect(), - ), - }, - IndexedCoalescedReadGroup::Prepared(chunks) => CoalescedChunkReadGroup { - authority: CoalescedChunkAuthority::Prepared( - chunks - .into_iter() - .map(|(context, hash, locator)| { - (context, MerkleHash::from(hash), locator) - }) - .collect(), - ), - }, - }); - match group { - Some(group) => read_coalesced_chunk_group( - &self.staging.root, - &self.staging.readers, - self.staging.metrics.as_deref(), - group, - ) - .await - .map(Some), - None => Ok(None), - } - } -} - -impl Drop for CoalescedChunkReadPlan<'_> { - fn drop(&mut self) { - if let Ok(index) = lock_index(&self.staging.index) { - let _ = index.clear_coalesced_read_plan(); - } - } -} - struct IndexedPreparedXorbStatsRef { payload_hash: [u8; 32], bytes: u64, @@ -1539,6 +1459,74 @@ async fn read_staged_chunks_batch( type PreparedXorbIdentity = ([u8; 32], [u8; 32], u64); +fn load_prepared_source(file: File, identity: PreparedXorbIdentity) -> Result { + let (xorb_hash, payload_hash, expected_bytes) = identity; + if expected_bytes > MAX_XORB_SIZE as u64 { + return Err(StagingError::StagingCorrupt(format!( + "prepared xorb {} declares {expected_bytes} bytes above the {MAX_XORB_SIZE}-byte format bound", + MerkleHash::from(xorb_hash).hex() + ))); + } + let file_bytes = file.metadata()?.len(); + if file_bytes != expected_bytes { + return Err(StagingError::StagingCorrupt(format!( + "prepared xorb {} payload changed: expected {expected_bytes} bytes, found {file_bytes} bytes", + MerkleHash::from(xorb_hash).hex() + ))); + } + let capacity = usize::try_from(expected_bytes).map_err(|_| { + StagingError::StagingCorrupt(format!( + "prepared xorb {} size does not fit memory", + MerkleHash::from(xorb_hash).hex() + )) + })?; + let mut payload = Vec::with_capacity(capacity); + file.take(expected_bytes.saturating_add(1)) + .read_to_end(&mut payload)?; + let payload = Bytes::from(payload); + let actual_payload_hash = blake3::hash(&payload); + let size_matches = u64::try_from(payload.len()) == Ok(expected_bytes); + let hash_matches = *actual_payload_hash.as_bytes() == payload_hash; + if !size_matches || !hash_matches { + return Err(StagingError::StagingCorrupt(format!( + "prepared xorb {} payload changed: expected {} bytes/{}, found {} bytes/{} (size_matches={size_matches}, hash_matches={hash_matches})", + MerkleHash::from(xorb_hash).hex(), + expected_bytes, + MerkleHash::from(payload_hash).hex(), + payload.len(), + actual_payload_hash.to_hex() + ))); + } + let parser = XorbParser::parse(payload)?; + if parser.hash() != MerkleHash::from(xorb_hash) { + return Err(StagingError::StagingCorrupt(format!( + "prepared xorb {} metadata hash changed", + MerkleHash::from(xorb_hash).hex() + ))); + } + parser.verify_payload_digest()?; + + Ok(parser) +} + +fn decode_prepared_chunk( + parser: &XorbParser, + expected_hash: MerkleHash, + chunk_index: u32, + expected_size: u32, +) -> Result { + let chunk = parser.get_chunk(chunk_index)?; + if chunk.hash != expected_hash || chunk.data.len() != expected_size as usize { + return Err(StagingError::StagingCorrupt(format!( + "prepared xorb chunk {} does not match its indexed placement", + expected_hash.hex() + ))); + } + // Raw chunks can share the encoded body. Detach each result so a consumed + // batch cannot retain old source bodies while the caller packs another one. + Ok(Bytes::copy_from_slice(&chunk.data)) +} + async fn read_prepared_staged_chunks_batch( root: &Path, metrics: Option<&dyn StagingMetrics>, @@ -1568,59 +1556,17 @@ async fn read_prepared_staged_chunks_batch( MerkleHash::from(xorb_hash).hex() ))); } - let file = File::open(&path)?; - let file_bytes = file.metadata()?.len(); - if file_bytes != expected_bytes { - return Err(StagingError::StagingCorrupt(format!( - "prepared xorb {} payload changed: expected {expected_bytes} bytes, found {file_bytes} bytes", - MerkleHash::from(xorb_hash).hex() - ))); - } - let capacity = usize::try_from(expected_bytes).map_err(|_| { - StagingError::StagingCorrupt(format!( - "prepared xorb {} size does not fit memory", - MerkleHash::from(xorb_hash).hex() - )) - })?; - let mut payload = Vec::with_capacity(capacity); - file.take(expected_bytes.saturating_add(1)) - .read_to_end(&mut payload)?; - let payload = Bytes::from(payload); - let actual_payload_hash = blake3::hash(&payload); - let size_matches = u64::try_from(payload.len()) == Ok(expected_bytes); - let hash_matches = *actual_payload_hash.as_bytes() == payload_hash; - if !size_matches || !hash_matches { - return Err(StagingError::StagingCorrupt(format!( - "prepared xorb {} payload changed: expected {} bytes/{}, found {} bytes/{} (size_matches={size_matches}, hash_matches={hash_matches})", - MerkleHash::from(xorb_hash).hex(), - expected_bytes, - MerkleHash::from(payload_hash).hex(), - payload.len(), - actual_payload_hash.to_hex() - ))); - } - let parser = XorbParser::parse(payload)?; - if parser.hash() != MerkleHash::from(xorb_hash) { - return Err(StagingError::StagingCorrupt(format!( - "prepared xorb {} metadata hash changed", - MerkleHash::from(xorb_hash).hex() - ))); - } - parser.verify_payload_digest()?; + let parser = load_prepared_source( + File::open(&path)?, + (xorb_hash, payload_hash, expected_bytes), + )?; let mut decoded = Vec::with_capacity(wanted.len()); for (position, expected_hash, chunk_index, expected_size) in wanted { - let chunk = parser.get_chunk(chunk_index)?; - if chunk.hash != expected_hash || chunk.data.len() != expected_size as usize { - return Err(StagingError::StagingCorrupt(format!( - "prepared xorb chunk {} does not match its indexed placement", - expected_hash.hex() - ))); - } - // Parser chunks are slices of the full serialized xorb. The - // caller can retain this batch while packing other sources, - // so detach each requested chunk before dropping the parser. - decoded.push((position, Bytes::copy_from_slice(&chunk.data))); + decoded.push(( + position, + decode_prepared_chunk(&parser, expected_hash, chunk_index, expected_size)?, + )); } Ok::<_, StagingError>((expected_bytes, decoded)) }); @@ -1640,62 +1586,6 @@ async fn read_prepared_staged_chunks_batch( Ok(out) } -async fn read_coalesced_chunk_group( - root: &Path, - readers: &ReaderPool, - metrics: Option<&dyn StagingMetrics>, - group: CoalescedChunkReadGroup, -) -> Result> { - match group.authority { - CoalescedChunkAuthority::Segments(chunks) => { - let locators = chunks - .iter() - .map(|(_, locator)| *locator) - .collect::>(); - let payloads = read_located_staged_chunks_batch( - pin_segment_read_groups(readers, &locators)?, - metrics, - &locators, - ) - .await?; - Ok(chunks - .into_iter() - .zip(payloads) - .map(|((position, locator), (_, payload))| (position, locator.hash, payload)) - .collect()) - } - CoalescedChunkAuthority::Prepared(chunks) => { - let contexts = chunks - .iter() - .map(|(context, _, _)| *context) - .collect::>(); - let indexed = chunks - .into_iter() - .enumerate() - .map(|(position, (_, hash, locator))| (position, hash, locator)) - .collect::>(); - Ok(read_prepared_staged_chunks_batch(root, metrics, &indexed) - .await? - .into_iter() - .map(|(position, payload)| { - indexed - .get(position) - .and_then(|(_, hash, _)| { - contexts - .get(position) - .map(|context| (*context, *hash, payload)) - }) - .ok_or_else(|| { - StagingError::Internal( - "prepared read returned an unknown position".to_owned(), - ) - }) - }) - .collect::>>()?) - } - } -} - async fn verify_pending_recipe_payload( root: &Path, index: &Mutex, @@ -3927,11 +3817,7 @@ impl StagingAreaReadOnly { /// Begin one disk-backed residual plan for bounded recipe-page appends. pub fn begin_coalesced_chunk_read_plan(&self) -> Result> { - lock_index(&self.index)?.begin_coalesced_read_plan()?; - Ok(CoalescedChunkReadPlan { - staging: self, - next_sequence: 0, - }) + CoalescedChunkReadPlan::new(self) } /// Check if a chunk exists in staging. @@ -5464,8 +5350,7 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn prepared_residual_plan_reads_each_unique_source_once() { + async fn prepared_residual_fixture() -> (tempfile::TempDir, Vec<(MerkleHash, Vec)>, Bytes) { let chunks = (0..8) .map(|index| make_chunk(100 + index, 4096)) .collect::>(); @@ -5536,6 +5421,16 @@ mod tests { .expect("remove pending segment authority"); } + (tmp, chunks, xorb_bytes) + } + + #[tokio::test(flavor = "multi_thread")] + async fn prepared_residual_plan_reads_each_unique_source_once() { + let (tmp, chunks, xorb_bytes) = prepared_residual_fixture().await; + let chunk_pairs = chunks + .iter() + .map(|(hash, data)| (*hash, data.len() as u64)) + .collect::>(); let metrics = Arc::new(CountingMetrics::default()); let mut staging = StagingAreaReadOnly::open(tmp.path().to_path_buf()) .await @@ -5574,6 +5469,132 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn bounded_prepared_reads_detach_batches_and_reopen_after_drop() { + let (tmp, chunks, xorb_bytes) = prepared_residual_fixture().await; + let metrics = Arc::new(CountingMetrics::default()); + let mut staging = StagingAreaReadOnly::open(tmp.path().to_owned()) + .await + .unwrap(); + staging.set_metrics(metrics.clone()); + for stop_early in [true, false] { + let mut plan = staging.begin_coalesced_chunk_read_plan().unwrap(); + let requests = chunks + .iter() + .map(|(hash, data)| (*hash, data.len() as u64, 7)) + .collect::>(); + plan.append(&requests).unwrap(); + let mut actual = Vec::new(); + let mut batch_sizes = Vec::new(); + while let Some(batch) = plan.read_next_bounded(2, 8192).await.unwrap() { + batch_sizes.push(batch.len()); + for (context, hash, data) in batch { + assert_eq!(context, 7); + assert!( + data.is_unique(), + "decoded batches cannot retain the source allocation" + ); + actual.push((hash, data.to_vec())); + } + if stop_early { + break; + } + } + if !stop_early { + assert_eq!((batch_sizes, actual), (vec![2; 4], chunks.clone())); + } + } + assert_eq!( + metrics.prepared_source_counts(), + (2, 2 * xorb_bytes.len() as u64, 1) + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn bounded_prepared_reads_reject_changed_payload_before_returning_chunks() { + let (tmp, chunks, _) = prepared_residual_fixture().await; + let staging = StagingAreaReadOnly::open(tmp.path().to_owned()) + .await + .unwrap(); + let mut plan = staging.begin_coalesced_chunk_read_plan().unwrap(); + plan.append(&[(chunks[0].0, chunks[0].1.len() as u64, 0)]) + .unwrap(); + let files = std::fs::read_dir(tmp.path().join("push-plans/payloads")) + .unwrap() + .flat_map(|directory| std::fs::read_dir(directory.unwrap().path()).unwrap()) + .map(|entry| entry.unwrap().path()) + .collect::>(); + assert_eq!(files.len(), 1); + let mut bytes = std::fs::read(&files[0]).unwrap(); + bytes[0] ^= 1; + std::fs::write(&files[0], bytes).unwrap(); + assert!(matches!( + plan.read_next_bounded(2, 8192).await, + Err(StagingError::StagingCorrupt(_)) + )); + } + + #[test] + fn cancelled_bounded_read_resumes_its_queued_source() { + use std::future::{Future, poll_fn}; + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .max_blocking_threads(1) + .enable_all() + .build() + .unwrap(); + let (tmp, chunks, _) = runtime.block_on(prepared_residual_fixture()); + let staging = runtime + .block_on(StagingAreaReadOnly::open(tmp.path().to_owned())) + .unwrap(); + let mut plan = staging.begin_coalesced_chunk_read_plan().unwrap(); + let requests = chunks + .iter() + .map(|(hash, data)| (*hash, data.len() as u64, 0)) + .collect::>(); + plan.append(&requests).unwrap(); + let (release, wait) = std::sync::mpsc::channel(); + let (ready, started) = std::sync::mpsc::channel(); + let blocker = runtime.handle().spawn_blocking(move || { + ready.send(()).unwrap(); + let _ = wait.recv(); + }); + started.recv().unwrap(); + runtime.block_on(async { + // Keep verification queued so cancellation occurs after source + // ownership is captured and before its worker can return a batch. + let mut read = Box::pin(plan.read_next_bounded(2, 8192)); + poll_fn(|cx| { + assert!(read.as_mut().poll(cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + drop(read); + #[cfg(unix)] + for directory in std::fs::read_dir(tmp.path().join("push-plans/payloads")).unwrap() { + for entry in std::fs::read_dir(directory.unwrap().path()).unwrap() { + std::fs::remove_file(entry.unwrap().path()).unwrap(); + } + } + release.send(()).unwrap(); + let mut actual = Vec::new(); + while let Some(batch) = plan.read_next_bounded(1, 4096).await.unwrap() { + assert_eq!( + batch.len(), + 1, + "resuming with a smaller budget must bound the queued result" + ); + actual.extend( + batch + .into_iter() + .map(|(_, hash, data)| (hash, data.to_vec())), + ); + } + assert_eq!(actual, chunks); + blocker.await.unwrap(); + }); + } + fn insert_extra_prepared_xorb_row( root: &std::path::Path, _file_hash: MerkleHash, From 6e9922e79242e63fbf0aabd963e2bbc9fccabe27 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:56:26 -0700 Subject: [PATCH 090/127] fix(push): repack mixed prepared xorbs --- crab/src/git/xet_publication.rs | 141 ++++++++------- .../xet_publication/tests/residual_tests.rs | 160 ++++++++++++++++++ 2 files changed, 236 insertions(+), 65 deletions(-) create mode 100644 crab/src/git/xet_publication/tests/residual_tests.rs diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 266fa1d6f..7e2f5e468 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -140,7 +140,6 @@ pub(crate) async fn prepare_delta( let mut xorb_bytes_uploaded = 0_u64; let mut prepared_files = Vec::with_capacity(unresolved.len()); let mut prepared_candidates = Vec::::new(); - let mut candidate_indices = HashMap::::new(); let mut staged_candidates = HashMap::new(); for (file_hash, file_size) in unresolved { @@ -311,7 +310,6 @@ pub(crate) async fn prepare_delta( xorb_entries.insert(hash, entry); origins.insert(hash, origin); } - drop(requested); let mut external_candidates = HashMap::>::new(); for (hash, candidate) in verified_staged.candidates { let xorb_ref = candidate.xorb_ref; @@ -378,13 +376,11 @@ pub(crate) async fn prepare_delta( } let mut candidate_chunks = placements.keys().copied().collect::>(); + let mut seen_candidates = HashMap::new(); for (_, _, _, plan) in &prepared_files { for planned in &plan.prepared_xorbs { let planned_hash = planned.hash()?; - if let Some(index) = candidate_indices.get(&planned_hash).copied() { - let existing = prepared_candidates.get(index).ok_or_else(|| { - CrabError::Internal("prepared xorb candidate index is invalid".to_owned()) - })?; + if let Some(existing) = seen_candidates.insert(planned_hash, planned) { if !planned_xorbs_match(existing, planned) { return Err(CrabError::StagingCorrupt(format!( "prepared xorb {} has conflicting plans", @@ -401,10 +397,13 @@ pub(crate) async fn prepare_delta( .iter() .map(crab_staging::push_plan::PlannedPlacement::to_placement) .collect::>>()?; - if planned_placements - .iter() - .all(|placement| candidate_chunks.contains(&placement.chunk_hash)) - { + // Direct adoption is only for wholly new outgoing payloads. A + // single missing chunk must not upload an otherwise reused xorb; + // residual packing below reads only the still-needed chunks. + if !planned_placements.iter().all(|placement| { + requested.contains(placement.chunk_hash.hex().as_str()) + && !candidate_chunks.contains(&placement.chunk_hash) + }) { continue; } candidate_chunks.extend( @@ -412,11 +411,12 @@ pub(crate) async fn prepare_delta( .iter() .map(|placement| placement.chunk_hash), ); - candidate_indices.insert(planned_hash, prepared_candidates.len()); prepared_candidates.push(planned.clone()); } } + drop(requested); + let uploads = futures_util::stream::iter(prepared_candidates.into_iter().enumerate().map( |(ordinal, planned)| async move { check_cancelled(cancel)?; @@ -484,68 +484,77 @@ pub(crate) async fn prepare_delta( xorb_entries.insert(planned_hash, entry); } - for (file_ordinal, (file_hash, file_size, chunks, _)) in prepared_files.into_iter().enumerate() - { - check_cancelled(cancel)?; - let mut missing = Vec::new(); - let mut seen = HashSet::new(); - for (hash, _) in &chunks { - if !placements.contains_key(hash) && seen.insert(*hash) { - missing.push(*hash); + let mut missing = prepared_files + .iter() + .flat_map(|(_, _, chunks, _)| chunks.iter().map(|(hash, _)| *hash)) + .filter(|hash| !placements.contains_key(hash)) + .collect::>(); + if !missing.is_empty() { + // The same prepared body can supply several files. Gather the whole + // residual schedule before reading, so its verified source opens once + // rather than once per file or fixed-size request page. + let mut read_plan = staging.begin_coalesced_chunk_read_plan()?; + let mut requests = Vec::with_capacity(256); + for (ordinal, (_, _, chunks, _)) in prepared_files.iter().enumerate() { + let context = u64::try_from(ordinal) + .map_err(|_| CrabError::Internal("pointer file ordinal overflowed".to_owned()))?; + for (hash, size) in chunks { + if !missing.remove(hash) { + continue; + } + requests.push((*hash, *size, context)); + if requests.len() == 256 { + check_cancelled(cancel)?; + read_plan.append(&requests)?; + requests.clear(); + } } } - if !missing.is_empty() { - let mut builder = XorbBuilder::new(); - for batch in missing.chunks(256) { - let loaded = staging.get_chunks_batch(batch).await?; - let by_hash = loaded.into_iter().collect::>(); - for hash in batch { - let data = by_hash - .get(hash) - .cloned() - .ok_or_else(|| CrabError::ChunkNotFound { hash: hash.hex() })?; - builder.push( - &Chunk { hash: *hash, data }, - RunId(u64::try_from(file_ordinal).map_err(|_| { - CrabError::Internal("pointer file ordinal overflowed".to_owned()) - })?), - )?; - while let Some(result) = builder.take_completed() { - let uploaded_bytes = publish_built_xorb( - layout, - caching_store, - result, - &mut placements, - &mut xorb_entries, - &mut uploaded_xorbs, - &mut origins, - ) - .await?; - xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); - if let Some(metrics) = metrics { - metrics.add_bytes_uploaded(uploaded_bytes); - } + read_plan.append(&requests)?; + drop(missing); + let mut builder = XorbBuilder::new(); + while let Some(payloads) = read_plan.read_next(256, 16 * 1024 * 1024).await? { + check_cancelled(cancel)?; + for (context, hash, data) in payloads { + builder.push(&Chunk { hash, data }, RunId(context))?; + while let Some(result) = builder.take_completed() { + let uploaded_bytes = publish_built_xorb( + layout, + caching_store, + result, + &mut placements, + &mut xorb_entries, + &mut uploaded_xorbs, + &mut origins, + ) + .await?; + xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); + if let Some(metrics) = metrics { + metrics.add_bytes_uploaded(uploaded_bytes); } } } - for result in builder.finalize()? { - let uploaded_bytes = publish_built_xorb( - layout, - caching_store, - result, - &mut placements, - &mut xorb_entries, - &mut uploaded_xorbs, - &mut origins, - ) - .await?; - xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); - if let Some(metrics) = metrics { - metrics.add_bytes_uploaded(uploaded_bytes); - } + } + for result in builder.finalize()? { + let uploaded_bytes = publish_built_xorb( + layout, + caching_store, + result, + &mut placements, + &mut xorb_entries, + &mut uploaded_xorbs, + &mut origins, + ) + .await?; + xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); + if let Some(metrics) = metrics { + metrics.add_bytes_uploaded(uploaded_bytes); } } + } + for (file_hash, file_size, chunks, _) in prepared_files { + check_cancelled(cancel)?; let ordered_hashes = chunks.iter().map(|(hash, _)| *hash).collect::>(); let file_info = file_info_from_placements(file_hash, &ordered_hashes, &placements)?; let dependency_hashes = file_info @@ -1486,6 +1495,8 @@ mod tests { } } + mod residual_tests; + #[derive(Default)] struct UploadObserver { active: std::sync::atomic::AtomicUsize, diff --git a/crab/src/git/xet_publication/tests/residual_tests.rs b/crab/src/git/xet_publication/tests/residual_tests.rs new file mode 100644 index 000000000..c5286fe77 --- /dev/null +++ b/crab/src/git/xet_publication/tests/residual_tests.rs @@ -0,0 +1,160 @@ +use super::*; + +#[tokio::test(flavor = "multi_thread")] +async fn mixed_prepared_xorb_uploads_only_unresolved_chunks() { + use crab_staging::stream::{ + StreamStageProgress, StreamStageXorbBuilder, stage_reader_streaming, + }; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[derive(Default)] + struct Reads(AtomicU64); + impl crab_staging::StagingMetrics for Reads { + fn inc_prepared_source_xorb_open(&self) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + let root = tempfile::tempdir().unwrap(); + let staging = crab_staging::StagingArea::open(root.path().to_owned()) + .await + .unwrap(); + let mut bytes = vec![0; 32 * 1024 * 1024]; + blake3::Hasher::new() + .update(b"mixed prepared publication") + .finalize_xof() + .fill(&mut bytes); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(&bytes), + Path::new("model.bin"), + &staging, + StreamStageProgress { + xorb_builder: Some(StreamStageXorbBuilder::new(1, || { + XorbBuilder::with_policy(Arc::new(FixedCompression::new(CompressionScheme::None)) + as Arc) + })), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await + .unwrap(); + staging + .delegate_publication(&[(&staged.batch_id, Path::new("model.bin"))]) + .unwrap(); + staging.close().await.unwrap(); + let mut staging = StagingAreaReadOnly::open(root.path().to_owned()) + .await + .unwrap(); + let reads = Arc::new(Reads::default()); + staging.set_metrics(reads.clone()); + let staging = Arc::new(staging); + let plans = staging + .prepared_xorbs_for_recipes(&[MerkleHash::from(staged.recipe.hash())]) + .unwrap(); + let plans = &plans[&MerkleHash::from(staged.recipe.hash())]; + assert_eq!(plans.len(), 1); + let original = XorbParser::parse(Bytes::from( + std::fs::read(crab_staging::push_plan::prepared_xorb_path( + staging.root(), + &plans[0].hash().unwrap(), + )) + .unwrap(), + )) + .unwrap(); + let count = plans[0].placements.len(); + assert!( + count > 257, + "fixture must span the old 256-chunk read batches" + ); + // Exercise both a small edit and an almost entirely new recipe. The + // latter must not reopen the same prepared body for every read batch. + for committed_count in [count - 1, 1] { + let mut builder = + XorbBuilder::with_policy(Arc::new(FixedCompression::new(CompressionScheme::None)) + as Arc); + for index in 0..committed_count { + builder + .push(&original.get_chunk(index as u32).unwrap(), RunId(0)) + .unwrap(); + } + let mut results = builder.finalize().unwrap(); + assert_eq!(results.len(), 1); + let committed = results.remove(0); + let entry = catalog_xorb_entry( + committed.bytes.len() as u64, + blake3::hash(&committed.bytes).to_hex().to_string(), + &committed.placements, + ) + .unwrap(); + let mut catalog = PointerCatalog::new(); + catalog.insert_xorb(committed.hash.hex(), entry).unwrap(); + // Only the selected new shard is reconstructed below. This old + // binding supplies the authenticated reverse-chunk lookup fixture. + let shard = "a".repeat(64); + catalog + .insert_shard( + shard.clone(), + ShardCatalogEntry::new(1, vec![committed.hash.hex()]), + ) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(1, shard)) + .unwrap(); + let (layout, base) = publish_catalog(catalog.encode().unwrap()).await; + layout + .store() + .create_strict(&layout.xorb_path(&committed.hash), committed.bytes) + .await + .unwrap(); + reads.0.store(0, Ordering::Relaxed); + let pointer = Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + let (delta, stats) = prepare_delta( + &layout, + &base, + std::slice::from_ref(&pointer), + Some(&staging), + None, + None, + &Default::default(), + 1, + false, + &CancellationToken::new(), + ) + .await + .unwrap(); + crab_read::verify_catalog_file_recipe( + &layout, + &pointer, + &delta.files()[&MerkleHash::from(staged.file_hash).hex()], + &CancellationToken::new(), + ) + .await + .unwrap(); + let expected = plans[0] + .placements + .iter() + .skip(committed_count) + .map(|placement| placement.chunk_hash.clone()) + .collect::>(); + let actual = delta + .xorbs() + .values() + .flat_map(|entry| entry.chunks()) + .map(|chunk| chunk.hash().to_owned()) + .collect::>(); + eprintln!( + "committed={committed_count}/{count}, uploaded_bytes={}, prepared_reads={}", + stats.xorb_bytes_uploaded, + reads.0.load(Ordering::Relaxed) + ); + assert_eq!( + (actual, reads.0.load(Ordering::Relaxed)), + (expected, 1), + "publication must upload only missing chunks and open their source once" + ); + } +} From 4f06570d44650186c5936175788fa2767be7127e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 13:16:42 -0700 Subject: [PATCH 091/127] perf(push): offload residual xorb compression --- crab/docs/design/add.md | 8 +++++ crab/src/git/xet_publication.rs | 53 ++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index 16fbe9e6e..fd06d91bb 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -298,6 +298,14 @@ closed and asks the user to run `crab add` again. Recipe leases and push snapshots retain shared bodies until the final owner is released; restaging reclaims only globally unleased payloads. +Capsule publication adopts a complete prepared xorb only when every chunk is +selected for the push and still unresolved. Mixed sources are repacked using +only missing chunks, retaining committed placements for reused bytes. The shared +staging read plan groups residual requests across files by source, verifies each +prepared body once, and yields bounded decoded batches. Compression runs off the +async worker. Ordered recipe occurrences still produce complete file terms; +local recipe ownership and flush-before-pointer publication are unchanged. + There is no persisted per-file JSON plan or per-file payload copy. Runtime `FilePushPlan` values are derived from normalized recipe, remote, prepared, and segment rows for the push attempt. diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 7e2f5e468..42d2b3637 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -513,29 +513,46 @@ pub(crate) async fn prepare_delta( read_plan.append(&requests)?; drop(missing); let mut builder = XorbBuilder::new(); - while let Some(payloads) = read_plan.read_next(256, 16 * 1024 * 1024).await? { + while let Some(payloads) = read_plan.read_next_bounded(256, 16 * 1024 * 1024).await? { check_cancelled(cancel)?; - for (context, hash, data) in payloads { - builder.push(&Chunk { hash, data }, RunId(context))?; - while let Some(result) = builder.take_completed() { - let uploaded_bytes = publish_built_xorb( - layout, - caching_store, - result, - &mut placements, - &mut xorb_entries, - &mut uploaded_xorbs, - &mut origins, - ) - .await?; - xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); - if let Some(metrics) = metrics { - metrics.add_bytes_uploaded(uploaded_bytes); + // Compression runs on a bounded decoded batch so ref-lease and + // upload tasks remain schedulable while residual payloads are packed. + let packing_cancel = cancel.clone(); + let (next_builder, completed) = tokio::task::spawn_blocking(move || { + let mut completed = Vec::new(); + for (context, hash, data) in payloads { + check_cancelled(&packing_cancel)?; + builder.push(&Chunk { hash, data }, RunId(context))?; + while let Some(result) = builder.take_completed() { + completed.push(result); } } + Ok::<_, CrabError>((builder, completed)) + }) + .await + .map_err(std::io::Error::other)??; + builder = next_builder; + for result in completed { + let uploaded_bytes = publish_built_xorb( + layout, + caching_store, + result, + &mut placements, + &mut xorb_entries, + &mut uploaded_xorbs, + &mut origins, + ) + .await?; + xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); + if let Some(metrics) = metrics { + metrics.add_bytes_uploaded(uploaded_bytes); + } } } - for result in builder.finalize()? { + let finalized = tokio::task::spawn_blocking(move || builder.finalize()) + .await + .map_err(std::io::Error::other)??; + for result in finalized { let uploaded_bytes = publish_built_xorb( layout, caching_store, From 94c8df2380caa81ae33c34356a58fb23ca18de6e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:41:57 -0700 Subject: [PATCH 092/127] fix(storage): retry corrupt immutable readbacks Share bounded hash verification and its existing corruption retry with acknowledged immutable writes. Retry only the GET and retain both identity fields from the response whose bytes passed verification. Persistent corruption still fails after two reads; exact and logical writes issue one PUT. Keep checksum-qualified and occupied-key contracts unchanged. --- crates/crab-storage/README.md | 7 ++ crates/crab-storage/src/store.rs | 127 ++++++++++++++++++++++++------- 2 files changed, 108 insertions(+), 26 deletions(-) diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index f82e41126..c9726cc4d 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -52,6 +52,13 @@ observe another object version. Occupied-key bytes still require caller validati `Store::verify_bounded` combines that content-hash contract with pre-body size admission and streamed length checks. It uses one GET for a successful read; the limit applies to buffered encoded bytes, not the caller's decoded objects. +Readback-required immutable writes use that same bounded verifier and corruption +retry: one corrupt hash response permits one more GET, while persistent corruption +still fails. An acknowledged PUT is not replayed during verification. A successful +logical create carries the ETag and version from the hash-verified GET, including +when that response follows a corrupt read. Checksum-qualified writes still skip +readback, and occupied logical content addresses still return their body for +caller validation rather than requiring the candidate's physical encoding. The admission token cancels pending admission, response headers, streamed body reads and pending listings without retrying the cancellation. diff --git a/crates/crab-storage/src/store.rs b/crates/crab-storage/src/store.rs index edfbb3d96..4c9619df6 100644 --- a/crates/crab-storage/src/store.rs +++ b/crates/crab-storage/src/store.rs @@ -662,18 +662,8 @@ impl Store { return Ok(created); } let readback_path = self.write_path(path); - let (stored, _) = self.get_with_etag_bounded(&readback_path, maximum).await?; - let actual_hash = *blake3::hash(&stored).as_bytes(); - if actual_hash != expected_hash { - return Err(StorageError::CorruptObject { - path: path.to_string(), - reason: format!( - "expected blake3 {}, got {}", - hex_lower(&expected_hash), - hex_lower(&actual_hash) - ), - }); - } + self.verify_bounded(&readback_path, &expected_hash, maximum) + .await?; Ok(created) } @@ -746,18 +736,7 @@ impl Store { }; if self.immutable_write_verification == ImmutableWriteVerification::ReadbackRequired { let maximum = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - let (stored, observed_identity) = self.get_with_etag_bounded(path, maximum).await?; - let actual_hash = *blake3::hash(&stored).as_bytes(); - if actual_hash != expected_hash { - return Err(StorageError::CorruptObject { - path: path.to_string(), - reason: format!( - "expected blake3 {}, got {}", - hex_lower(&expected_hash), - hex_lower(&actual_hash) - ), - }); - } + let (_, observed_identity) = self.verified_body(path, &expected_hash, maximum).await?; // Bind proof to the object version whose bytes were verified. A // later HEAD may observe a replacement with a different encoding. identity = observed_identity; @@ -1486,13 +1465,27 @@ impl Store { expected_hash: &[u8; 32], max_bytes: u64, ) -> Result { + self.verified_body(path, expected_hash, max_bytes) + .await + .map(|(bytes, _)| bytes) + } + + // Readback and standalone verification share the corruption retry. Replay + // only the GET: an acknowledged immutable PUT must not be repeated, and + // its receipt must name the response whose bytes passed verification. + async fn verified_body( + &self, + path: &Path, + expected_hash: &[u8; 32], + max_bytes: u64, + ) -> Result<(Bytes, ETag)> { retry(&self.retry, || { let path = path.clone(); async move { - let (bytes, _etag) = self.get_with_etag_bounded(&path, max_bytes).await?; + let (bytes, identity) = self.get_with_etag_bounded(&path, max_bytes).await?; let actual = *blake3::hash(&bytes).as_bytes(); if actual == *expected_hash { - Ok(bytes) + Ok((bytes, identity)) } else { Err(StorageError::CorruptObject { path: path.to_string(), @@ -3767,6 +3760,7 @@ mod tests { BodyError, Range(Range), DistinctIdentities, + CorruptReads { bad_reads: u64, attempts: AtomicU64 }, } #[derive(Debug)] @@ -3853,6 +3847,20 @@ mod tests { response.meta.e_tag = Some("get-etag".to_owned()); response.meta.version = Some("get-version".to_owned()); } + ReadFault::CorruptReads { + bad_reads, + attempts, + } => { + let attempt = attempts.fetch_add(1, Ordering::Relaxed) + 1; + response.meta.e_tag = Some(format!("get-{attempt}-etag")); + response.meta.version = Some(format!("get-{attempt}-version")); + if attempt <= *bad_reads { + response.payload = object_store::GetResultPayload::Stream( + futures_util::stream::iter([Ok(Bytes::from_static(b"changed!!"))]) + .boxed(), + ); + } + } } } Ok(response) @@ -4281,6 +4289,73 @@ mod tests { assert!(matches!(error, StorageError::CorruptObject { .. })); } + #[tokio::test] + async fn logical_immutable_create_retries_one_corrupt_readback() { + let inner = Arc::new(ReadFaultStore { + inner: Arc::new(InMemory::new()), + fault: ReadFault::CorruptReads { + bad_reads: 1, + attempts: AtomicU64::new(0), + }, + }); + let store = memory_store_with_inner(inner); + let outcome = store + .create_or_read_immutable( + &Path::from("blobs/transient-corrupt-readback"), + Bytes::from_static(b"candidate"), + 1024, + ) + .await + .unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Created(ETag { + e_tag: Some("get-2-etag".to_owned()), + version: Some("get-2-version".to_owned()), + }) + ); + } + + #[tokio::test] + async fn immutable_readback_retries_only_reads_and_still_rejects_persistent_corruption() { + for logical in [false, true] { + for bad_reads in 0..=2 { + let inner = Arc::new(crate::test_support::CountingObjectStore::new(Arc::new( + ReadFaultStore { + inner: Arc::new(InMemory::new()), + fault: ReadFault::CorruptReads { + bad_reads, + attempts: AtomicU64::new(0), + }, + }, + ))); + let store = memory_store_with_inner(inner.clone()); + let path = Path::from("blobs/verified-create"); + let body = Bytes::from_static(b"candidate"); + let outcome = if logical { + store + .create_or_read_immutable(&path, body, 1024) + .await + .map(|_| ()) + } else { + store.put_if_absent_verified(&path, body).await.map(|_| ()) + }; + let success = outcome.is_ok(); + let corrupt = matches!(outcome, Err(StorageError::CorruptObject { .. })); + assert_eq!( + (success, corrupt, inner.put_requests(), inner.counts().full), + ( + bad_reads < 2, + bad_reads == 2, + 1, + if bad_reads == 0 { 1 } else { 2 } + ), + "logical={logical}, bad_reads={bad_reads}" + ); + } + } + } + #[tokio::test] async fn logical_immutable_create_rejects_oversized_existing_body() { let store = memory_store(); From 2f9f328c3adfbb3bca376993dba7d79b1880016e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 12:23:48 -0700 Subject: [PATCH 093/127] perf(metadata): coalesce visibility proof reads Compacted capsules separate proof pages with Git payloads. Copy proof-only sections into one footer-bound region and share a bounded page/block cache to remove per-edit network round trips. Keep visibility reconciliation and dictionary limits intact. Advance the unshipped run footer to version 5; audited published release objects do not contain this capsule format. Logical replay and historical byte growth remain separate required work. --- .../docs/architecture/indexed-remote-reads.md | 35 ++ .../e2e/run_add_commit_push_rustfs_smoke.py | 2 +- .../e2e/test_add_commit_push_source.py | 4 +- crates/crab-metadata/Cargo.toml | 2 +- crates/crab-metadata/README.md | 20 +- .../src/capsule_protocol/catalog_capture.rs | 129 ++++++++ .../crab-metadata/src/capsule_protocol/run.rs | 64 +++- .../capsule_protocol/run/visibility_pool.rs | 191 +++++++++++ .../visibility/paged/reader.rs | 5 +- crates/crab-metadata/src/pages/cache.rs | 306 ++++++++++++++++-- 10 files changed, 722 insertions(+), 36 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/run/visibility_pool.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 866cc3b3d..b2cc50d1f 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2399,3 +2399,38 @@ Raw before/after evidence remains outside the checkout in representative S3 latency, additional history/ref scaling and fuller Git blame semantics remain open gates. Earlier component receipts retain their original source identities and are not final combined-source proof. + + +### Co-located visibility proof reads + +The isolated current-catalog opening fixture holds one ref, one physical run and +an empty catalog fixed. Before this change, 2, 8 and 32 logical edits require 8, +20 and 68 provider read attempts. Compaction preserves complete nested capsules, +so detached visibility headers and change pages otherwise remain separated by +arbitrary Git payload spans. A run-wide cache alone cannot safely read ahead +across those payloads. + +Run footer version 5 adds a contiguous visibility-proof pool after the Git index +pool and before the aggregate catalog. Only compacted runs duplicate those +sections; leaf readers keep their original section coordinates. Full inspection +verifies the pool digest and each section against the original capsule identity. +Footer-only readers validate exact region coverage and original section lengths. +Their shared page cache reads 64 KiB blocks clipped to that proof-only region, +with one 8 MiB / 256-entry budget for opaque blocks and authenticated decoded +pages. Every selected page verifies its existing root commitment. Cold and warm +query admission charges the complete block spans before starting I/O. + +This change preserves visibility reconciliation, including expected-old values, +borrowed captured closures, removal validity, canonical transaction/catalog +ordering and the ten-million-distinct-object proof limit. It removes the isolated +per-edit request pattern for compacted small proofs. It does not yet remove +logical control replay, full addition enumeration for distinct dictionary counts, +or growth in the number of physical runs. Those remain required opening and +history-scaling work; a passing request-count test is insufficient to close them. + +A fresh release audit inspected the six published tag objects and 19 known local +tag objects. None contains the current capsule-run modules or magic values. +Published `v1.2.4` differs from the untouched local tag; both objects were checked. +The unshipped run footer versions 1 through 4 are rejected without a fallback. +Published v1 storage formats are unaffected. Runtime, corruption, resource-budget +and affected-consumer validation must accompany this format change before push. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index fd9cf3c17..c2e018145 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -254,7 +254,7 @@ def capsule_member_range(body: bytes, kind: str) -> tuple[int, int]: try: footer = json.loads(body[footer_start:-48]) members = footer["git_packs"] - if footer["version"] != 4 or len(members) != 1: + if footer["version"] != 5 or len(members) != 1: raise SmokeError("fault fixture requires one current-format pack member") offset, length = members[0][kind]["offset"], members[0][kind]["length"] except (KeyError, TypeError, ValueError) as exc: diff --git a/crab/scripts/e2e/test_add_commit_push_source.py b/crab/scripts/e2e/test_add_commit_push_source.py index 4f2c42e1a..278cf0b41 100644 --- a/crab/scripts/e2e/test_add_commit_push_source.py +++ b/crab/scripts/e2e/test_add_commit_push_source.py @@ -216,7 +216,7 @@ def failed_run(smoke: SMOKE.AddCommitPushSmoke) -> None: class CapsuleFaultRangeTests(unittest.TestCase): @staticmethod - def fixture(*, offset: int = 8, length: int = 9, version: int = 4) -> bytes: + def fixture(*, offset: int = 8, length: int = 9, version: int = 5) -> bytes: body = b"PACKbody\xfftOcindex" footer = json.dumps({ "version": version, @@ -240,7 +240,7 @@ def test_refuses_ranges_that_could_damage_unselected_controls(self) -> None: def test_refuses_other_container_formats_and_malformed_trailers(self) -> None: body = self.fixture() - for malformed in (body[:-1], body[:20], self.fixture(version=3), body[:-48] + bytes(48)): + for malformed in (body[:-1], body[:20], self.fixture(version=3), self.fixture(version=4), body[:-48] + bytes(48)): with self.subTest(size=len(malformed)), self.assertRaises(SMOKE.SmokeError): SMOKE.capsule_member_range(malformed, "pack") diff --git a/crates/crab-metadata/Cargo.toml b/crates/crab-metadata/Cargo.toml index 02bd000ab..6b768e047 100644 --- a/crates/crab-metadata/Cargo.toml +++ b/crates/crab-metadata/Cargo.toml @@ -17,7 +17,7 @@ default = [] file-index-reader = ["storage", "dep:futures-util", "dep:object_store", "dep:slatedb", "dep:tokio", "dep:tokio-util"] local-index = ["dep:rusqlite"] remote-index = ["storage", "dep:futures-util", "dep:object_store", "dep:slatedb"] -storage = ["dep:base64", "dep:crab-storage", "dep:futures-util", "dep:object_store"] +storage = ["dep:base64", "dep:crab-storage", "dep:futures-util", "dep:object_store", "dep:tokio"] [dependencies] base64 = { version = "0.22", optional = true } diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 993fdbcbb..524d3bedd 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -538,7 +538,8 @@ loads, resumable construction, compaction and maintenance scans retain markers. The retired unreleased version 5 is rebuilt by existing maintenance, never read through an alternate codec. -Run footer version 4 adds an embedded `CRBRCG01` catalog index. Its four paged +Run footer version 5 retains the embedded `CRBRCG01` catalog index introduced in +version 4. Its four paged key directories retain exact capsule occurrences and group equal immutable values by a semantic commitment that excludes physical page offsets. Full run verification rebuilds the index from nested capsules, so re-signed membership @@ -554,5 +555,18 @@ than loading catalog bytes. Full maintenance inspection materializes those captured sections explicitly. Run footers contain catalog roots, never inline catalog bodies. This reader wiring is under qualification; it does not establish the complete history-scaling, lifecycle or 10 GiB acceptance gates. -The affected run format is absent from the audited release tags; unshipped -version 3 is rejected rather than retained through a fallback. +Compacted runs also concatenate copies of their visibility sections after the +Git index pool. The footer binds the pool's span and digest; full verification +checks each copy against its original capsule section. Logical controls share +one cache of decoded pages and 64 KiB proof blocks, bounded to 8 MiB and 256 +entries per physical run. Read-ahead stays inside the visibility pool, never +crossing Git payloads, and charges the complete fetched blocks on cold and warm +queries. Every requested page still authenticates against its capsule's roots. +Expected-old checks, captured prior closures, canonical transaction ordering and +the distinct-object proof limit remain unchanged. This reduces per-edit network +round trips; logical replay and distinct historical-object growth remain open +qualification gates. +The current capsule format is absent from the six published release tags and +19 known local tag objects audited on 2026-10-15. The published `v1.2.4` object +differs from the local tag; both were inspected without replacing that tag. +Unshipped run footer versions 1 through 4 are rejected, with no fallback reader. diff --git a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs index 78c162da2..48c123c73 100644 --- a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs +++ b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs @@ -173,3 +173,132 @@ fn corrupt(reason: impl Into) -> MetadataError { reason: reason.into(), } } + +#[cfg(all(test, feature = "file-index-reader"))] +#[expect(clippy::unwrap_used, reason = "test assertions")] +mod tests { + use super::*; + use crate::capsule_protocol::{ + Capsule, CapsulePointer, CapsuleRefEdit, CapsuleRefHead, CapsuleRun, CapsuleSection, + CapsuleSectionKind, CapsuleTransaction, CapsuleVisibilityDelta, RepositoryRoot, RootRecord, + capsule_ref_name_key, create_root, + }; + use crate::file_index_lookup::FileIndexLookupSession; + use crate::git_visibility::GitVisibilityEdit; + use std::collections::BTreeSet; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[tokio::test] + async fn current_catalog_opening_reads_follow_compacted_runs() { + let mut observations = Vec::new(); + // Metadata-only history keeps the catalog empty. One physical run + // isolates visibility replay from run fan-out and payload lookup. + for count in [2, 8, 32] { + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_read_request_observer(Arc::new(move |_| { + observed.fetch_add(1, Ordering::SeqCst); + })); + let layout = StoreLayout::new(store, "catalog-opening".to_owned()); + let root = create_root( + &layout, + RootRecord::encode( + RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(), + ) + .unwrap(), + ) + .await + .unwrap(); + let mut leaves = Vec::new(); + let mut previous = None; + for version in 1..=count { + let oid = format!("{version:040x}"); + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + previous.clone(), + Some(oid.clone()), + None, + )], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/main".to_owned(), + GitVisibilityEdit::from_delta_objects( + previous, + oid.clone(), + vec![oid.clone()], + Vec::new(), + ), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + )], + ) + .unwrap(); + leaves.push(CapsuleRun::leaf(capsule).unwrap()); + previous = Some(oid); + } + let run = CapsuleRun::compact(leaves).unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let head = CapsuleRefHead::from_root( + "refs/heads/main", + root.record().root().ref_epoch().to_owned(), + None, + None, + ) + .unwrap(); + let state = head + .successor_state( + &BTreeSet::new(), + previous, + None, + run.transaction_ids().last().unwrap().clone(), + vec![pointer], + ) + .unwrap(); + layout + .store() + .create_strict( + &layout.capsule_ref_head_path(&capsule_ref_name_key("refs/heads/main")), + head.commit(state).unwrap().encode().unwrap(), + ) + .await + .unwrap(); + reads.store(0, Ordering::SeqCst); + let session = + FileIndexLookupSession::open_for_storage(layout.store(), "catalog-opening") + .await + .unwrap(); + observations.push((count, reads.load(Ordering::SeqCst))); + session.close().await.unwrap(); + } + eprintln!("compacted catalog opening read attempts: {observations:?}"); + assert!( + observations.windows(2).all(|pair| pair[0].1 == pair[1].1), + "provider reads should follow compacted proof blocks rather than logical edits: {observations:?}" + ); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index c07b638c0..e0f8cfebb 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -11,7 +11,7 @@ use crate::error::{MetadataError, Result}; use crate::validation::validate_content_hash; const RUN_MAGIC: &[u8; 8] = b"CRBRUN01"; -const RUN_VERSION: u32 = 4; +const RUN_VERSION: u32 = 5; const RUN_TRAILER_BYTES: usize = 8 + 32 + RUN_MAGIC.len(); pub(super) const MAX_RUN_FOOTER_BYTES: usize = 8 * 1024 * 1024; /// Largest number of push capsules coalesced before a repository checkpoint. @@ -64,6 +64,7 @@ struct CapsuleRunFooter { capsules: Vec, git_packs: Vec, index_pool: Option, + visibility_pool: Option, #[serde(default)] admission: Option, admission_pages: Option, @@ -73,6 +74,7 @@ struct CapsuleRunFooter { mod admission; pub(in crate::capsule_protocol) mod catalog; +mod visibility_pool; use admission::AdmissionPages; pub use admission::CapsuleRunAdmission; #[cfg(feature = "storage")] @@ -105,6 +107,7 @@ pub struct CapsuleRunControl { capsules: Vec, git_packs: Vec, index_pool: Option, + visibility_pool: Option, admission: Option, admission_info: Option<(RunSectionLocation, AdmissionPages)>, catalog_info: Option<(RunSectionLocation, CatalogPages)>, @@ -370,6 +373,7 @@ impl CapsuleRunControl { .ok_or_else(|| corrupt("capsule run range overflowed"))?; } expected_offset = validate_index_pool(&footer, expected_offset, control_offset)?; + expected_offset = visibility_pool::validate(&footer, expected_offset, control_offset)?; expected_offset = validate_catalog(&footer, expected_offset, control_offset)?; if expected_offset != control_offset { return Err(corrupt( @@ -442,6 +446,7 @@ impl CapsuleRunControl { capsules: controls, git_packs: footer.git_packs, index_pool: footer.index_pool, + visibility_pool: footer.visibility_pool, admission, admission_info, catalog_info: footer.catalog.zip(footer.catalog_pages), @@ -507,19 +512,56 @@ impl CapsuleRunControl { layout: &crab_storage::StoreLayout, capsule_hash: &str, maximum: u64, + ) -> Result> { + self.visibility_reader_with_cache(layout, capsule_hash, maximum, self.visibility_cache()) + } + + #[cfg(feature = "storage")] + fn visibility_cache(&self) -> std::sync::Arc { + std::sync::Arc::new(match &self.visibility_pool { + Some(pool) => { + crate::pages::PageCache::for_region(pool.offset..pool.offset + pool.length) + } + None => crate::pages::PageCache::default(), + }) + } + + #[cfg(feature = "storage")] + fn visibility_reader_with_cache( + &self, + layout: &crab_storage::StoreLayout, + capsule_hash: &str, + maximum: u64, + cache: std::sync::Arc, ) -> Result> { let location = self .capsules .iter() .find(|location| location.hash == capsule_hash) .ok_or_else(|| corrupt("visibility capsule is absent from its run"))?; - match (&location.visibility, &location.visibility_pages) { + let mut range = location.visibility.clone(); + if let Some(pool) = &self.visibility_pool { + let preceding = self + .capsules + .iter() + .take_while(|item| item.hash != capsule_hash) + .filter_map(|item| item.visibility.as_ref()) + .try_fold(pool.offset, |offset, item| { + offset.checked_add(item.length()) + }) + .ok_or_else(|| corrupt("pooled visibility offset overflows"))?; + range = range + .map(|range| PackRange::from_parts(preceding, range.length(), range.blake3())) + .transpose()?; + } + match (&range, &location.visibility_pages) { (Some(range), Some(pages)) => super::visibility::CapsuleVisibilityReader::new( layout, &self.hash, range, pages.clone(), maximum, + cache, ) .map(Some), (None, None) => Ok(None), @@ -625,12 +667,17 @@ impl CapsuleRunControl { &self, layout: &crab_storage::StoreLayout, ) -> Result> { + // One immutable run owns this bounded cache. All of its logical edits + // share co-located proof blocks; each decoded page still authenticates + // against the original capsule's roots and consumes its query budget. + let cache = self.visibility_cache(); let mut controls = self.materialize_controls( |location| { - self.visibility_reader( + self.visibility_reader_with_cache( layout, &location.hash, crate::git_visibility::MAX_GIT_VISIBILITY_INDEX_BYTES, + cache.clone(), ) .map(|reader| { reader.map(|reader| ControlVisibility::Paged(std::sync::Arc::new(reader))) @@ -1015,6 +1062,7 @@ impl CapsuleRun { } else { None }; + let visibility_pool = visibility_pool::build(&mut body, level, &locations)?; let catalog = CatalogPages::build(&capsules)?.map(|(bytes, descriptor)| { let range = RunSectionLocation { offset: body.len() as u64, @@ -1059,6 +1107,7 @@ impl CapsuleRun { capsules: locations.clone(), git_packs: git_packs.clone(), index_pool: index_pool.clone(), + visibility_pool, catalog: catalog.as_ref().map(|(range, _)| range.clone()), catalog_pages: catalog.as_ref().map(|(_, descriptor)| descriptor.clone()), admission: admission.as_ref().map(|(_, range)| range.clone()), @@ -1172,7 +1221,9 @@ impl CapsuleRun { } } } - expected_offset = validate_catalog(&footer, pool_end, footer_start as u64)?; + let visibility_end = visibility_pool::validate(&footer, pool_end, footer_start as u64)?; + visibility_pool::verify(&bytes, &footer)?; + expected_offset = validate_catalog(&footer, visibility_end, footer_start as u64)?; if let (Some(range), Some(pages)) = (&footer.catalog, &footer.catalog_pages) { let body = bytes .get(range.offset as usize..expected_offset as usize) @@ -1653,8 +1704,9 @@ fn validate_index_pool( return Ok(capsule_end); }; let end = footer - .catalog + .visibility_pool .as_ref() + .or(footer.catalog.as_ref()) .or(footer.admission.as_ref()) .map_or(control_offset, |range| range.offset); validate_body_range(pool, capsule_end, end)?; @@ -2000,7 +2052,7 @@ mod tests { #[test] fn unsupported_run_footer_versions_are_rejected() { let run = CapsuleRun::leaf(capsule('1', '2')).unwrap(); - for version in [0, 1, 2, 3, 5, 6, 7] { + for version in [0, 1, 2, 3, 4, 6, 7] { let mut footer = run.footer.clone(); footer.version = version; let footer = serde_json::to_vec(&footer).unwrap(); diff --git a/crates/crab-metadata/src/capsule_protocol/run/visibility_pool.rs b/crates/crab-metadata/src/capsule_protocol/run/visibility_pool.rs new file mode 100644 index 000000000..e9ac745a6 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/run/visibility_pool.rs @@ -0,0 +1,191 @@ +//! Co-located visibility proofs keep compacted control reads away from Git payloads. +use super::*; + +pub(super) fn build( + body: &mut Vec, + level: u8, + capsules: &[RunCapsuleLocation], +) -> Result> { + if level == 0 || capsules.iter().all(|capsule| capsule.visibility.is_none()) { + return Ok(None); + } + let offset = body.len(); + for range in capsules + .iter() + .filter_map(|capsule| capsule.visibility.as_ref()) + { + let start = usize::try_from(range.offset) + .map_err(|_| contract_error("visibility pool offset cannot be represented"))?; + let end = range + .offset + .checked_add(range.length) + .and_then(|end| usize::try_from(end).ok()) + .filter(|end| *end <= offset) + .ok_or_else(|| contract_error("visibility pool source is outside its capsules"))?; + body.extend_from_within(start..end); + } + Ok(Some(RunSectionLocation { + offset: offset as u64, + length: (body.len() - offset) as u64, + hash: blake3::hash(&body[offset..]).to_hex().to_string(), + })) +} + +pub(super) fn validate(footer: &CapsuleRunFooter, offset: u64, control_offset: u64) -> Result { + let required = footer.level > 0 + && footer + .capsules + .iter() + .any(|capsule| capsule.visibility.is_some()); + if required != footer.visibility_pool.is_some() { + return Err(corrupt( + "visibility pool differs from its level and capsule sections", + )); + } + let Some(pool) = &footer.visibility_pool else { + return Ok(offset); + }; + let end = footer + .catalog + .as_ref() + .or(footer.admission.as_ref()) + .map_or(control_offset, |range| range.offset); + validate_body_range(pool, offset, end)?; + let length = footer + .capsules + .iter() + .filter_map(|capsule| capsule.visibility.as_ref()) + .try_fold(0u64, |length, section| length.checked_add(section.length)) + .ok_or_else(|| corrupt("visibility pool length overflows"))?; + if pool.length != length { + return Err(corrupt( + "visibility pool does not cover its capsule sections", + )); + } + Ok(end) +} + +pub(super) fn verify(bytes: &[u8], footer: &CapsuleRunFooter) -> Result<()> { + let Some(pool) = &footer.visibility_pool else { + return Ok(()); + }; + let body = bytes + .get(pool.offset as usize..(pool.offset + pool.length) as usize) + .ok_or_else(|| corrupt("visibility pool is outside its run"))?; + if blake3::hash(body).to_hex().as_str() != pool.hash { + return Err(corrupt("visibility pool hash differs")); + } + let mut offset = 0; + for range in footer + .capsules + .iter() + .filter_map(|capsule| capsule.visibility.as_ref()) + { + let end = offset + range.length as usize; + let section = body + .get(offset..end) + .ok_or_else(|| corrupt("pooled visibility section is truncated"))?; + if blake3::hash(section).to_hex().as_str() != range.hash { + return Err(corrupt( + "pooled visibility differs from its original capsule", + )); + } + offset = end; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capsule_protocol::{CapsuleRefEdit, CapsuleSection}; + use crate::git_visibility::GitVisibilityEdit; + + fn compacted() -> CapsuleRun { + let leaves = (1..=2) + .map(|version| { + let oid = format!("{version:040x}"); + let tx = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(oid.clone()), + None, + )], + ) + .unwrap(); + let delta = CapsuleVisibilityDelta::new(BTreeMap::from([( + "refs/heads/main".into(), + GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid]), + )])) + .unwrap(); + CapsuleRun::leaf( + Capsule::build( + &tx, + vec![], + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + delta.encode().unwrap(), + )], + ) + .unwrap(), + ) + .unwrap() + }) + .collect(); + CapsuleRun::compact(leaves).unwrap() + } + + fn resign(mut body: Vec, footer: &CapsuleRunFooter) -> Bytes { + let footer = serde_json::to_vec(footer).unwrap(); + body.extend_from_slice(&footer); + body.extend_from_slice(&(footer.len() as u64).to_be_bytes()); + body.extend_from_slice(blake3::hash(&footer).as_bytes()); + body.extend_from_slice(RUN_MAGIC); + body.into() + } + + #[test] + fn invalid_pool_routing_fails_full_and_footer_only_decoding() { + let run = compacted(); + for change in ["missing", "offset", "length", "hash"] { + let mut footer = run.footer.clone(); + match change { + "missing" => footer.visibility_pool = None, + "offset" => footer.visibility_pool.as_mut().unwrap().offset += 1, + "length" => footer.visibility_pool.as_mut().unwrap().length += 1, + "hash" => footer.visibility_pool.as_mut().unwrap().hash = "invalid".into(), + _ => unreachable!(), + } + let body = resign( + run.bytes()[..run.control_offset() as usize].to_vec(), + &footer, + ); + let full = CapsuleRun::decode(body.clone()).is_err(); + let control = CapsuleRunControl::decode_suffix( + body.slice(run.control_offset() as usize..), + body.len() as u64, + run.hash(), + run.level(), + &run.transaction_ids(), + run.newest_base_root_digest(), + ) + .is_err(); + assert_eq!((full, control), (true, true), "{change}"); + } + } + + #[test] + fn re_signed_pool_cannot_replace_original_capsule_evidence() { + let run = compacted(); + let mut footer = run.footer.clone(); + let pool = footer.visibility_pool.as_mut().unwrap(); + let mut body = run.bytes()[..run.control_offset() as usize].to_vec(); + body[pool.offset as usize] ^= 1; + pool.hash = blake3::hash(&body[pool.offset as usize..(pool.offset + pool.length) as usize]) + .to_hex() + .to_string(); + assert!(CapsuleRun::decode(resign(body, &footer)).is_err()); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs index faff57b6d..f15a10bb5 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs @@ -21,7 +21,7 @@ pub struct CapsuleVisibilityReader { length: u64, hash: String, complete: std::sync::OnceLock>, - cache: PageCache, + cache: std::sync::Arc, } impl CapsuleVisibilityReader { @@ -31,6 +31,7 @@ impl CapsuleVisibilityReader { range: &super::super::super::PackRange, descriptor: Descriptor, maximum: u64, + cache: std::sync::Arc, ) -> Result { descriptor.validate_region(range.length())?; range @@ -45,7 +46,7 @@ impl CapsuleVisibilityReader { length: range.length(), hash: range.blake3().to_owned(), complete: std::sync::OnceLock::new(), - cache: PageCache::default(), + cache, }) } diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs index 1138d86cf..f5be7b4d0 100644 --- a/crates/crab-metadata/src/pages/cache.rs +++ b/crates/crab-metadata/src/pages/cache.rs @@ -7,15 +7,50 @@ use std::sync::{Arc, Mutex, MutexGuard}; const CACHE_BYTES: usize = 8 * 1024 * 1024; const CACHE_PAGES: usize = 256; +const READ_AHEAD_BYTES: u64 = 64 * 1024; #[derive(Debug, Default)] -pub(crate) struct PageCache(Mutex); +pub(crate) struct PageCache { + state: Mutex, + read_ahead: Option>, + block_reads: [tokio::sync::Mutex<()>; 4], +} #[derive(Debug, Default)] struct State { pages: BTreeMap<(usize, u64), (Kind, PageRef, Arc)>, - order: VecDeque<(usize, u64)>, + order: VecDeque, bytes: usize, + blocks: BTreeMap, +} + +#[derive(Debug)] +enum Entry { + Page((usize, u64)), + Block(u64), +} + +impl State { + fn evict_for(&mut self, weight: usize) { + while self.bytes.saturating_add(weight) > CACHE_BYTES + || self.pages.len() + self.blocks.len() >= CACHE_PAGES + { + let Some(oldest) = self.order.pop_front() else { + break; + }; + let removed = match oldest { + Entry::Page(key) => self + .pages + .remove(&key) + .map_or(0, |(_, reference, page)| page_weight(&reference, &page)), + Entry::Block(offset) => self + .blocks + .remove(&offset) + .map_or(0, |body| block_weight(&body)), + }; + self.bytes = self.bytes.saturating_sub(removed); + } + } } pub(crate) struct PageWork { @@ -25,11 +60,11 @@ pub(crate) struct PageWork { } impl PageWork { - fn admit(&mut self, layer: usize, offset: u64, reference: &PageRef) -> Result<()> { + fn admit(&mut self, layer: usize, offset: u64, bytes: u64) -> Result<()> { if self.seen.insert((layer, offset)) { self.bytes = self .bytes - .checked_add(u64::from(reference.bytes)) + .checked_add(bytes) .ok_or_else(|| corruption("query byte count overflows"))?; if self.bytes > self.maximum { return Err(corruption("metadata query exceeds its byte limit")); @@ -40,6 +75,42 @@ impl PageWork { } impl PageCache { + pub(crate) fn for_region(region: std::ops::Range) -> Self { + Self { + read_ahead: Some(region), + ..Self::default() + } + } + + fn blocks(&self, range: std::ops::Range) -> Vec> { + let Some(region) = self + .read_ahead + .as_ref() + .filter(|region| region.start <= range.start && range.end <= region.end) + else { + return vec![range]; + }; + let start = + region.start + (range.start - region.start) / READ_AHEAD_BYTES * READ_AHEAD_BYTES; + (start..range.end) + .step_by(READ_AHEAD_BYTES as usize) + .map(|start| start..start.saturating_add(READ_AHEAD_BYTES).min(region.end)) + .collect() + } + + fn admit( + &self, + work: &mut PageWork, + layer: usize, + offset: u64, + reference: &PageRef, + ) -> Result<()> { + for block in self.blocks(offset..offset + u64::from(reference.bytes)) { + work.admit(layer, block.start, block.end - block.start)?; + } + Ok(()) + } + #[expect( clippy::too_many_arguments, reason = "page lookup binds routing, origin and caller-owned aggregate admission" @@ -70,7 +141,7 @@ impl PageCache { while !pending.is_empty() { for (reference, _) in &pending { reference.validate(kind, source_bytes)?; - work.admit(layer, source.start + reference.offset, reference)?; + self.admit(work, layer, source.start + reference.offset, reference)?; } // Charge the entire wave before starting I/O, including warm pages. let pages = futures_util::stream::iter(std::mem::take(&mut pending)) @@ -140,7 +211,7 @@ impl PageCache { while !pending.is_empty() { for reference in &pending { reference.validate(kind, source_bytes)?; - work.admit(layer, source.start + reference.offset, reference)?; + self.admit(work, layer, source.start + reference.offset, reference)?; } let pages = futures_util::stream::iter(std::mem::take(&mut pending)) .map(|reference| { @@ -168,7 +239,7 @@ impl PageCache { // Guards protect local cache work only, never origin I/O. fn state(&self) -> Result> { - self.0 + self.state .lock() .map_err(|_| MetadataError::Internal("metadata page cache is poisoned".to_owned())) } @@ -197,9 +268,21 @@ impl PageCache { } return Ok(page.clone()); } - let bytes = store - .range_get(path, offset..offset + u64::from(reference.bytes)) - .await?; + let range = offset..offset + u64::from(reference.bytes); + let bytes = if self.read_ahead.is_some() { + let mut bytes = Vec::with_capacity(reference.bytes as usize); + for block in self.blocks(range.clone()) { + let body = self.read_block(store, path, block.clone()).await?; + let start = range.start.max(block.start) - block.start; + let end = range.end.min(block.end) - block.start; + bytes.extend_from_slice(body.get(start as usize..end as usize).ok_or_else( + || corruption("metadata read-ahead block does not cover its page"), + )?); + } + bytes::Bytes::from(bytes) + } else { + store.range_get(path, range).await? + }; let page = Arc::new(decode( &bytes, reference, @@ -216,24 +299,64 @@ impl PageCache { } return Ok(existing.clone()); } - while cache.bytes.saturating_add(weight) > CACHE_BYTES - || cache.pages.len() >= CACHE_PAGES - { - let Some(oldest) = cache.order.pop_front() else { - break; - }; - if let Some((_, reference, old)) = cache.pages.remove(&oldest) { - cache.bytes = cache.bytes.saturating_sub(page_weight(&reference, &old)); - } - } + cache.evict_for(weight); cache.bytes += weight; - cache.order.push_back(key); + cache.order.push_back(Entry::Page(key)); cache .pages .insert(key, (kind, reference.clone(), page.clone())); } Ok(page) } + + async fn read_block( + &self, + store: &Store, + path: &object_store::path::Path, + block: std::ops::Range, + ) -> Result { + if let Some(body) = self.state()?.blocks.get(&block.start).cloned() { + return Ok(body); + } + // A page wave can request neighboring pages in the same proof block. + // Coalesce those reads so admission does not undercount duplicate I/O; + // warm hits stay independent of cold reads in the four fetch lanes. + let lane = (block.start / READ_AHEAD_BYTES) as usize % self.block_reads.len(); + let _guard = self.block_reads[lane].lock().await; + if let Some(body) = self.state()?.blocks.get(&block.start).cloned() { + return Ok(body); + } + let body = store.range_get(path, block.clone()).await?; + if body.len() as u64 != block.end - block.start { + return Err(corruption("metadata read-ahead block is truncated")); + } + self.cache_block(block.start, body.clone())?; + Ok(body) + } + + fn cache_block(&self, offset: u64, body: bytes::Bytes) -> Result<()> { + let mut cache = self.state()?; + if cache.blocks.contains_key(&offset) { + return Ok(()); + } + if body.len() <= CACHE_BYTES { + let weight = block_weight(&body); + if weight > CACHE_BYTES { + return Ok(()); + } + cache.evict_for(weight); + cache.bytes += weight; + cache.order.push_back(Entry::Block(offset)); + cache.blocks.insert(offset, body); + } + Ok(()) + } +} + +fn block_weight(body: &bytes::Bytes) -> usize { + body.len() + .saturating_add(std::mem::size_of::()) + .saturating_add(128) } fn page_weight(reference: &PageRef, page: &Page) -> usize { @@ -244,3 +367,144 @@ fn page_weight(reference: &PageRef, page: &Page) -> usize { .saturating_add(reference.last.capacity()) .saturating_add(128) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + + #[tokio::test] + async fn coalesced_reads_stay_inside_proof_region_and_charge_cold_and_warm_work() { + let mut writer = Writer::default(); + let root = writer + .tree(Kind::VisibilityChanges, vec![(vec![1; 20], vec![])]) + .unwrap(); + let bytes = Arc::new(AtomicU64::new(0)); + let observed = bytes.clone(); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_read_byte_observer(Arc::new(move |count| { + observed.fetch_add(count, Ordering::SeqCst); + })); + let path = object_store::path::Path::from("proof"); + let region = 128..128 + writer.bytes.len() as u64; + let mut body = vec![0; 128]; + body.extend_from_slice(&writer.bytes); + body.extend_from_slice(&[0; 128]); + store.create_strict(&path, body.into()).await.unwrap(); + let cache = PageCache::for_region(region.clone()); + let mut work = PageWork { + maximum: u64::MAX, + bytes: 0, + seen: BTreeSet::new(), + }; + cache + .lookup( + &store, + &path, + 0, + region.clone(), + &root, + Kind::VisibilityChanges, + &[vec![1; 20]], + &mut work, + ) + .await + .unwrap(); + let cold = bytes.swap(0, Ordering::SeqCst); + let mut limited = PageWork { + maximum: cold - 1, + bytes: 0, + seen: BTreeSet::new(), + }; + let rejected = cache + .lookup( + &store, + &path, + 0, + region.clone(), + &root, + Kind::VisibilityChanges, + &[vec![1; 20]], + &mut limited, + ) + .await + .is_err(); + assert_eq!( + (cold, work.bytes, rejected, bytes.load(Ordering::SeqCst)), + (region.end - region.start, cold, true, 0) + ); + } + + #[tokio::test] + async fn prefetched_pages_still_require_their_own_authentication() { + let mut writer = Writer::default(); + let first = writer + .tree(Kind::VisibilityChanges, vec![(vec![1; 20], vec![])]) + .unwrap(); + let second = writer + .tree(Kind::VisibilityChanges, vec![(vec![2; 20], vec![])]) + .unwrap(); + writer.bytes[second.offset as usize] ^= 1; + let store = Store::new(Arc::new(object_store::memory::InMemory::new())); + let path = object_store::path::Path::from("proof"); + let region = 0..writer.bytes.len() as u64; + store + .create_strict(&path, writer.bytes.into()) + .await + .unwrap(); + let cache = PageCache::for_region(region.clone()); + cache + .fetch( + &store, + &path, + 0, + region.clone(), + &first, + Kind::VisibilityChanges, + ) + .await + .unwrap(); + assert!( + cache + .fetch(&store, &path, 0, region, &second, Kind::VisibilityChanges) + .await + .is_err() + ); + } + + #[tokio::test] + async fn proof_blocks_and_decoded_pages_share_one_cache_bound() { + let region = 0..256 * READ_AHEAD_BYTES; + let cache = PageCache::for_region(region.clone()); + for offset in 1..256 { + cache + .cache_block( + offset * READ_AHEAD_BYTES, + vec![0; READ_AHEAD_BYTES as usize].into(), + ) + .unwrap(); + } + let mut writer = Writer::default(); + let root = writer + .tree(Kind::VisibilityChanges, vec![(vec![1; 20], vec![])]) + .unwrap(); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())); + let path = object_store::path::Path::from("proof"); + writer.bytes.resize(region.end as usize, 0); + store + .create_strict(&path, writer.bytes.into()) + .await + .unwrap(); + let page = cache + .fetch(&store, &path, 0, region, &root, Kind::VisibilityChanges) + .await + .unwrap(); + let mut state = cache.state().unwrap(); + state.evict_for(MAX_BYTES); + assert!( + state.bytes + MAX_BYTES <= CACHE_BYTES + && state.pages.len() + state.blocks.len() <= CACHE_PAGES + && Arc::ptr_eq(&state.pages.get(&(0, root.offset)).unwrap().2, &page) + ); + } +} From 6f299e5577fe9753cd6c2ec9a786900491b0c315 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 13:24:11 -0700 Subject: [PATCH 094/127] perf(metadata): bound locator layer pressure --- crates/crab-metadata/README.md | 7 +++++++ crates/crab-metadata/src/git_object_locator/writer.rs | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 524d3bedd..ecbe60207 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -403,6 +403,13 @@ covered by their pinned repository snapshot, then open the named checkpoint for the actual lookup. A missing marker means publication is incomplete; malformed or name-mismatched marker content fails closed. +Locator maintenance bounds the shared L0 frontier and compaction fan-in at 16. +The coordinator, worker and result commit poll every 500 ms while maintenance +is enabled. This prevents frequent checkpoint flushes from starving sorted-run +compaction or waiting on the dependency's five-second coordinator default. +Publication enables maintenance under measured L0 pressure; coverage-only and +reader-repair writers keep their existing maintenance-free behavior. + ### Snapshot identity `RepositorySnapshot` also captures the validated canonical layout descriptor. diff --git a/crates/crab-metadata/src/git_object_locator/writer.rs b/crates/crab-metadata/src/git_object_locator/writer.rs index 0041ce127..bc423b198 100644 --- a/crates/crab-metadata/src/git_object_locator/writer.rs +++ b/crates/crab-metadata/src/git_object_locator/writer.rs @@ -33,7 +33,10 @@ const DB_LABEL: &str = "git_object_catalog_db"; const MAX_BATCH_ROWS: usize = 25_000; const MAX_BATCH_LOGICAL_BYTES: usize = 2 * 1024 * 1024; const LOCATOR_L0_SST_BYTES: usize = 64 * 1024 * 1024; -const LOCATOR_L0_MAX_SSTS: usize = 32; +// The scheduler favors L0 until similarly sized sorted runs fill this fan-in. +// Bound both frontiers so frequent checkpoint flushes cannot accumulate dozens +// of runs while new L0 work continually takes the sole compaction slot. +const LOCATOR_L0_MAX_SSTS: usize = 16; const LOCATOR_COMPACTION_TRIGGER_SSTS: usize = LOCATOR_L0_MAX_SSTS / 2; const LOCATOR_COMPACTION_MAX_CONCURRENT: usize = 1; const LOCATOR_COMPACTION_MAX_SUBCOMPACTIONS: usize = 1; @@ -1596,6 +1599,10 @@ fn should_start_compactor(l0_ssts: usize, planned_object_rows: u64) -> bool { fn locator_settings(compact: bool) -> Settings { let compactor_options = compact.then(|| { let mut compactor = CompactorOptions { + // Flush backpressure waits for coordinator scheduling, not worker + // polling. Keep all three phases aligned so a full frontier does + // not idle for SlateDB's five-second default coordinator tick. + poll_interval: std::time::Duration::from_millis(500), commit_compacted_interval: std::time::Duration::from_millis(500), max_concurrent_compactions: LOCATOR_COMPACTION_MAX_CONCURRENT, ..CompactorOptions::default() @@ -1969,7 +1976,7 @@ mod tests { .expect("locator compactor enabled"); assert_eq!( compactor.poll_interval, - CompactorOptions::default().poll_interval + std::time::Duration::from_millis(500) ); assert_eq!( compactor.commit_compacted_interval, From a9be52a20c059668b9da9cc15fbe6eaed5368379 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 14:49:53 -0700 Subject: [PATCH 095/127] perf(metadata): reuse immutable checkpoint catalogs Separate catalog ownership from checkpoint bytes so ref-only maintenance can reuse authenticated catalogs without body I/O. Keep catalog durability before publication and retain dependencies through history, browse roots, fsck and GC. Replace unshipped checkpoint format 7 and browse retention format 1 with format 8 and format 2. Released v1 storage is unaffected; unreleased fixtures must be recreated. --- .../docs/architecture/indexed-remote-reads.md | 40 ++ crab/src/cmd/fsck_store.rs | 14 +- crab/src/cmd/gc/captured_index_tests.rs | 106 +++- crab/src/cmd/gc/mod.rs | 25 +- crab/src/cmd/history_recovery_v2.rs | 24 +- .../cmd/history_recovery_v2/layered_tests.rs | 25 +- crab/src/cmd/metadb/capsule_tests.rs | 4 +- crates/crab-metadata/README.md | 21 +- .../src/capsule_protocol/browse_retention.rs | 43 +- .../browse_retention/tests.rs | 54 ++- .../src/capsule_protocol/catalog_capture.rs | 9 +- .../src/capsule_protocol/history.rs | 6 +- .../src/capsule_protocol/layered.rs | 181 +++---- .../src/capsule_protocol/layered/catalog.rs | 324 +++++++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 6 +- .../pointer/paged/reader/tests.rs | 18 +- .../src/capsule_protocol/root.rs | 5 +- .../crab-metadata/src/capsule_protocol/run.rs | 6 + .../src/capsule_protocol/store.rs | 31 +- .../store/catalog_order_tests.rs | 456 ++++++++++++++++++ crates/crab-metadata/src/error.rs | 3 + .../src/file_index_lookup/catalog_tests.rs | 15 +- crates/crab-read/src/capsule_protocol.rs | 76 ++- .../src/capsule_protocol/dependency_tests.rs | 1 + crates/crab-read/src/hydrator.rs | 1 + crates/crab-remote/src/checkpoint.rs | 96 ++-- crates/crab-remote/tests/checkpoint.rs | 4 + .../tests/checkpoint/publication_catalog.rs | 68 +++ crates/crab-storage/src/layout.rs | 13 + crates/crab-write/src/capsule_protocol.rs | 176 ++++++- 30 files changed, 1641 insertions(+), 210 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/layered/catalog.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index b2cc50d1f..16a7d4502 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2434,3 +2434,43 @@ Published `v1.2.4` differs from the untouched local tag; both objects were check The unshipped run footer versions 1 through 4 are rejected without a fallback. Published v1 storage formats are unaffected. Runtime, corruption, resource-budget and affected-consumer validation must accompany this format change before push. + + +### Independent checkpoint catalog lifetime + +Checkpoint format 8 (`CRBCKP08`) replaces the inline catalog section with an +immutable `v2/catalogs//` commitment containing the full body +hash, encoded size and authenticated page roots. Selected queries keep the same +bounded page reader and complete-or-error dependency closure. Checkpoint loading +reads visibility and commitments; full catalog materialization is explicit. + +The shared writer verifies newly encoded catalog bytes before checkpoint or +history publication and before root CAS. Logical ref-only checkpoints and +physical repacking carry the existing catalog commitment when no selected +capsule has a catalog delta. They do not read, decode, re-encode or upload that +body. A selected catalog delta still invokes ordered full assembly: this slice +establishes ownership and unchanged-catalog reuse, not bounded changed-catalog +maintenance or bounded repository opening. + +GC marks the catalog from current and historical checkpoints. Retained browse +record format 2 records the complete checkpoint pointer and catalog identity. +Maintenance authenticates the checkpoint footer and rejects omitted or +mismatched catalog/source dependencies before trusting the retention paths. +A retained catalog survives replacement or deletion of its original checkpoint; +its lifetime belongs directly to each referencing checkpoint or retained root. +Full fsck and history verification authenticate the complete catalog body, and +history byte/object accounting includes the external dependency. + +Release boundary: all six upstream release tags (v1.0.1 and v1.2.0 through +v1.2.4) lack the capsule-protocol codec. Checkpoint format 7 and browse retention +record format 1 are unshipped development formats and are rejected. Recreate +unreleased qualification fixtures rather than adding an indefinite fallback or +silently rewriting authoritative history. Earlier format-7 measurements in this +document remain historical evidence, not format-8 qualification. + +Required proof for this slice: zero catalog body I/O on ref-only checkpointing; +cold historical hydration after original checkpoint replacement; full-size +admission before catalog reads; missing/corrupt pages; ordinary/restore +publication failures; and current/history/retained-only catalog survival through +GC with the existing grace period and fences. No format-8 10 GiB or hosted-S3 +performance acceptance is established by the earlier combined-format run. diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index a3845ee03..b1a3a55a2 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -972,10 +972,16 @@ async fn verify_capsule_history( let mut sources = BTreeMap::new(); let mut checkpoints = futures_util::stream::iter(checkpoints.into_values().map(|pointer| async move { - crab_metadata::capsule_protocol::load_layered_checkpoint(layout, &pointer) - .await - .map(|checkpoint| checkpoint.sources().to_vec()) - .map_err(CrabError::from) + let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( + layout, + &pointer, + u64::MAX, + ) + .await?; + checkpoint + .pointer_catalog_from_store(layout, u64::MAX) + .await?; + Ok::<_, CrabError>(checkpoint.sources().to_vec()) })) .buffer_unordered(16); while let Some(checkpoint_sources) = checkpoints.try_next().await? { diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index a210fc397..0d7d5baa1 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -483,6 +483,7 @@ async fn capsule_gc_lists_each_candidate_namespace_once() { let mut expected = [ "v2/capsules/", "v2/checkpoints/", + "v2/catalogs/", "v2/pack-layers/", "v2/history/", "metadata/", @@ -494,7 +495,7 @@ async fn capsule_gc_lists_each_candidate_namespace_once() { expected.sort(); assert_eq!( (actual, outcome.list_requests, outcome.list_parallelism), - (expected, 7, 7) + (expected, 8, 8) ); } @@ -550,3 +551,106 @@ async fn capsule_gc_preserves_metadata_outside_owned_index_directories() { "GC removed metadata outside its owned directories: {retained:?}" ); } + +#[tokio::test] +async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { + use crab_metadata::capsule_protocol::{FileCatalogEntry, PointerCatalog, ShardCatalogEntry}; + let backend = Arc::new(AgedObjects { + inner: InMemory::new(), + paths: Mutex::new(HashSet::new()), + modified: SystemTime::now() - Duration::from_secs(7200), + list_calls: Mutex::new(Vec::new()), + }); + let store = Store::new(backend.clone()); + let layout = StoreLayout::new( + store.as_storage().clone(), + "org/checkpoint-catalog-gc".to_owned(), + ); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + publish_blob(&layout, "visible").await; + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(1, Vec::new())) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) + .unwrap(); + let before = view(&layout).await; + crab_remote::checkpoint::publish_capsule_checkpoint_with_catalog_from_view( + &layout, + &before, + catalog.clone(), + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + let first = view(&layout).await; + let source = first.layered_checkpoint().unwrap().catalog().unwrap(); + let retained = layout.capsule_catalog_path(source.hash()); + backend.paths.lock().unwrap().insert(retained.clone()); + // Publish a different current catalog; the old one is now history-owned. + catalog + .insert_file("c".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) + .unwrap(); + crab_remote::checkpoint::publish_capsule_checkpoint_with_catalog_from_view( + &layout, + &first, + catalog.clone(), + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + let current = view(&layout).await; + let current_path = layout.capsule_catalog_path( + current + .layered_checkpoint() + .unwrap() + .catalog() + .unwrap() + .hash(), + ); + backend.paths.lock().unwrap().insert(current_path.clone()); + let mut orphans = Vec::new(); + for (index, aged) in [(1, true), (2, false)] { + let bytes = Bytes::from(format!("unreferenced-{index}")); + let path = layout.capsule_catalog_path(&blake3::hash(&bytes).to_hex().to_string()); + layout + .store() + .put_if_absent_verified(&path, bytes) + .await + .unwrap(); + if aged { + backend.paths.lock().unwrap().insert(path.clone()); + } + orphans.push(path); + } + let snapshot = current.git_snapshot().unwrap(); + sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + current.root().root(), + current.capsule_run_pointers(), + &snapshot.manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now(), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + let mut existing = Vec::new(); + for path in [&retained, ¤t_path, &orphans[0], &orphans[1]] { + existing.push(match layout.store().head(path).await { + Ok(_) => true, + Err(crab_storage::StorageError::NotFound { .. }) => false, + Err(error) => panic!("unexpected catalog HEAD failure: {error}"), + }); + } + assert_eq!(existing, [true, true, false, true]); +} diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index ce5fd23b6..a341b72e8 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3238,6 +3238,7 @@ async fn sweep_capsule_objects( let capsule_prefix = layout.repo_path("v2/capsules/"); let checkpoint_prefix = layout.repo_path("v2/checkpoints/"); + let catalog_prefix = layout.capsule_catalog_prefix(); let pack_layer_prefix = layout.repo_path("v2/pack-layers/"); let history_prefix = layout.repo_path("v2/history/"); let metadata_prefix = layout.repo_path("metadata/"); @@ -3251,6 +3252,7 @@ async fn sweep_capsule_objects( let ( capsules, checkpoints, + catalogs, pack_layers, history, metadata_objects, @@ -3259,6 +3261,7 @@ async fn sweep_capsule_objects( ) = tokio::try_join!( store.list_prefix(&capsule_prefix), store.list_prefix(&checkpoint_prefix), + store.list_prefix(&catalog_prefix), store.list_prefix(&pack_layer_prefix), store.list_prefix(&history_prefix), store.list_prefix(&metadata_prefix), @@ -3317,6 +3320,7 @@ async fn sweep_capsule_objects( let candidates = capsules .into_iter() .chain(checkpoints) + .chain(catalogs) .chain(pack_layers) .chain(history) .chain(derived_index_objects) @@ -3382,8 +3386,8 @@ async fn sweep_capsule_objects( } } Ok(GcOutcome { - list_requests: 7, - list_parallelism: 7, + list_requests: 8, + list_parallelism: 8, list_wall_seconds: started.elapsed().as_secs_f64(), dry_run: args.dry_run, ..accounting @@ -3512,9 +3516,20 @@ async fn mark_layered_checkpoint_sources( pointer: &crab_metadata::capsule_protocol::CheckpointPointer, reachable: &mut HashSet, ) -> Result<()> { - let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint(layout, pointer) - .await - .map_err(CrabError::from)?; + let checkpoint = + crab_metadata::capsule_protocol::load_layered_checkpoint(layout, pointer, u64::MAX) + .await + .map_err(CrabError::from)?; + if let Some(catalog) = checkpoint.catalog() { + let path = layout.capsule_catalog_path(catalog.hash()).to_string(); + // Ref-only checkpoints share one catalog. Authenticate it once per + // sweep, so retained checkpoint count does not multiply body decoding. + if reachable.insert(path) { + checkpoint + .pointer_catalog_from_store(layout, u64::MAX) + .await?; + } + } for source in checkpoint.sources() { let path = match source.kind() { crab_metadata::capsule_protocol::PackSourceKind::CapsuleRun => { diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index 0aa66055d..2672217e3 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -196,14 +196,19 @@ async fn verify_history( ) -> Result { check_cancelled(cancel)?; let segment = select_history(layout, root, generation, digest).await?; - let checkpoint = - crab_metadata::capsule_protocol::load_layered_checkpoint(layout, segment.checkpoint()) - .await?; + let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( + layout, + segment.checkpoint(), + u64::MAX, + ) + .await?; let (source_count, source_bytes) = verify_history_sources(layout, &segment, &checkpoint, cancel).await?; check_cancelled(cancel)?; - let catalog = checkpoint.pointer_catalog()?; + let catalog = checkpoint + .pointer_catalog_from_store(layout, u64::MAX) + .await?; validate_visibility(&segment, &checkpoint)?; let workspace = verify_git_checkpoint(layout, &segment, &checkpoint).await?; let dependencies = crab_read::capsule_protocol::verify_installed_dependencies( @@ -221,15 +226,15 @@ async fn verify_history( let xorb_count = u64::try_from(catalog.xorbs().len()) .map_err(|_| CrabError::Internal("history xorb count overflowed".to_owned()))?; let dependency_objects = 2_u64 - .checked_add(source_count) + .checked_add(u64::from(checkpoint.catalog().is_some())) + .and_then(|total| total.checked_add(source_count)) .and_then(|total| total.checked_add(shard_count)) .and_then(|total| total.checked_add(xorb_count)) .and_then(|total| total.checked_add(dependencies.reachable_lfs_objects)) .ok_or_else(|| CrabError::Internal("history dependency count overflowed".to_owned()))?; let segment_bytes = u64::try_from(segment.bytes().len()) .map_err(|_| CrabError::Internal("history segment size overflowed".to_owned()))?; - let checkpoint_bytes = u64::try_from(checkpoint.bytes().len()) - .map_err(|_| CrabError::Internal("history checkpoint size overflowed".to_owned()))?; + let checkpoint_bytes = checkpoint.encoded_metadata_bytes()?; let dependency_bytes = segment_bytes .checked_add(checkpoint_bytes) .and_then(|total| total.checked_add(source_bytes)) @@ -643,7 +648,10 @@ async fn restore_history( let current_catalog = view.pointer_catalog().await?; // Keep both catalogs: history may use a recipe no longer in the current // view, while newer content must remain available after a rollback. - let mut restored_catalog = verified.checkpoint.pointer_catalog()?; + let mut restored_catalog = verified + .checkpoint + .pointer_catalog_from_store(layout, u64::MAX) + .await?; restored_catalog.apply(¤t_catalog)?; let outcome = crab_remote::checkpoint::publish_capsule_checkpoint_from_view( layout, diff --git a/crab/src/cmd/history_recovery_v2/layered_tests.rs b/crab/src/cmd/history_recovery_v2/layered_tests.rs index 1df07fc96..a0d0027b8 100644 --- a/crab/src/cmd/history_recovery_v2/layered_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_tests.rs @@ -129,7 +129,7 @@ async fn layered_history_verification_preserves_historical_refs_and_bytes() { let (layout, root) = fixture().await; let chain = history_chain(&layout, &root).await.unwrap(); let oldest = chain.last().unwrap(); - assert_eq!(oldest.checkpoint().format(), 7); + assert_eq!(oldest.checkpoint().format(), 8); let verified = verify_history( &layout, @@ -252,10 +252,13 @@ async fn layered_restore_preserves_sources_and_allows_new_epoch_publication() { let (layout, root) = fixture().await; let chain = history_chain(&layout, &root).await.unwrap(); let oldest = chain.last().unwrap(); - let historical = - crab_metadata::capsule_protocol::load_layered_checkpoint(&layout, oldest.checkpoint()) - .await - .unwrap(); + let historical = crab_metadata::capsule_protocol::load_layered_checkpoint( + &layout, + oldest.checkpoint(), + u64::MAX, + ) + .await + .unwrap(); let store = Store::from_storage(layout.store().clone()); let router = StoreLayout::new(store.clone(), layout.repo_prefix().to_owned()); let cancel = CancellationToken::new(); @@ -286,6 +289,7 @@ async fn layered_restore_preserves_sources_and_allows_new_epoch_publication() { let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( &layout, restored.record().root().checkpoint().unwrap(), + u64::MAX, ) .await .unwrap(); @@ -350,10 +354,13 @@ async fn corrupt_layered_history_cannot_restore_or_leak_the_sweep_lease() { let (layout, root) = fixture().await; let chain = history_chain(&layout, &root).await.unwrap(); let oldest = chain.last().unwrap(); - let checkpoint = - crab_metadata::capsule_protocol::load_layered_checkpoint(&layout, oldest.checkpoint()) - .await - .unwrap(); + let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( + &layout, + oldest.checkpoint(), + u64::MAX, + ) + .await + .unwrap(); let source = &checkpoint.sources()[0]; let path = layout.capsule_path(source.object_hash()); let (body, _) = layout.store().get_with_etag(&path).await.unwrap(); diff --git a/crab/src/cmd/metadb/capsule_tests.rs b/crab/src/cmd/metadb/capsule_tests.rs index 5532d045c..df948c274 100644 --- a/crab/src/cmd/metadb/capsule_tests.rs +++ b/crab/src/cmd/metadb/capsule_tests.rs @@ -109,7 +109,7 @@ async fn layered_diagnosis_and_rebuild_verify_nonempty_repository_without_rewrit .await .unwrap(); let before = root.record().digest().to_owned(); - assert_eq!(root.record().root().checkpoint().unwrap().format(), 7); + assert_eq!(root.record().root().checkpoint().unwrap().format(), 8); let diagnosis = diagnose_capsule(&layout, root.clone(), DbSelector::Both, true, &cancel) .await .unwrap(); @@ -183,6 +183,7 @@ async fn layered_metadata_verification_preserves_missing_source_failure() { let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( &layout, root.record().root().checkpoint().unwrap(), + u64::MAX, ) .await .unwrap(); @@ -226,6 +227,7 @@ async fn layered_metadata_verification_rejects_corruption_outside_git_pack_range let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( &layout, root.record().root().checkpoint().unwrap(), + u64::MAX, ) .await .unwrap(); diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index ecbe60207..3a65c046b 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -60,8 +60,18 @@ capsules can therefore appear in differently compacted ref frontiers without inflating pack counts or appearing to conflict; differing payload commitments remain rejected. -Checkpoint format 7 (`CRBCKP07`) carries authenticated catalog page roots in -its footer. `LayeredCheckpoint::pointer_catalog_lookup` captures these roots; +Checkpoint format 8 (`CRBCKP08`) commits to a separate immutable catalog at +`v2/catalogs//`, including its exact size and authenticated +page roots. Catalog durability precedes checkpoint/history writes and root CAS. +Ref-only checkpoint publication and physical repacking reuse that commitment +without decoding, re-encoding, reading or rewriting the catalog body. A visible +catalog delta still uses the canonical ordered full assembly; incremental delta +updates remain required. Full checkpoint loading verifies visibility and catalog +commitments; explicit maintenance materializes and verifies the catalog body. +GC follows catalog objects from current, historical and retained browse roots. +Retained browse record format 2 carries the complete checkpoint pointer and +catalog identity; loading authenticates the footer and rejects omitted or +mismatched catalog/source dependencies. `LayeredCheckpoint::pointer_catalog_lookup` captures these roots; `PointerCatalogLookup` batches file/chunk/xorb keys through one bounded page cache and byte budget. Its `dependencies` query selects files plus explicit shard/xorb identities in one aggregate budget, including complete dependency @@ -83,8 +93,11 @@ The current-state opener also captures authenticated catalog roots and leaves catalog bodies cold. Explicitly pinned callers pass their existing capture, while current-state sessions bind once to the first root they open. Catalog and Git views share stable ref capture, ref-frontier selection, visibility reconciliation -and transaction ordering; neither path invents a separate precedence policy. All 19 audited local release tags omit these -capsule checkpoint formats; unshipped format 6 is rejected without a fallback. +and transaction ordering; neither path invents a separate precedence policy. +All six upstream release tags audited for format 8 omit the capsule protocol +codec. Earlier formats, including unshipped checkpoint format 7 and browse +retention record format 1, are rejected explicitly. Unreleased fixtures must be +recreated; no fallback reader or automatic authoritative conversion is provided. Checkpoint and selected run catalog roots share one lookup budget and page cache. CLI and shared capsule-publication preparation use these captured queries. Protected receive selects from separately authorized source/filtered captures. diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 407d15424..773f76a7b 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -5,7 +5,7 @@ use crab_storage::{ETag, Store, StoreLayout}; use serde::{Deserialize, Serialize}; use super::{ - BrowseIndexes, CapsulePointer, LayeredCheckpoint, MAX_BROWSE_INDEXES_BYTES, + BrowseIndexes, CapsulePointer, CheckpointPointer, LayeredCheckpoint, MAX_BROWSE_INDEXES_BYTES, MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, PackSourceKind, }; use crate::error::{MetadataError, Result}; @@ -27,7 +27,8 @@ struct Payload { generation: u64, indexes: BrowseIndexes, previous: Option, - checkpoint: Option, + checkpoint: Option, + catalog: Option, sources: BTreeMap, } @@ -63,7 +64,13 @@ impl RetainedBrowseIndexes { self.payload .checkpoint .iter() - .map(|hash| layout.capsule_checkpoint_path(hash)) + .map(|pointer| layout.capsule_checkpoint_path(pointer.hash())) + .chain( + self.payload + .catalog + .iter() + .map(|hash| layout.capsule_catalog_path(hash)), + ) .chain(self.payload.sources.iter().map(|(hash, kind)| match kind { PackSourceKind::CapsuleRun => layout.capsule_path(hash), PackSourceKind::PackLayer => layout.capsule_pack_layer_path(hash), @@ -84,14 +91,19 @@ impl Payload { } fn validate(&self) -> Result<()> { - if self.version != 1 { + if self.version != 2 { return Err(corrupt("unsupported retained browse version")); } self.indexes.encode()?; + if let Some(pointer) = &self.checkpoint { + super::root::validate_checkpoint_pointer(pointer)?; + } else if self.catalog.is_some() { + return Err(corrupt("retained catalog has no checkpoint owner")); + } for hash in self .previous .iter() - .chain(self.checkpoint.iter()) + .chain(self.catalog.iter()) .chain(self.sources.keys()) { validate_content_hash(hash, "retention dependency", "browse retention")?; @@ -169,6 +181,20 @@ async fn load_payload( let payload: Payload = serde_json::from_slice(&bytes) .map_err(|source| MetadataError::BrowseIndexRecord { source })?; payload.validate()?; + if let Some(pointer) = &payload.checkpoint { + // A retention record cannot omit an owned dependency from GC marking. + // Authenticate the checkpoint footer before trusting its captured paths. + let checkpoint = super::load_layered_checkpoint_control(layout, pointer).await?; + let catalog = checkpoint.catalog().map(|value| value.hash()); + if payload.catalog.as_deref() != catalog + || checkpoint + .sources() + .iter() + .any(|source| payload.sources.get(source.object_hash()) != Some(&source.kind())) + { + return Err(corrupt("retained dependencies differ from checkpoint")); + } + } Ok((payload, bytes.len() as u64)) } @@ -217,11 +243,14 @@ pub async fn retain_browse_indexes( } } let mut payload = Payload { - version: 1, + version: 2, generation, indexes: indexes.clone(), previous: previous.as_ref().map(|(head, _)| head.hash.clone()), - checkpoint: checkpoint.map(|value| value.hash().to_owned()), + checkpoint: checkpoint.map(LayeredCheckpoint::pointer).transpose()?, + catalog: checkpoint + .and_then(LayeredCheckpoint::catalog) + .map(|value| value.hash().to_owned()), sources, }; if let Some((head, _)) = &previous { diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index 4fb9f5491..e97fd20df 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -176,14 +176,29 @@ async fn integrity_fixture() -> (StoreLayout, Vec payload.catalog = None, + "catalog-replaced" => payload.catalog = Some("f".repeat(64)), + _ => payload.sources.clear(), + } + // Rehash the malformed record: raw object integrity alone cannot + // prove that its dependency list owns the authenticated checkpoint. + let bytes = payload.encode().unwrap(); + let head = Head::append(None, &bytes).unwrap(); + layout + .store() + .put_overwrite(&layout.browse_retained_root_path(&head.hash), bytes) + .await + .unwrap(); + layout + .store() + .put_overwrite( + &layout.browse_retained_head_path(), + Bytes::from(serde_json::to_vec(&head).unwrap()), + ) + .await + .unwrap(); + assert!( + load_retained_browse_indexes(&layout).await.is_err(), + "{corruption}" + ); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs index 48c123c73..e14f7d6b1 100644 --- a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs +++ b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs @@ -31,9 +31,12 @@ pub(super) async fn materialize_from_root( ) -> Result { let captured = capture(layout, root).await?; let base_catalog = match root.record().root().checkpoint() { - Some(pointer) => load_layered_checkpoint(layout, pointer) - .await? - .pointer_catalog()?, + Some(pointer) => { + load_layered_checkpoint(layout, pointer, MAX_CATALOG_BYTES) + .await? + .pointer_catalog_from_store(layout, MAX_CATALOG_BYTES) + .await? + } None => PointerCatalog::new(), }; // Concurrent ref histories have a canonical binding order, not a global diff --git a/crates/crab-metadata/src/capsule_protocol/history.rs b/crates/crab-metadata/src/capsule_protocol/history.rs index d6b354f26..bfdc03249 100644 --- a/crates/crab-metadata/src/capsule_protocol/history.rs +++ b/crates/crab-metadata/src/capsule_protocol/history.rs @@ -441,7 +441,7 @@ mod tests { #[test] fn history_rejects_unsupported_checkpoint_formats() { - for format in [0, 1, 2, 3, 4, 5, 6, 8, u32::MAX] { + for format in [0, 1, 2, 3, 4, 5, 6, 7, 9, u32::MAX] { let segment = history_with_unvalidated_pointers(format, 1); let built = HistorySegment::build( segment.checkpoint().clone(), @@ -467,8 +467,8 @@ mod tests { fn history_rejects_implicit_checkpoint_format() { let segment = HistorySegment::build(checkpoint(), None, state()).unwrap(); let body = std::str::from_utf8(segment.bytes()).unwrap(); - assert_eq!(body.matches("\"format\":7,").count(), 1); - let bytes = Bytes::from(body.replace("\"format\":7,", "")); + assert_eq!(body.matches("\"format\":8,").count(), 1); + let bytes = Bytes::from(body.replace("\"format\":8,", "")); let error = HistorySegment::decode(bytes) .expect_err("an implicit retired checkpoint format must fail"); diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index 1003e13db..bdd82644e 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -11,20 +11,26 @@ use crate::git_visibility::{ }; use crate::validation::{validate_content_hash, validate_sha1}; +mod catalog; +use catalog::CatalogInput; +pub use catalog::CheckpointCatalog; + mod visibility_pages; #[cfg(feature = "storage")] pub use visibility_pages::LayeredVisibilityReader; -const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP07"; -pub(super) const CHECKPOINT_VERSION: u32 = 7; +const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP08"; +pub(super) const CHECKPOINT_VERSION: u32 = 8; const LAYER_MAGIC: &[u8; 8] = b"CRBPKL01"; const LAYER_VERSION: u32 = 1; const TRAILER_BYTES: usize = 8 + 32 + 8; -const MAX_CHECKPOINT_FOOTER_BYTES: usize = 8 * 1024 * 1024; +pub(super) const MAX_CHECKPOINT_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_LAYER_FOOTER_BYTES: usize = 8 * 1024 * 1024; const MAX_PHYSICAL_SOURCES: usize = 64; const MAX_SOURCE_MEMBERS: usize = 512; const MAX_LAYERED_VISIBILITY_BYTES: usize = 128 * 1024 * 1024; +pub(super) const MAX_CHECKPOINT_BYTES: u64 = + (MAX_LAYERED_VISIBILITY_BYTES + MAX_CHECKPOINT_FOOTER_BYTES + TRAILER_BYTES) as u64; const MAX_LAYERED_VISIBILITY_REFS: usize = 100_000; const MAX_LAYERED_VISIBILITY_TRANSITIONS: usize = 1_000_000; const VISIBILITY_OBJECT_SET_DIGEST_DOMAIN: &[u8] = b"crab layered visibility object set\0"; @@ -933,7 +939,7 @@ struct CheckpointFooter { #[serde(default)] visibility_transitions: BTreeMap>, visibility_pages: Option, - catalog_pages: Option, + catalog: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -946,7 +952,7 @@ struct LayerSectionLocation { } /// Metadata-only authenticated v2 checkpoint. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct LayeredCheckpoint { bytes: Bytes, hash: String, @@ -955,8 +961,25 @@ pub struct LayeredCheckpoint { object_size: u64, control_offset: u64, control_only: bool, + catalog_bytes: Option, +} + +impl PartialEq for LayeredCheckpoint { + fn eq(&self, other: &Self) -> bool { + // Loaded/pending catalog bytes are runtime state. The footer already + // commits to their exact identity, independently of cache residency. + self.bytes == other.bytes + && self.hash == other.hash + && self.footer_hash == other.footer_hash + && self.footer == other.footer + && self.object_size == other.object_size + && self.control_offset == other.control_offset + && self.control_only == other.control_only + } } +impl Eq for LayeredCheckpoint {} + impl LayeredCheckpoint { /// Build a metadata-only checkpoint with the compact ordinal proof. pub fn build( @@ -965,6 +988,41 @@ impl LayeredCheckpoint { sources: Vec, pointer_catalog: PointerCatalog, visibility: Option, + ) -> Result { + Self::build_with_catalog( + covered_generation, + covered_root_digest, + sources, + CatalogInput::New(pointer_catalog), + visibility, + ) + } + + /// Reuse a captured immutable catalog when no visible catalog delta changed it. + /// + /// The caller retains the source checkpoint's catalog through publication. + pub fn build_reusing_catalog( + covered_generation: u64, + covered_root_digest: &str, + sources: Vec, + catalog: Option, + visibility: Option, + ) -> Result { + Self::build_with_catalog( + covered_generation, + covered_root_digest, + sources, + CatalogInput::Retained(catalog), + visibility, + ) + } + + fn build_with_catalog( + covered_generation: u64, + covered_root_digest: &str, + sources: Vec, + catalog: CatalogInput, + visibility: Option, ) -> Result { if let Some(snapshot) = &visibility { for member in snapshot.member_admission().unwrap_or_default() { @@ -997,7 +1055,7 @@ impl LayeredCheckpoint { covered_generation, covered_root_digest, sources, - pointer_catalog, + catalog, visibility, cold_clone_object_set_digest, cold_clone_object_count, @@ -1009,7 +1067,7 @@ impl LayeredCheckpoint { covered_generation: u64, covered_root_digest: &str, sources: Vec, - pointer_catalog: PointerCatalog, + catalog: CatalogInput, visibility: Option<(CapsuleSectionKind, Bytes)>, cold_clone_object_set_digest: Option, cold_clone_object_count: Option, @@ -1027,14 +1085,7 @@ impl LayeredCheckpoint { } let mut body = Vec::new(); let mut sections = Vec::new(); - if !pointer_catalog.is_empty() { - append_section( - &mut body, - &mut sections, - CapsuleSectionKind::CatalogDelta, - pointer_catalog.encode()?, - )?; - } + let (catalog, catalog_bytes) = catalog.prepare()?; if let Some((kind, visibility)) = visibility { append_section(&mut body, &mut sections, kind, visibility)?; } @@ -1047,15 +1098,6 @@ impl LayeredCheckpoint { ) }) .transpose()?; - let catalog_pages = sections - .iter() - .find(|section| section.kind == CapsuleSectionKind::CatalogDelta) - .map(|section| { - super::pointer::paged::Descriptor::decode( - &body[section.offset as usize..(section.offset + section.length) as usize], - ) - }) - .transpose()?; let control_offset = 0_u64; let footer = CheckpointFooter { version: CHECKPOINT_VERSION, @@ -1068,7 +1110,7 @@ impl LayeredCheckpoint { cold_clone_object_count, visibility_transitions, visibility_pages, - catalog_pages, + catalog, }; validate_source_inventory(&footer.sources)?; for source in &footer.sources { @@ -1088,11 +1130,16 @@ impl LayeredCheckpoint { body.extend_from_slice(&(footer_bytes.len() as u64).to_be_bytes()); body.extend_from_slice(blake3::hash(&footer_bytes).as_bytes()); body.extend_from_slice(CHECKPOINT_MAGIC); - Self::decode(Bytes::from(body)) + let mut checkpoint = Self::decode(Bytes::from(body))?; + checkpoint.catalog_bytes = catalog_bytes; + Ok(checkpoint) } /// Decode and authenticate one layered checkpoint object. pub fn decode(bytes: Bytes) -> Result { + if bytes.len() as u64 > MAX_CHECKPOINT_BYTES { + return Err(corrupt("layered checkpoint exceeds its encoded size bound")); + } let (footer, footer_start, footer_hash) = decode_checkpoint_footer(&bytes)?; validate_checkpoint_footer(&footer, footer_start as u64, bytes.len() as u64)?; let hash = blake3::hash(&bytes).to_hex().to_string(); @@ -1108,6 +1155,7 @@ impl LayeredCheckpoint { object_size, control_offset, control_only: false, + catalog_bytes: None, }) } @@ -1123,8 +1171,11 @@ impl LayeredCheckpoint { expected_hash: &str, expected_footer_hash: &str, ) -> Result { - if bytes.is_empty() { - return Err(corrupt("layered checkpoint control range is empty")); + if object_size > MAX_CHECKPOINT_BYTES + || bytes.is_empty() + || bytes.len() > MAX_CHECKPOINT_FOOTER_BYTES + TRAILER_BYTES + { + return Err(corrupt("layered checkpoint control range is out of bounds")); } validate_content_hash( expected_hash, @@ -1168,6 +1219,7 @@ impl LayeredCheckpoint { object_size, control_offset: footer_start, control_only: true, + catalog_bytes: None, }) } @@ -1206,21 +1258,9 @@ impl LayeredCheckpoint { layout: &crab_storage::StoreLayout, maximum: u64, ) -> Result { - let Some(descriptor) = &self.footer.catalog_pages else { - return Ok(super::PointerCatalogLookup::empty(maximum)); - }; - let section = self - .footer - .sections - .iter() - .find(|section| section.kind == CapsuleSectionKind::CatalogDelta) - .ok_or_else(|| corrupt("checkpoint catalog pages have no section"))?; - let range = PackRange::from_parts(section.offset, section.length, section.blake3.clone())?; - super::PointerCatalogLookup::checkpoint( - layout.capsule_checkpoint_path(&self.hash), - range, - descriptor.clone(), - maximum, + self.footer.catalog.as_ref().map_or_else( + || Ok(super::PointerCatalogLookup::empty(maximum)), + |catalog| catalog.lookup(layout, maximum), ) } @@ -1340,21 +1380,6 @@ impl LayeredCheckpoint { &self.footer.visibility_transitions } - /// Decode the complete pointer catalog compacted by this checkpoint. - pub fn pointer_catalog(&self) -> Result { - let Some(section) = self.section(CapsuleSectionKind::CatalogDelta)? else { - return Ok(PointerCatalog::new()); - }; - if Some(&super::pointer::paged::Descriptor::decode(§ion)?) - != self.footer.catalog_pages.as_ref() - { - return Err(corrupt( - "checkpoint catalog body roots differ from its authenticated footer", - )); - } - PointerCatalog::decode(§ion) - } - /// Decode the compact ordinal proof and validate its checkpoint source bindings. pub fn visibility_ordinal_snapshot(&self) -> Result> { let Some(section) = self.section(CapsuleSectionKind::VisibilityOrdinalSnapshot)? else { @@ -1592,26 +1617,12 @@ fn validate_checkpoint_footer( )); } } - let catalogs = footer - .sections - .iter() - .filter(|section| section.kind == CapsuleSectionKind::CatalogDelta) - .collect::>(); - match (&footer.catalog_pages, catalogs.as_slice()) { - (Some(descriptor), [section]) => descriptor.validate(section.length)?, - (None, []) => {} - _ => { - return Err(corrupt( - "checkpoint catalog section and footer roots differ", - )); - } + if let Some(catalog) = &footer.catalog { + catalog.validate()?; } let mut expected_offset = 0_u64; for section in &footer.sections { - if !matches!( - section.kind, - CapsuleSectionKind::CatalogDelta | CapsuleSectionKind::VisibilityOrdinalSnapshot - ) { + if !matches!(section.kind, CapsuleSectionKind::VisibilityOrdinalSnapshot) { return Err(corrupt("checkpoint section kind is retired or unsupported")); } validate_content_hash( @@ -1763,7 +1774,7 @@ mod tests { } #[test] - fn checkpoint_control_requires_catalog_roots_for_exactly_one_section() { + fn checkpoint_control_rejects_invalid_catalog_roots() { let source = PackLayer::build(&pack()) .unwrap() .source_descriptor() @@ -1783,20 +1794,22 @@ mod tests { .unwrap(); let checkpoint = LayeredCheckpoint::build(0, &"3".repeat(64), vec![source], catalog, None).unwrap(); - for remove_section in [false, true] { - let mut footer = checkpoint.footer.clone(); - if remove_section { - footer.sections.clear(); - } else { - footer.catalog_pages = None; + for field in ["hash", "bytes", "body_bytes"] { + let mut footer = serde_json::to_value(&checkpoint.footer).unwrap(); + match field { + "hash" => footer["catalog"]["hash"] = serde_json::json!("invalid"), + "bytes" => footer["catalog"]["bytes"] = serde_json::json!(0), + _ => footer["catalog"]["pages"]["body_bytes"] = serde_json::json!(u64::MAX), } + let footer = serde_json::from_value(footer).unwrap(); assert!( validate_checkpoint_footer( &footer, checkpoint.control_offset(), - checkpoint.bytes.len() as u64 + checkpoint.bytes.len() as u64, ) - .is_err() + .is_err(), + "accepted invalid {field}" ); } } diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs new file mode 100644 index 000000000..0bcc63bc4 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs @@ -0,0 +1,324 @@ +use super::*; +use crate::capsule_protocol::pointer::paged::{Descriptor, MAX_CATALOG_BYTES}; + +/// Authenticated page roots and identity of an immutable checkpoint catalog. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointCatalog { + hash: String, + bytes: u64, + pages: Descriptor, +} + +impl CheckpointCatalog { + /// Return the content identity retained by checkpoints and captured readers. + #[must_use] + pub fn hash(&self) -> &str { + &self.hash + } + + /// Return the full encoded size used for read admission. + #[must_use] + pub const fn encoded_bytes(&self) -> u64 { + self.bytes + } + + pub(super) fn validate(&self) -> Result<()> { + validate_content_hash(&self.hash, "checkpoint catalog", "layered checkpoint")?; + if self.bytes == 0 || self.bytes > MAX_CATALOG_BYTES { + return Err(corrupt("checkpoint catalog size is out of bounds")); + } + self.pages.validate(self.bytes) + } + + #[cfg(feature = "storage")] + fn verify(&self, bytes: &Bytes) -> Result<()> { + self.validate()?; + if bytes.len() as u64 != self.bytes + || blake3::hash(bytes).to_hex().as_str() != self.hash + || Descriptor::decode(bytes)? != self.pages + { + return Err(corrupt( + "checkpoint catalog differs from its committed roots", + )); + } + Ok(()) + } + + #[cfg(feature = "storage")] + async fn read(&self, layout: &crab_storage::StoreLayout) -> Result { + let hash = blake3::Hash::from_hex(&self.hash) + .map_err(|_| corrupt("checkpoint catalog has an invalid content identity"))?; + let bytes = layout + .store() + .verify_bounded( + &layout.capsule_catalog_path(&self.hash), + hash.as_bytes(), + self.bytes, + ) + .await?; + self.verify(&bytes)?; + Ok(bytes) + } + + #[cfg(feature = "storage")] + pub(super) fn lookup( + &self, + layout: &crab_storage::StoreLayout, + maximum: u64, + ) -> Result { + self.validate()?; + super::super::PointerCatalogLookup::checkpoint( + layout.capsule_catalog_path(&self.hash), + PackRange::from_parts(0, self.bytes, self.hash.clone())?, + self.pages.clone(), + maximum, + ) + } +} + +pub(super) enum CatalogInput { + New(PointerCatalog), + Retained(Option), +} + +impl CatalogInput { + pub(super) fn prepare(self) -> Result<(Option, Option)> { + let catalog = match self { + Self::New(catalog) => catalog, + Self::Retained(source) => { + if let Some(source) = &source { + source.validate()?; + } + return Ok((source, None)); + } + }; + if catalog.is_empty() { + return Ok((None, None)); + } + let bytes = catalog.encode()?; + let source = CheckpointCatalog { + hash: blake3::hash(&bytes).to_hex().to_string(), + bytes: bytes.len() as u64, + pages: Descriptor::decode(&bytes)?, + }; + source.validate()?; + Ok((Some(source), Some(bytes))) + } +} + +impl LayeredCheckpoint { + /// Capture the authenticated checkpoint identity, including footer-only readers. + pub fn pointer(&self) -> Result { + super::super::CheckpointPointer::new_layered( + self.hash(), + self.object_size, + self.control_offset(), + self.control_size(), + self.footer_hash(), + self.covered_generation(), + self.covered_root_digest(), + self.pack_count()?, + self.object_count()?, + ) + } + + /// Return the aggregate encoded checkpoint and catalog size. + pub fn encoded_metadata_bytes(&self) -> Result { + self.object_size + .checked_add(self.catalog().map_or(0, CheckpointCatalog::encoded_bytes)) + .ok_or_else(|| corrupt("checkpoint dependency size overflows")) + } + + /// Return the catalog commitment without loading catalog pages. + #[must_use] + pub fn catalog(&self) -> Option<&CheckpointCatalog> { + self.footer.catalog.as_ref() + } + + /// Persist newly built catalog bytes before publishing their checkpoint. + /// + /// A retained commitment must come from a captured published checkpoint; + /// the caller owns its GC fence until the replacement root is visible. + #[cfg(feature = "storage")] + pub async fn publish_catalog( + &self, + layout: &crab_storage::StoreLayout, + ) -> Result<()> { + if let (Some(source), Some(bytes)) = (&self.footer.catalog, &self.catalog_bytes) { + source.verify(bytes)?; + layout + .store() + .put_if_absent_verified(&layout.capsule_catalog_path(source.hash()), bytes.clone()) + .await?; + } + Ok(()) + } + + /// Materialize only the catalog body using its captured origin and byte limit. + #[cfg(feature = "storage")] + pub async fn pointer_catalog_from_store( + &self, + layout: &crab_storage::StoreLayout, + maximum_bytes: u64, + ) -> Result { + let Some(source) = &self.footer.catalog else { + return Ok(PointerCatalog::new()); + }; + if source.bytes > maximum_bytes { + return Err(MetadataError::CheckpointSize { + bytes: source.bytes, + maximum: maximum_bytes, + }); + } + if self.catalog_bytes.is_some() { + return self.pointer_catalog(); + } + PointerCatalog::decode(&source.read(layout).await?) + } + + #[cfg(feature = "storage")] + fn pointer_catalog(&self) -> Result { + let Some(source) = &self.footer.catalog else { + return Ok(PointerCatalog::new()); + }; + let bytes = self + .catalog_bytes + .as_ref() + .ok_or_else(|| corrupt("checkpoint catalog bytes have not been loaded"))?; + source.verify(bytes)?; + PointerCatalog::decode(bytes) + } +} + +#[cfg(all(test, feature = "storage"))] +mod tests { + use super::*; + use crate::capsule_protocol::{FileCatalogEntry, PackLayer, ShardCatalogEntry}; + use crab_storage::{Store, StoreLayout}; + use std::sync::Arc; + + fn fixture() -> (StoreLayout, LayeredCheckpoint, PointerCatalog) { + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "checkpoint-catalog".to_owned(), + ); + let pack = CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "1".repeat(40), + 1, + ) + .unwrap(); + let source = PackLayer::build(&pack) + .unwrap() + .source_descriptor() + .unwrap(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(1, Vec::new())) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) + .unwrap(); + let checkpoint = + LayeredCheckpoint::build(1, &"c".repeat(64), vec![source], catalog.clone(), None) + .unwrap(); + (layout, checkpoint, catalog) + } + + #[tokio::test] + async fn cold_checkpoint_load_admits_catalog_before_reading_and_verifies_its_body() { + let (layout, checkpoint, catalog) = fixture(); + let pointer = checkpoint.pointer().unwrap(); + layout + .store() + .put_if_absent_verified( + &layout.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + // The asset is deliberately absent: a byte-budget error must precede + // the otherwise observable missing dependency error. + assert!(matches!( + crate::capsule_protocol::load_layered_checkpoint(&layout, &pointer, pointer.size()) + .await, + Err(MetadataError::CheckpointSize { .. }) + )); + let cold = crate::capsule_protocol::load_layered_checkpoint(&layout, &pointer, u64::MAX) + .await + .unwrap(); + assert!( + cold.pointer_catalog_from_store(&layout, u64::MAX) + .await + .is_err() + ); + checkpoint.publish_catalog(&layout).await.unwrap(); + let loaded = crate::capsule_protocol::load_layered_checkpoint(&layout, &pointer, u64::MAX) + .await + .unwrap(); + assert_eq!( + loaded + .pointer_catalog_from_store(&layout, u64::MAX) + .await + .unwrap(), + catalog + ); + let path = layout.capsule_catalog_path(checkpoint.catalog().unwrap().hash()); + let mut bytes = catalog.encode().unwrap().to_vec(); + bytes[0] ^= 1; + layout + .store() + .put_overwrite(&path, Bytes::from(bytes)) + .await + .unwrap(); + assert!( + loaded + .pointer_catalog_from_store(&layout, u64::MAX) + .await + .is_err() + ); + } + + #[tokio::test] + async fn retained_catalog_survives_replacement_of_its_original_checkpoint() { + let (layout, checkpoint, catalog) = fixture(); + checkpoint.publish_catalog(&layout).await.unwrap(); + let next = LayeredCheckpoint::build_reusing_catalog( + 2, + &"d".repeat(64), + checkpoint.sources().to_vec(), + checkpoint.catalog().cloned(), + None, + ) + .unwrap(); + layout + .store() + .put_if_absent_verified( + &layout.capsule_checkpoint_path(next.hash()), + next.bytes().clone(), + ) + .await + .unwrap(); + let control = crate::capsule_protocol::load_layered_checkpoint_control( + &layout, + &next.pointer().unwrap(), + ) + .await + .unwrap(); + assert_eq!(control.pointer().unwrap(), next.pointer().unwrap()); + assert_eq!( + control + .pointer_catalog_from_store(&layout, u64::MAX) + .await + .unwrap(), + catalog + ); + // No original checkpoint was uploaded. The new commitment owns the + // catalog directly, rather than borrowing that old object's lifetime. + assert_eq!(control.catalog(), checkpoint.catalog()); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index bbecfb74f..904694a3f 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -57,9 +57,9 @@ pub use history::{ #[cfg(feature = "storage")] pub use layered::LayeredVisibilityReader; pub use layered::{ - LayeredCheckpoint, LayeredObjectMember, LayeredVisibilitySnapshot, PackLayer, - PackMemberDescriptor, PackRange, PackSourceDescriptor, PackSourceKind, source_catalog_digest, - visibility_object_set_digest, + CheckpointCatalog, LayeredCheckpoint, LayeredObjectMember, LayeredVisibilitySnapshot, + PackLayer, PackMemberDescriptor, PackRange, PackSourceDescriptor, PackSourceKind, + source_catalog_digest, visibility_object_set_digest, }; #[cfg(feature = "storage")] pub use plan::{ diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs index 308031030..2e386c733 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs @@ -57,6 +57,7 @@ async fn fixture( .unwrap() .source_descriptor() .unwrap(); + let catalog_bytes = catalog.encode().unwrap(); let checkpoint = LayeredCheckpoint::build(1, &"3".repeat(64), vec![source], catalog, None).unwrap(); let reads = Arc::new(AtomicUsize::new(0)); @@ -66,6 +67,13 @@ async fn fixture( observed.fetch_add(1, Ordering::SeqCst); })); let layout = StoreLayout::new(store, "catalog-lookup".to_owned()); + if let Some(source) = checkpoint.catalog() { + layout + .store() + .put_overwrite(&layout.capsule_catalog_path(source.hash()), catalog_bytes) + .await + .unwrap(); + } layout .store() .put_overwrite( @@ -293,11 +301,17 @@ async fn chunk_selection_uses_lowest_xorb_then_first_occurrence_across_sources() async fn missing_and_corrupted_pages_are_errors_instead_of_catalog_misses() { for missing in [false, true] { let (layout, checkpoint, lookup, _) = fixture(catalog(2), 1024 * 1024).await; - let path = layout.capsule_checkpoint_path(checkpoint.hash()); + let path = layout.capsule_catalog_path(checkpoint.catalog().unwrap().hash()); if missing { layout.store().delete(&path).await.unwrap(); } else { - let mut bytes = checkpoint.bytes().to_vec(); + let mut bytes = layout + .store() + .get_with_etag(&path) + .await + .unwrap() + .0 + .to_vec(); let source = lookup.checkpoint.as_ref().unwrap(); let offset = source.range.offset() + source.descriptor.files.as_ref().unwrap().offset; bytes[offset as usize] ^= 1; diff --git a/crates/crab-metadata/src/capsule_protocol/root.rs b/crates/crab-metadata/src/capsule_protocol/root.rs index 624c8d170..43c59ae32 100644 --- a/crates/crab-metadata/src/capsule_protocol/root.rs +++ b/crates/crab-metadata/src/capsule_protocol/root.rs @@ -1330,15 +1330,16 @@ mod tests { fn root_rejects_retired_or_implicit_checkpoint_formats() { let record = checkpointed_record(); let payload = serde_json::to_string(record.root()).unwrap(); - assert_eq!(payload.matches("\"format\":7,").count(), 1); + assert_eq!(payload.matches("\"format\":8,").count(), 1); for replacement in [ "", "\"format\":3,", "\"format\":4,", "\"format\":5,", "\"format\":6,", + "\"format\":7,", ] { - let payload = payload.replace("\"format\":7,", replacement); + let payload = payload.replace("\"format\":8,", replacement); // Recompute the envelope digest so only format admission can fail. let mut bytes = ROOT_MAGIC.to_vec(); bytes.extend_from_slice(&ROOT_VERSION.to_be_bytes()); diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index e0f8cfebb..3210c7435 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -876,6 +876,12 @@ impl CapsuleControl { PointerCatalog::decode_delta(&bytes).map(Some) } + /// Report the presence of an authenticated catalog delta without loading its body. + #[must_use] + pub fn has_catalog_delta(&self) -> bool { + self.catalog_identity.is_some() + } + /// Return this capsule's authenticated pack directory with capsule-relative ranges. pub fn git_packs(&self) -> &[PackMemberDescriptor] { &self.git_packs diff --git a/crates/crab-metadata/src/capsule_protocol/store.rs b/crates/crab-metadata/src/capsule_protocol/store.rs index 8cc7fe82f..f2887bca1 100644 --- a/crates/crab-metadata/src/capsule_protocol/store.rs +++ b/crates/crab-metadata/src/capsule_protocol/store.rs @@ -15,6 +15,9 @@ use crate::error::Result; #[cfg(test)] mod tests; +#[cfg(test)] +mod catalog_order_tests; + /// A verified stored root and the provider token protecting its next update. #[derive(Debug, Clone)] pub struct RootSnapshot { @@ -294,11 +297,21 @@ pub async fn load_history_chain( Ok(segments) } -/// Load and verify one immutable metadata-only layered checkpoint. +/// Load and verify a checkpoint's visibility body and external catalog commitment. +/// +/// Catalog pages remain cold until lookup or explicit full materialization. pub async fn load_layered_checkpoint( router: &StoreLayout, pointer: &super::CheckpointPointer, + maximum_bytes: u64, ) -> Result { + let checkpoint_limit = maximum_bytes.min(super::layered::MAX_CHECKPOINT_BYTES); + if pointer.size() > checkpoint_limit { + return Err(MetadataError::CheckpointSize { + bytes: pointer.size(), + maximum: checkpoint_limit, + }); + } if pointer.format() != super::layered::CHECKPOINT_VERSION { return Err(corrupt( &router.capsule_checkpoint_path(pointer.hash()), @@ -317,6 +330,13 @@ pub async fn load_layered_checkpoint( "layered checkpoint does not match its authenticated pointer", )); } + let total = checkpoint.encoded_metadata_bytes()?; + if total > maximum_bytes { + return Err(MetadataError::CheckpointSize { + bytes: total, + maximum: maximum_bytes, + }); + } Ok(checkpoint) } @@ -331,6 +351,15 @@ pub async fn load_layered_checkpoint_control( "checkpoint pointer does not name the layered checkpoint format", )); } + super::root::validate_checkpoint_pointer(pointer)?; + if pointer.size() > super::layered::MAX_CHECKPOINT_BYTES + || pointer.control_size() > super::layered::MAX_CHECKPOINT_FOOTER_BYTES as u64 + 48 + { + return Err(corrupt( + &router.capsule_checkpoint_path(pointer.hash()), + "checkpoint control exceeds format byte bounds", + )); + } // A checkpoint without catalog or visibility sections is entirely footer. // Its valid control range starts at zero; only an empty range is invalid. if pointer.control_size() == 0 { diff --git a/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs b/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs new file mode 100644 index 000000000..21a6d0c48 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs @@ -0,0 +1,456 @@ +#![expect(clippy::unwrap_used, reason = "test assertions")] + +use super::*; +use crate::capsule_protocol::{ + Capsule, CapsuleGitPack, CapsulePointer, CapsuleRefEdit, CapsuleSection, CapsuleTransaction, + CapsuleVisibilityDelta, CheckpointPointer, FileCatalogEntry, HistorySegmentState, + LayeredObjectMember, LayeredVisibilitySnapshot, PackSourceDescriptor, RepositoryRoot, + ShardCatalogEntry, VisibilityReconciliation, XorbCatalogEntry, XorbChunkEntry, + apply_control_refs, load_pointer_catalog_lookup_from_root, order_ref_controls, + source_catalog_digest, +}; +use crate::git_visibility::{GitVisibilityEdit, GitVisibilityIndex}; +use bytes::Bytes; +use std::sync::Arc; + +async fn expose( + layout: &StoreLayout, + root: &RootSnapshot, + transaction: &CapsuleTransaction, + pointer: CapsulePointer, +) { + let edit = &transaction.edits()[0]; + let head = CapsuleRefHead::from_root( + edit.ref_name(), + root.record().root().ref_epoch().to_owned(), + None, + None, + ) + .unwrap(); + let state = head + .successor_state( + &BTreeSet::new(), + edit.new_oid().map(str::to_owned), + None, + transaction.id().unwrap(), + vec![pointer], + ) + .unwrap(); + layout + .store() + .create_strict( + &layout.capsule_ref_head_path(&capsule_ref_name_key(edit.ref_name())), + head.commit(state).unwrap().encode().unwrap(), + ) + .await + .unwrap(); +} + +struct BorrowingFixture { + layout: StoreLayout, + root: RootSnapshot, + original: CapsuleTransaction, + borrowed: CapsuleTransaction, + earlier: CapsuleTransaction, + controls: BTreeMap, + pointers: BTreeMap, + original_source: PackSourceDescriptor, + file: String, + original_shard: String, + borrowed_shard: String, +} + +async fn borrowing_fixture() -> BorrowingFixture { + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "checkpoint-precedence".to_owned(), + ); + let root = create_root( + &layout, + RootRecord::encode(RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap()) + .unwrap(), + ) + .await + .unwrap(); + let tip = "1".repeat(40); + let transaction = |name: String| { + CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new(name, None, Some(tip.clone()), None)], + ) + .unwrap() + }; + let borrowed = (0..64) + .map(|i| transaction(format!("refs/heads/borrow-{i}"))) + .find(|t| t.id().unwrap().as_bytes()[0] < b'8') + .unwrap(); + let borrowed_id = borrowed.id().unwrap(); + let mut producers = (0..256) + .map(|i| transaction(format!("refs/heads/source-{i}"))) + .filter(|t| t.id().unwrap() > borrowed_id) + .take(2) + .collect::>(); + producers.sort_by_key(|t| t.id().unwrap()); + let earlier = producers.remove(0); + let original = producers.remove(0); + let original_id = original.id().unwrap(); + let earlier_id = earlier.id().unwrap(); + let file = "3".repeat(64); + let original_shard = "4".repeat(64); + let borrowed_shard = "5".repeat(64); + let xorb = "6".repeat(64); + let mut descriptors = PointerCatalog::new(); + descriptors + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 99, + "8".repeat(64), + vec![XorbChunkEntry::new("7".repeat(64), 16)], + ), + ) + .unwrap(); + for shard in [&original_shard, &borrowed_shard] { + descriptors + .insert_shard( + shard.clone(), + ShardCatalogEntry::new(80, vec![xorb.clone()]), + ) + .unwrap(); + } + let mut controls = BTreeMap::new(); + let mut pointers = BTreeMap::new(); + let mut original_source = None; + for t in [&original, &borrowed, &earlier] { + let id = t.id().unwrap(); + let edit = if id == borrowed_id { + GitVisibilityEdit::from_delta_objects(Some(tip.clone()), tip.clone(), vec![], vec![]) + } else { + GitVisibilityEdit::from_replacement_objects(None, tip.clone(), vec![tip.clone()]) + }; + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + t.edits()[0].ref_name().to_owned(), + edit, + )])) + .unwrap(); + let mut catalog = if id == original_id { + descriptors.clone() + } else { + PointerCatalog::new() + }; + if id != earlier_id { + catalog + .insert_file( + file.clone(), + FileCatalogEntry::new( + 16, + if id == borrowed_id { + borrowed_shard.clone() + } else { + original_shard.clone() + }, + ), + ) + .unwrap(); + } + // These bodies exercise metadata codecs; this is not native Git or + // reconstruction qualification. The single object admits the shared tip. + let packs = if id == original_id { + vec![ + CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + tip.clone(), + 1, + ) + .unwrap(), + ] + } else { + vec![] + }; + let run = CapsuleRun::leaf( + Capsule::build( + t, + packs, + vec![ + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + ), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + ), + ], + ) + .unwrap(), + ) + .unwrap(); + layout + .store() + .create_strict(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let (control, mut capsules) = run.verified_controls().unwrap(); + if id == original_id { + original_source = Some(control.source_descriptor().unwrap()); + } + controls.insert(id.clone(), capsules.remove(0)); + pointers.insert( + id, + CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(), + ); + } + BorrowingFixture { + layout, + root, + original, + borrowed, + earlier, + controls, + pointers, + original_source: original_source.unwrap(), + file, + original_shard, + borrowed_shard, + } +} + +#[tokio::test] +async fn checkpoint_replay_base_binds_catalog_precedence() { + let BorrowingFixture { + layout, + root, + original, + borrowed, + earlier, + controls, + pointers, + original_source, + file, + original_shard, + borrowed_shard, + } = borrowing_fixture().await; + let original_id = original.id().unwrap(); + let borrowed_id = borrowed.id().unwrap(); + let earlier_id = earlier.id().unwrap(); + for t in [&original, &borrowed] { + expose(&layout, &root, t, pointers[&t.id().unwrap()].clone()).await; + } + let before = load_pointer_catalog_from_root(&layout, &root) + .await + .unwrap(); + let empty = + GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new()).unwrap(); + let ordered = order_ref_controls( + &BTreeMap::new(), + &BTreeMap::new(), + VisibilityReconciliation::Complete(empty.clone()), + &BTreeMap::new(), + controls + .iter() + .filter(|(id, _)| **id != earlier_id) + .map(|(id, c)| (id.clone(), c.clone())) + .collect(), + ) + .await + .unwrap(); + let mut visibility = VisibilityReconciliation::Complete(empty); + let mut refs = BTreeMap::new(); + let mut peeled = BTreeMap::new(); + for control in &ordered { + apply_control_refs(control, &mut refs, &mut peeled).unwrap(); + visibility.apply(control).await.unwrap(); + } + let VisibilityReconciliation::Complete(index) = visibility else { + unreachable!() + }; + let sources = vec![original_source]; + let snapshot = LayeredVisibilitySnapshot::from_index_with_member_admission( + &index, + &source_catalog_digest(&sources).unwrap(), + vec![LayeredObjectMember::new(0, 0)], + ) + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + root.record().root().generation(), + root.record().digest(), + sources, + before.clone(), + Some(snapshot), + ) + .unwrap(); + checkpoint.publish_catalog(&layout).await.unwrap(); + layout + .store() + .create_strict( + &layout.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + let pointer = CheckpointPointer::new_layered( + checkpoint.hash(), + checkpoint.bytes().len() as u64, + checkpoint.control_offset(), + checkpoint.control_size(), + checkpoint.footer_hash(), + checkpoint.covered_generation(), + checkpoint.covered_root_digest(), + checkpoint.pack_count().unwrap(), + checkpoint.object_count().unwrap(), + ) + .unwrap(); + let positions = [&original, &borrowed] + .into_iter() + .map(|t| (t.edits()[0].ref_name().to_owned(), t.id().unwrap())) + .collect::>(); + let history = HistorySegment::build( + pointer.clone(), + None, + HistorySegmentState::new( + refs.clone(), + peeled.clone(), + root.record().root().head().to_owned(), + positions.clone(), + vec![ + pointers[&original_id].clone(), + pointers[&borrowed_id].clone(), + ], + ), + ) + .unwrap(); + layout + .store() + .create_strict( + &layout.capsule_history_segment_path(history.hash()), + history.bytes().clone(), + ) + .await + .unwrap(); + let next = RootRecord::encode( + root.record() + .root() + .install_ref_checkpoint( + root.record().digest(), + pointer, + history.pointer().unwrap(), + refs, + peeled, + positions, + ) + .unwrap(), + ) + .unwrap(); + let etag = layout + .store() + .update( + &layout.capsule_root_path(), + next.bytes().clone(), + root.etag().clone(), + ) + .await + .unwrap(); + let compacted = root.committed_ref_checkpoint(next, etag).unwrap(); + // Keep the original base to characterize explicitly pinned snapshot inputs; + // concurrent writer admission is a separate contract from this metadata API. + expose(&layout, &root, &earlier, pointers[&earlier_id].clone()).await; + let mut winners = Vec::new(); + for captured in [&root, &compacted] { + let full = load_pointer_catalog_from_root(&layout, captured) + .await + .unwrap(); + let selected = load_pointer_catalog_lookup_from_root(&layout, captured) + .await + .unwrap() + .files(layout.store(), std::slice::from_ref(&file)) + .await + .unwrap(); + assert_eq!(selected.files().get(&file), full.files().get(&file)); + winners.push(selected.files()[&file].shard_hash().to_owned()); + } + // A checkpoint is an authority boundary, not a transparent reader shortcut. + // It freezes the old replay base even when later visible refs supply an old tip. + assert_eq!( + (before.files()[&file].shard_hash().to_owned(), winners), + (borrowed_shard.clone(), vec![original_shard, borrowed_shard]) + ); +} + +#[tokio::test] +async fn independent_visibility_can_reorder_prior_borrowing_controls() { + let fixture = borrowing_fixture().await; + let original_id = fixture.original.id().unwrap(); + let borrowed_id = fixture.borrowed.id().unwrap(); + let earlier_id = fixture.earlier.id().unwrap(); + let mut paged = BTreeMap::new(); + for pointer in fixture.pointers.values() { + let (_, mut controls) = load_capsule_run_git_control(&fixture.layout, pointer) + .await + .unwrap(); + let control = controls.remove(0); + paged.insert(control.transaction_id().to_owned(), control); + } + for (is_paged, controls) in [(false, fixture.controls), (true, paged)] { + for with_earlier in [false, true] { + let index = GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new()) + .unwrap(); + let visibility = if is_paged { + VisibilityReconciliation::Paged(Box::new( + crate::capsule_protocol::CapturedVisibility::new( + fixture.layout.store().clone(), + index, + std::iter::empty(), + ), + )) + } else { + VisibilityReconciliation::Complete(index) + }; + let pending = controls + .iter() + .filter(|(id, _)| with_earlier || **id != earlier_id) + .map(|(id, c)| (id.clone(), c.clone())) + .collect(); + let ordered = order_ref_controls( + &BTreeMap::new(), + &BTreeMap::new(), + visibility, + &BTreeMap::new(), + pending, + ) + .await + .unwrap(); + let expected = if with_earlier { + vec![ + earlier_id.as_str(), + borrowed_id.as_str(), + original_id.as_str(), + ] + } else { + vec![original_id.as_str(), borrowed_id.as_str()] + }; + // Another visible producer can release an old borrowing control sooner; + // physical run presence alone must never change the captured selection. + assert_eq!( + ordered + .iter() + .map(CapsuleControl::transaction_id) + .collect::>(), + expected, + "paged={is_paged}, with_earlier={with_earlier}" + ); + } + } +} diff --git a/crates/crab-metadata/src/error.rs b/crates/crab-metadata/src/error.rs index 958e35629..6fb201f7b 100644 --- a/crates/crab-metadata/src/error.rs +++ b/crates/crab-metadata/src/error.rs @@ -6,6 +6,9 @@ pub type Result = std::result::Result; /// Errors raised by metadata schema and local index helpers. #[derive(thiserror::Error, Debug)] pub enum MetadataError { + /// Checkpoint metadata and its catalog exceed the caller's aggregate read budget. + #[error("checkpoint metadata uses {bytes} encoded bytes, exceeding its {maximum}-byte limit")] + CheckpointSize { bytes: u64, maximum: u64 }, /// A constructed path-state index cannot be admitted by its intended reader. #[error("path-state index uses {bytes} encoded bytes, exceeding its {maximum}-byte limit")] PathStateSize { bytes: u64, maximum: u64 }, diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs index 3c4efd60e..9634c7d6b 100644 --- a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -73,6 +73,7 @@ pub(super) async fn capture( .unwrap(), ) .unwrap(); + let catalog_bytes = catalog.encode().unwrap(); let checkpoint = LayeredCheckpoint::build( base.root().generation(), base.digest(), @@ -81,6 +82,15 @@ pub(super) async fn capture( None, ) .unwrap(); + if let Some(source) = checkpoint.catalog() { + // Seed immutable fixture data without charging a publication readback + // to the reader-construction and selected-query measurements below. + layout + .store() + .put_overwrite(&layout.capsule_catalog_path(source.hash()), catalog_bytes) + .await + .unwrap(); + } let pointer = CheckpointPointer::new_layered( checkpoint.hash(), checkpoint.bytes().len() as u64, @@ -142,7 +152,10 @@ pub(super) async fn capture( .unwrap(); ( control.pointer_catalog_lookup(layout, maximum).unwrap(), - path, + checkpoint + .catalog() + .map(|catalog| layout.capsule_catalog_path(catalog.hash())) + .unwrap_or(path), root, ) } diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 1272dd352..225d5b0c7 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -782,6 +782,21 @@ impl CapsuleRepositoryView { Ok(lookup.with_run_sources(sources.clone(), &order)?) } + /// Report authenticated catalog deltas not yet folded into the checkpoint. + pub fn has_catalog_deltas(&self) -> Result { + self.catalog_sources + .as_ref() + .ok_or(ReadError::IncompleteCatalogView)?; + Ok(self.capsules.iter().any(|capsule| { + capsule.sections().iter().any(|section| { + section.kind() == crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta + }) + }) || self + .capsule_controls + .iter() + .any(CapsuleControl::has_catalog_delta)) + } + /// Materialize the complete generation-pinned external pointer catalog. /// /// Fetch-only captures return `ReadError::IncompleteCatalogView`. @@ -789,23 +804,20 @@ impl CapsuleRepositoryView { self.catalog_sources .as_ref() .ok_or(ReadError::IncompleteCatalogView)?; - let base_catalog = - match self.checkpoint.as_ref() { - Some(checkpoint) if checkpoint.is_control_only() => { - let layout = self.metadata_layout.as_ref().ok_or_else(|| { - ReadError::internal("catalog materialization has no captured origin") - })?; - let pointer = - self.root.record().root().checkpoint().ok_or_else(|| { - ReadError::internal("captured checkpoint has no pointer") - })?; - crab_metadata::capsule_protocol::load_layered_checkpoint(layout, pointer) - .await? - .pointer_catalog()? - } - Some(checkpoint) => checkpoint.pointer_catalog()?, - None => PointerCatalog::new(), - }; + let base_catalog = match self.checkpoint.as_ref() { + Some(checkpoint) => { + let layout = self.metadata_layout.as_ref().ok_or_else(|| { + ReadError::internal("catalog materialization has no captured origin") + })?; + checkpoint + .pointer_catalog_from_store( + layout, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await? + } + None => PointerCatalog::new(), + }; let mut catalog = PointerCatalogAssembly::new(base_catalog); for capsule in &self.capsules { if let Some(delta) = capsule.pointer_catalog_delta()? { @@ -4288,7 +4300,7 @@ pub async fn open_view_from_root_for_repack( .await } -/// Load complete checkpoint catalog/visibility metadata and frontier controls. +/// Load complete checkpoint visibility, catalog commitments and frontier controls. /// /// Pack bodies remain in their immutable sources. Before the first checkpoint, /// this view retains complete capsules for callers using the inline Git reader. @@ -4331,14 +4343,16 @@ pub async fn open_compacted_view_from_root( .checkpoint() .ok_or_else(|| ReadError::internal("compacted view requires a layered checkpoint"))?; let checkpoint = load_layered_checkpoint(router, pointer, limits).await?; - compacted_view_from_checkpoint(snapshot, checkpoint, limits) + compacted_view_from_checkpoint(router, snapshot, checkpoint, limits) } /// Bind an already loaded complete checkpoint to its exact committed root. /// /// This performs the same pointer and visibility validation as a stored read, -/// without I/O. It excludes newer ref heads and rejects footer-only metadata. +/// without I/O. It excludes newer ref heads and rejects footer-only metadata; +/// catalog bodies remain lazy and bound to the supplied origin. pub fn compacted_view_from_checkpoint( + router: &StoreLayout, snapshot: crab_metadata::capsule_protocol::RootSnapshot, checkpoint: LayeredCheckpoint, limits: CapsuleReadLimits, @@ -4356,7 +4370,7 @@ pub fn compacted_view_from_checkpoint( "compacted view requires the complete checkpoint authenticated by its root", )); } - if pointer.size() > limits.max_capsule_bytes { + if checkpoint.encoded_metadata_bytes()? > limits.max_capsule_bytes { return Err(ReadError::CapsuleReadLimit { resource: "layered checkpoint bytes", maximum: limits.max_capsule_bytes, @@ -4378,7 +4392,7 @@ pub fn compacted_view_from_checkpoint( capsules: Vec::new(), capsule_controls: Vec::new(), tip_bound_transitions, - metadata_layout: None, + metadata_layout: Some(router.clone()), tip_bound_controls: Vec::new(), visible_ref_transactions: BTreeMap::new(), ref_capsule_counts: BTreeMap::new(), @@ -5242,9 +5256,21 @@ async fn load_layered_checkpoint( maximum: limits.max_capsule_bytes, }); } - crab_metadata::capsule_protocol::load_layered_checkpoint(router, pointer) - .await - .map_err(|error| corrupt_path("capsule-protocol layered checkpoint", error.to_string())) + crab_metadata::capsule_protocol::load_layered_checkpoint( + router, + pointer, + limits.max_capsule_bytes, + ) + .await + .map_err(|error| match error { + crab_metadata::error::MetadataError::CheckpointSize { maximum, .. } => { + ReadError::CapsuleReadLimit { + resource: "layered checkpoint bytes", + maximum, + } + } + error => error.into(), + }) } async fn load_layered_checkpoint_control( diff --git a/crates/crab-read/src/capsule_protocol/dependency_tests.rs b/crates/crab-read/src/capsule_protocol/dependency_tests.rs index 9bb0eaeb9..5d94fb7fd 100644 --- a/crates/crab-read/src/capsule_protocol/dependency_tests.rs +++ b/crates/crab-read/src/capsule_protocol/dependency_tests.rs @@ -438,6 +438,7 @@ async fn git_view_catalog_materialization_uses_the_captured_checkpoint_after_roo .unwrap(), ) .unwrap(); + checkpoint.publish_catalog(&layout).await.unwrap(); layout .store() .put( diff --git a/crates/crab-read/src/hydrator.rs b/crates/crab-read/src/hydrator.rs index d5e48f4d8..84bd21c0e 100644 --- a/crates/crab-read/src/hydrator.rs +++ b/crates/crab-read/src/hydrator.rs @@ -1273,6 +1273,7 @@ mod tests { None, ) .unwrap(); + checkpoint.publish_catalog(&router).await.unwrap(); router .store() .put_overwrite( diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index f93487e31..bfe0af437 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -198,10 +198,10 @@ pub async fn maintain_capsule_repository_from_view( let logical_due = count >= u64::from(threshold) && (count > 0 || view.layered_checkpoint().is_none()); let logical = if logical_due { - publish_checkpoint_with_catalog( + publish_checkpoint( layout, view, - view.pointer_catalog().await?, + catalog_for_checkpoint(view).await?, maximum_bytes, cancel, ) @@ -225,7 +225,9 @@ pub async fn maintain_capsule_repository_from_view( }); let compacted = retained .map(|(root, checkpoint)| { - crab_read::capsule_protocol::compacted_view_from_checkpoint(root, checkpoint, limits) + crab_read::capsule_protocol::compacted_view_from_checkpoint( + layout, root, checkpoint, limits, + ) }) .transpose()?; let physical = match compacted.as_ref() { @@ -237,7 +239,9 @@ pub async fn maintain_capsule_repository_from_view( let published = physical .committed .map(|(root, checkpoint)| { - crab_read::capsule_protocol::compacted_view_from_checkpoint(root, checkpoint, limits) + crab_read::capsule_protocol::compacted_view_from_checkpoint( + layout, root, checkpoint, limits, + ) }) .transpose()?; let inventory = published @@ -295,7 +299,7 @@ pub async fn repack_capsule_repository_from_view( view, consolidation.sources, member_oids, - view.pointer_catalog().await?, + catalog_for_checkpoint(view).await?, maximum_bytes, cancel, ) @@ -308,10 +312,10 @@ pub async fn repack_capsule_repository_from_view( committed, }; } else if logical_due { - publication = publish_checkpoint_with_catalog( + publication = publish_checkpoint( layout, view, - view.pointer_catalog().await?, + catalog_for_checkpoint(view).await?, maximum_bytes, cancel, ) @@ -326,7 +330,9 @@ pub async fn repack_capsule_repository_from_view( let published = publication .committed .map(|(root, checkpoint)| { - crab_read::capsule_protocol::compacted_view_from_checkpoint(root, checkpoint, limits) + crab_read::capsule_protocol::compacted_view_from_checkpoint( + layout, root, checkpoint, limits, + ) }) .transpose()?; let inventory = published.as_ref().unwrap_or(view); @@ -342,6 +348,8 @@ pub async fn repack_capsule_repository_from_view( /// Reusing the caller's pinned view avoids a second mutable-root and ref-head /// capture. Publication still compares against that exact root, so concurrent /// writers either win cleanly or leave this maintenance pass as a no-op. +/// The view must retain complete pack-member admission from the metadata entry +/// point; a Git query view deliberately omits that maintenance proof. pub async fn publish_capsule_checkpoint_from_view( layout: &StoreLayout, view: &crab_read::capsule_protocol::CapsuleRepositoryView, @@ -353,9 +361,12 @@ pub async fn publish_capsule_checkpoint_from_view( if view.capsule_count()? < u64::from(threshold) { return Ok(CheckpointOutcome::default()); } - let catalog = view.pointer_catalog().await?; - publish_capsule_checkpoint_with_catalog_from_view(layout, view, catalog, maximum_bytes, cancel) - .await + let catalog = catalog_for_checkpoint(view).await?; + Ok( + publish_checkpoint(layout, view, catalog, maximum_bytes, cancel) + .await? + .work, + ) } /// Publish one checkpoint from a pinned view with a replacement pointer catalog. @@ -370,17 +381,41 @@ pub async fn publish_capsule_checkpoint_with_catalog_from_view( maximum_bytes: u64, cancel: &CancellationToken, ) -> Result { - Ok( - publish_checkpoint_with_catalog(layout, view, catalog, maximum_bytes, cancel) - .await? - .work, + Ok(publish_checkpoint( + layout, + view, + CatalogWrite::New(catalog), + maximum_bytes, + cancel, ) + .await? + .work) +} + +enum CatalogWrite { + New(PointerCatalog), + Retained(Option), +} + +async fn catalog_for_checkpoint( + view: &crab_read::capsule_protocol::CapsuleRepositoryView, +) -> Result { + if view.has_catalog_deltas()? { + return Ok(CatalogWrite::New(view.pointer_catalog().await?)); + } + // The checkpoint freezes prior replay precedence. Reuse only when every + // newly selected control lacks a catalog delta, even if other refs moved. + Ok(CatalogWrite::Retained( + view.layered_checkpoint() + .and_then(LayeredCheckpoint::catalog) + .cloned(), + )) } -async fn publish_checkpoint_with_catalog( +async fn publish_checkpoint( layout: &StoreLayout, view: &crab_read::capsule_protocol::CapsuleRepositoryView, - catalog: PointerCatalog, + catalog: CatalogWrite, maximum_bytes: u64, cancel: &CancellationToken, ) -> Result { @@ -486,7 +521,7 @@ async fn repack_checkpoint_from_view( let Some(selected_start) = layered_suffix_start(checkpoint.sources())? else { return Ok(CheckpointPublication::default()); }; - let catalog = checkpoint.pointer_catalog()?; + let catalog = CatalogWrite::Retained(checkpoint.catalog().cloned()); let consolidation = consolidate_layered_suffix( layout, view, @@ -520,7 +555,7 @@ async fn publish_layered_sources( view: &crab_read::capsule_protocol::CapsuleRepositoryView, sources: Vec, member_oids: BTreeMap>>, - catalog: PointerCatalog, + catalog: CatalogWrite, maximum_bytes: u64, cancel: &CancellationToken, ) -> Result< @@ -543,14 +578,21 @@ async fn publish_layered_sources( &catalog_digest, member_admission, )?; - let checkpoint = crab_metadata::capsule_protocol::LayeredCheckpoint::build( - view.root().root().generation(), - view.root().digest(), - sources, - catalog, - Some(visibility), - )?; - if checkpoint.bytes().len() as u64 > maximum_bytes && maximum_bytes > 0 { + let generation = view.root().root().generation(); + let root_digest = view.root().digest(); + let checkpoint = match catalog { + CatalogWrite::New(catalog) => { + LayeredCheckpoint::build(generation, root_digest, sources, catalog, Some(visibility))? + } + CatalogWrite::Retained(catalog) => LayeredCheckpoint::build_reusing_catalog( + generation, + root_digest, + sources, + catalog, + Some(visibility), + )?, + }; + if checkpoint.encoded_metadata_bytes()? > maximum_bytes && maximum_bytes > 0 { return Err(CheckpointError::OutputLimit { maximum: maximum_bytes, }); diff --git a/crates/crab-remote/tests/checkpoint.rs b/crates/crab-remote/tests/checkpoint.rs index b3030b58f..93d8cba40 100644 --- a/crates/crab-remote/tests/checkpoint.rs +++ b/crates/crab-remote/tests/checkpoint.rs @@ -100,6 +100,7 @@ async fn retained_checkpoint_view_requires_complete_metadata_and_exact_root_bind let layered = checkpointed.layered_checkpoint().unwrap(); publish_blob(&layout, "later").await; let retained = crab_read::capsule_protocol::compacted_view_from_checkpoint( + &layout, checkpointed.root_snapshot().clone(), layered.clone(), LIMITS, @@ -133,6 +134,7 @@ async fn retained_checkpoint_view_requires_complete_metadata_and_exact_root_bind .unwrap(); assert!( crab_read::capsule_protocol::compacted_view_from_checkpoint( + &layout, checkpointed.root_snapshot().clone(), footer, LIMITS, @@ -141,6 +143,7 @@ async fn retained_checkpoint_view_requires_complete_metadata_and_exact_root_bind ); assert!(matches!( crab_read::capsule_protocol::compacted_view_from_checkpoint( + &layout, checkpointed.root_snapshot().clone(), layered.clone(), CapsuleReadLimits { @@ -153,6 +156,7 @@ async fn retained_checkpoint_view_requires_complete_metadata_and_exact_root_bind let newer = checkpoint(&layout).await; assert!( crab_read::capsule_protocol::compacted_view_from_checkpoint( + &layout, newer.root_snapshot().clone(), layered.clone(), LIMITS, diff --git a/crates/crab-remote/tests/checkpoint/publication_catalog.rs b/crates/crab-remote/tests/checkpoint/publication_catalog.rs index b1493ff7c..0aa6569b4 100644 --- a/crates/crab-remote/tests/checkpoint/publication_catalog.rs +++ b/crates/crab-remote/tests/checkpoint/publication_catalog.rs @@ -289,3 +289,71 @@ async fn open_history_view(layout: &StoreLayout) -> CapsuleRepositoryView .await .unwrap() } + +#[tokio::test(flavor = "multi_thread")] +async fn ref_only_checkpoints_reuse_catalog_without_body_io() { + let (layout, observations) = empty_fixture().await; + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(1, Vec::new())) + .unwrap(); + for index in 0..10_000 { + catalog + .insert_file( + format!("{index:064x}"), + FileCatalogEntry::new(1, "a".repeat(64)), + ) + .unwrap(); + } + assert!(catalog.encode().unwrap().len() > 64 * 1024); + publish_history(&layout, catalog.clone()).await; + // Publication needs exact pack-member admission. The maintenance view + // supplies it while checkpoint catalog bodies remain cold. + let first = view(&layout).await; + crab_remote::checkpoint::publish_capsule_checkpoint_from_view( + &layout, + &first, + 0, + 64 * LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + let first = open_history_view(&layout).await; + let original = first.layered_checkpoint().unwrap(); + let identity = original.catalog().unwrap().clone(); + publish_blob(&layout, "unrelated").await; + let before = view(&layout).await; + assert!(!before.has_catalog_deltas().unwrap()); + observations.0.lock().unwrap().clear(); + let result = crab_remote::checkpoint::publish_capsule_checkpoint_from_view( + &layout, + &before, + 0, + 64 * LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + let io = observations.0.lock().unwrap().clone(); + assert!(result.published); + assert!( + io.iter().map(|op| op.bytes_read).sum::() < 64 * 1024, + "catalog read on ref-only checkpoint: {io:?}" + ); + assert!( + io.iter().map(|op| op.bytes_written).sum::() < 64 * 1024, + "catalog rewritten on ref-only checkpoint: {io:?}" + ); + let after = open_history_view(&layout).await; + assert_eq!( + after.layered_checkpoint().unwrap().catalog(), + Some(&identity) + ); + layout + .store() + .delete(&layout.capsule_checkpoint_path(original.hash())) + .await + .unwrap(); + assert_eq!(after.pointer_catalog().await.unwrap(), catalog); +} diff --git a/crates/crab-storage/src/layout.rs b/crates/crab-storage/src/layout.rs index 0cc04761c..0f67b3ca6 100644 --- a/crates/crab-storage/src/layout.rs +++ b/crates/crab-storage/src/layout.rs @@ -214,6 +214,19 @@ impl StoreLayout { self.repo_path(&format!("v2/checkpoints/{partition}/{hash}")) } + /// Prefix containing immutable checkpoint catalog objects. + #[must_use] + pub fn capsule_catalog_prefix(&self) -> ObjectPath { + self.repo_path("v2/catalogs/") + } + + /// Path to an immutable checkpoint catalog object. + #[must_use] + pub fn capsule_catalog_path(&self, hash: &str) -> ObjectPath { + let partition = hash.get(..GLOBAL_CONTENT_FANOUT_WIDTH).unwrap_or(hash); + self.repo_path(&format!("v2/catalogs/{partition}/{hash}")) + } + /// Path to one immutable protocol-v2 geometric Git pack layer. #[must_use] pub fn capsule_pack_layer_path(&self, hash: &str) -> ObjectPath { diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index 737bf42be..9dc4a619b 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -4,8 +4,8 @@ use std::time::Instant; use bytes::Bytes; use crab_metadata::capsule_protocol::{ - Capsule, CapsulePointer, CapsuleRun, CapsuleTransaction, CheckpointPointer, HistorySegment, - HistorySegmentState, LayeredCheckpoint, RepositoryRoot, RootRecord, create_root, load_root, + Capsule, CapsulePointer, CapsuleRun, CapsuleTransaction, HistorySegment, HistorySegmentState, + LayeredCheckpoint, RepositoryRoot, RootRecord, create_root, load_root, }; use crab_storage::{ETag, StorageError, Store, StoreLayout}; use futures_util::future::try_join_all; @@ -1589,20 +1589,6 @@ struct CheckpointRefState { capsule_runs: Vec, } -fn checkpoint_pointer(checkpoint: &LayeredCheckpoint) -> Result { - Ok(CheckpointPointer::new_layered( - checkpoint.hash(), - checkpoint.bytes().len() as u64, - checkpoint.control_offset(), - checkpoint.control_size(), - checkpoint.footer_hash(), - checkpoint.covered_generation(), - checkpoint.covered_root_digest(), - checkpoint.pack_count()?, - checkpoint.object_count()?, - )?) -} - async fn publish_checkpoint_inner( router: &StoreLayout, base: RootSnapshot, @@ -1623,7 +1609,7 @@ async fn publish_checkpoint_inner( reason: "checkpoint does not cover the exact CAS base".to_owned(), }); } - let pointer = checkpoint_pointer(checkpoint)?; + let pointer = checkpoint.pointer()?; if let Some(state) = &ref_state { let retained_transactions = state .capsule_runs @@ -1679,6 +1665,9 @@ async fn publish_checkpoint_inner( }) .transpose()?; let path = router.capsule_checkpoint_path(checkpoint.hash()); + // The checkpoint advertises immutable catalog roots. Their bytes must be + // verified and durable before either checkpoint or history can expose them. + checkpoint.publish_catalog(router).await?; if let Some(history) = &history { let history_path = router.capsule_history_segment_path(history.hash()); tokio::try_join!( @@ -1838,7 +1827,8 @@ pub async fn restore_checkpoint( reason: "restore checkpoint visibility does not authenticate its ref tips".to_owned(), }); } - let pointer = checkpoint_pointer(checkpoint)?; + let pointer = checkpoint.pointer()?; + checkpoint.publish_catalog(router).await?; router .store() .put_if_absent_verified( @@ -3142,6 +3132,7 @@ mod tests { let stored = crab_metadata::capsule_protocol::load_layered_checkpoint( &router, checkpointed.record().root().checkpoint().unwrap(), + u64::MAX, ) .await .unwrap(); @@ -3303,7 +3294,7 @@ mod tests { ) .await .unwrap(); - assert_eq!(restored.record().root().checkpoint().unwrap().format(), 7); + assert_eq!(restored.record().root().checkpoint().unwrap().format(), 8); assert_eq!( restored.record().root().generation(), fenced.record().root().generation() + 1 @@ -3404,6 +3395,7 @@ mod tests { let stored = crab_metadata::capsule_protocol::load_layered_checkpoint( &router, segments[0].checkpoint(), + u64::MAX, ) .await .unwrap(); @@ -3468,6 +3460,7 @@ mod tests { let stored = crab_metadata::capsule_protocol::load_layered_checkpoint( &router, segments[0].checkpoint(), + u64::MAX, ) .await .unwrap(); @@ -3600,6 +3593,151 @@ mod tests { assert_eq!(visible.visible.oid(), Some("3".repeat(40).as_str())); } + #[tokio::test] + async fn catalog_write_failure_cannot_publish_checkpoint_or_restore() { + use crab_metadata::capsule_protocol::{ + FileCatalogEntry, GcFence, LayeredVisibilitySnapshot, PointerCatalog, + ShardCatalogEntry, source_catalog_digest, + }; + for restore in [false, true] { + let inner = Arc::new(InMemory::new()); + let seed = StoreLayout::new( + Store::new(inner.clone()), + "repositories/catalog-barrier".to_owned(), + ); + let mut base = initialize(&seed, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let transaction = transaction(&base, None, &"2".repeat(40)); + let run = CapsuleRun::leaf(capsule(&transaction)).unwrap(); + let source = + crab_metadata::capsule_protocol::PackSourceDescriptor::from_capsule_run(&run) + .unwrap(); + seed.store() + .put_if_absent_verified(&seed.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + // Generation zero forbids checkpoint state. Establish a valid + // generation before testing replacement and restore barriers. + let initial = LayeredCheckpoint::build( + 0, + base.record().digest(), + vec![source.clone()], + PointerCatalog::new(), + None, + ) + .unwrap(); + base = publish_ref_layered_checkpoint( + &seed, + base, + &initial, + Default::default(), + Default::default(), + Default::default(), + Vec::new(), + ) + .await + .unwrap(); + if restore { + base = begin_gc( + &seed, + base, + GcFence::new(&"e".repeat(64), u64::MAX).unwrap(), + ) + .await + .unwrap(); + } + let visibility = crab_metadata::git_visibility::GitVisibilityIndex::new( + 0, + "a".repeat(64), + "b".repeat(64), + Default::default(), + ) + .unwrap(); + let visibility = LayeredVisibilitySnapshot::from_index( + &visibility, + &source_catalog_digest(std::slice::from_ref(&source)).unwrap(), + ) + .unwrap(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(1, Vec::new())) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + base.record().root().generation(), + base.record().digest(), + vec![source], + catalog.clone(), + Some(visibility), + ) + .unwrap(); + let fault = Arc::new(LostHeadReplyStore { + inner, + head_path: seed + .capsule_catalog_path(checkpoint.catalog().unwrap().hash()) + .to_string(), + lost: AtomicBool::new(true), + reject: AtomicBool::new(true), + }); + let router = StoreLayout::new( + Store::with_retry( + fault.clone(), + crab_storage::RetryPolicy { + max_attempts: 1, + base: std::time::Duration::ZERO, + cap: std::time::Duration::ZERO, + }, + ), + seed.repo_prefix().to_owned(), + ); + let publish = |base| async { + if restore { + restore_checkpoint( + &router, + base, + &checkpoint, + Default::default(), + Default::default(), + "refs/heads/main".to_owned(), + ) + .await + } else { + publish_layered_checkpoint(&router, base, &checkpoint).await + } + }; + assert!(publish(base.clone()).await.is_err(), "restore={restore}"); + assert_eq!( + open_root(&seed).await.unwrap().record().digest(), + base.record().digest() + ); + assert!(matches!( + seed.store() + .head(&seed.capsule_checkpoint_path(checkpoint.hash())) + .await, + Err(StorageError::NotFound { .. }) + )); + fault.reject.store(false, Ordering::Release); + let published = publish(base).await.unwrap(); + let loaded = crab_metadata::capsule_protocol::load_layered_checkpoint( + &seed, + published.record().root().checkpoint().unwrap(), + u64::MAX, + ) + .await + .unwrap(); + assert_eq!( + loaded + .pointer_catalog_from_store(&seed, u64::MAX) + .await + .unwrap(), + catalog + ); + } + } + #[tokio::test] async fn lost_ref_head_update_reply_reconciles_as_committed_success() { let inner = Arc::new(InMemory::new()); From f1368a9826314407d949a723181b41423f6517dd Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:14:14 -0700 Subject: [PATCH 096/127] fix(errors): retain checkpoint admission failures --- crab/src/core/error.rs | 29 +++++++++++++++++++++++++++ crates/crab-auth-server/src/error.rs | 30 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index 88963550d..7f443d104 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -2046,6 +2046,7 @@ impl From for CrabError { | crab_metadata::error::MetadataError::FileLookupWorker { .. } | crab_metadata::error::MetadataError::CapsuleContract { .. } | crab_metadata::error::MetadataError::PathStateSize { .. } + | crab_metadata::error::MetadataError::CheckpointSize { .. } | crab_metadata::error::MetadataError::BrowseIndexRecord { .. } | crab_metadata::error::MetadataError::SourceRetentionRecord { .. } | crab_metadata::error::MetadataError::GitObjectIndexRecord { .. } @@ -4160,6 +4161,34 @@ mod tests { ); } + #[test] + fn checkpoint_admission_error_survives_direct_and_read_boundaries() { + for through_read in [false, true] { + let source = crab_metadata::error::MetadataError::CheckpointSize { + bytes: 1024, + maximum: 512, + }; + let error = if through_read { + CrabError::from(crab_read::ReadError::Metadata(source)) + } else { + CrabError::from(source) + }; + let CrabError::Io(error) = error else { + panic!("expected typed checkpoint admission error"); + }; + let source = error + .get_ref() + .and_then(|source| source.downcast_ref::()); + assert!(matches!( + source, + Some(crab_metadata::error::MetadataError::CheckpointSize { + bytes: 1024, + maximum: 512, + }) + )); + } + } + #[test] fn capsule_protocol_contract_error_retains_its_source() { let error = CrabError::from(crab_metadata::error::MetadataError::CapsuleContract { diff --git a/crates/crab-auth-server/src/error.rs b/crates/crab-auth-server/src/error.rs index 52d7b3b8a..c41f9faf1 100644 --- a/crates/crab-auth-server/src/error.rs +++ b/crates/crab-auth-server/src/error.rs @@ -154,6 +154,7 @@ impl From for AuthServerError { } error @ (crab_metadata::error::MetadataError::FileLookupAdmission { .. } | crab_metadata::error::MetadataError::FileLookupWorker { .. } + | crab_metadata::error::MetadataError::CheckpointSize { .. } | crab_metadata::error::MetadataError::FileLookupLimit { .. } | crab_metadata::error::MetadataError::PlanAlreadyAttempted { .. } | crab_metadata::error::MetadataError::RefJournalCommitUncertain { .. }) => { @@ -224,6 +225,7 @@ impl From for AuthServerError { crab_read::ReadError::Metadata( source @ (crab_metadata::error::MetadataError::FileLookupAdmission { .. } | crab_metadata::error::MetadataError::FileLookupWorker { .. } + | crab_metadata::error::MetadataError::CheckpointSize { .. } | crab_metadata::error::MetadataError::FileLookupLimit { .. } | crab_metadata::error::MetadataError::RefJournalCommitUncertain { .. }), ) => Self::from(source), @@ -370,6 +372,34 @@ mod tests { } } + #[test] + fn checkpoint_admission_error_survives_direct_and_read_boundaries() { + for through_read in [false, true] { + let source = crab_metadata::error::MetadataError::CheckpointSize { + bytes: 1024, + maximum: 512, + }; + let error = if through_read { + AuthServerError::from(crab_read::ReadError::Metadata(source)) + } else { + AuthServerError::from(source) + }; + let AuthServerError::Io(error) = error else { + panic!("expected typed checkpoint admission error"); + }; + let source = error + .get_ref() + .and_then(|source| source.downcast_ref::()); + assert!(matches!( + source, + Some(crab_metadata::error::MetadataError::CheckpointSize { + bytes: 1024, + maximum: 512, + }) + )); + } + } + #[tokio::test] async fn metadata_lookup_errors_survive_direct_and_read_wrapping() { for through_read in [false, true] { From b9fdd7005c40b389d6f5bcd4e6469d4acb9556ad Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:06:35 -0700 Subject: [PATCH 097/127] fix(remote): own ref visibility across deletion Publish captured replacement closures for new refs and cross-ref bases. Preserve same-ref additive evidence. Remove deleted closures in every reader and prove both deletion orderings against native Git. --- crates/crab-metadata/README.md | 4 + .../capsule_protocol/captured_visibility.rs | 18 +- .../captured_visibility/tests.rs | 45 ++++ .../store/catalog_order_tests.rs | 138 +++++++++++- .../src/capsule_protocol/store/tests.rs | 42 ++-- crates/crab-read/README.md | 4 +- crates/crab-read/src/capsule_protocol.rs | 109 ++++++++- crates/crab-remote/README.md | 3 + crates/crab-remote/src/prepare.rs | 59 +++++ .../crab-remote/tests/capsule_preparation.rs | 73 +++--- .../capsule_preparation/ref_visibility.rs | 207 ++++++++++++++++++ 11 files changed, 637 insertions(+), 65 deletions(-) create mode 100644 crates/crab-remote/tests/capsule_preparation/ref_visibility.rs diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 3a65c046b..d0bff8872 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -105,6 +105,10 @@ Captured revision, remote-mount and current-state hydration use the shared paged lookup. Control capture and selected closure queries are measured separately; complete opening and history-amplification qualification remain required. +Ref deletion removes its live visibility closure even when the capsule has no +visibility payload. Retained dictionary objects cannot supply a new ref's base; +complete and paged reconciliation enforce the same deletion boundary. + External pointer catalogs use `CRBPCP01` authenticated pages. A bounded descriptor commits file, shard, xorb and reverse-chunk directories; nested shard closures and ordered xorb chunk lists are paged too. Keys preserve canonical lowercase Xet hex diff --git a/crates/crab-metadata/src/capsule_protocol/captured_visibility.rs b/crates/crab-metadata/src/capsule_protocol/captured_visibility.rs index 60dd7d5cb..f4b2ed08e 100644 --- a/crates/crab-metadata/src/capsule_protocol/captured_visibility.rs +++ b/crates/crab-metadata/src/capsule_protocol/captured_visibility.rs @@ -42,7 +42,16 @@ impl VisibilityReconciliation { index, ) } - Self::Complete(_) => Ok(()), + Self::Complete(index) => { + // A deletion-only capsule has no visibility payload. Its ref + // closure must still disappear before another control borrows it. + for edit in control.transaction().edits() { + if edit.new_oid().is_none() { + index.remove_ref(edit.ref_name()); + } + } + Ok(()) + } Self::Paged(index) => index.apply(control).await, } } @@ -335,6 +344,13 @@ impl CapturedVisibility { /// Apply one authenticated control without widening to mutable ref state. pub async fn apply(&mut self, control: &CapsuleControl) -> Result<()> { let Some((reader, headers)) = self.headers(control).await? else { + // No payload skips OID changes, not deletion of an entire ref. + // Keep historical dictionary entries separate from live closures. + for edit in control.transaction().edits() { + if edit.new_oid().is_none() { + self.refs.remove(edit.ref_name()); + } + } return Ok(()); }; for (edit, header) in control.transaction().edits().iter().zip(headers) { diff --git a/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs b/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs index e7cd4593b..164fa3215 100644 --- a/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/captured_visibility/tests.rs @@ -169,6 +169,51 @@ async fn removed_invisible_objects_do_not_supply_new_ref_bases() { ); } +#[tokio::test] +async fn deleted_ref_objects_remain_in_dictionary_but_cannot_supply_new_bases() { + let layout = StoreLayout::new( + Store::new(Arc::new(InMemory::new())), + "repositories/deleted".into(), + ); + let mut lazy = CapturedVisibility::new(layout.store().clone(), initial(), std::iter::empty()); + let create = control( + &layout, + "refs/heads/main", + None, + GitVisibilityEdit::from_replacement_objects(None, hex(1), vec![hex(1), hex(3)]), + ) + .await; + lazy.apply(&create).await.unwrap(); + let transaction = CapsuleTransaction::new( + &"1".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + Some(hex(1)), + None, + None, + )], + ) + .unwrap(); + let run = + CapsuleRun::leaf(Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap()).unwrap(); + let (_, mut controls) = run.verified_controls().unwrap(); + lazy.apply(&controls.remove(0)).await.unwrap(); + let branch = control( + &layout, + "refs/heads/feature", + None, + GitVisibilityEdit::from_delta_objects(Some(hex(3)), hex(4), vec![hex(4)], vec![]), + ) + .await; + assert_eq!( + ( + lazy.dictionary_contains(&[object(3)]).await.unwrap(), + lazy.ready(&branch, &BTreeMap::new()).await.unwrap() + ), + (vec![true], false) + ); +} + #[tokio::test] async fn removal_outside_captured_prior_closure_is_rejected() { let layout = StoreLayout::new( diff --git a/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs b/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs index 21a6d0c48..32e4e5d48 100644 --- a/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/store/catalog_order_tests.rs @@ -60,6 +60,14 @@ struct BorrowingFixture { borrowed_shard: String, } +// Fixed publication identities keep ordering coverage independent of nonce sampling. +fn ordering_transaction(root: &RootSnapshot, edit: CapsuleRefEdit) -> CapsuleTransaction { + let edits = vec![edit]; + let identity = blake3::hash(&serde_json::to_vec(&edits).unwrap()); + CapsuleTransaction::for_protected_source(root.record().digest(), &identity.to_hex(), edits) + .unwrap() +} + async fn borrowing_fixture() -> BorrowingFixture { let layout = StoreLayout::new( Store::new(Arc::new(object_store::memory::InMemory::new())), @@ -74,15 +82,14 @@ async fn borrowing_fixture() -> BorrowingFixture { .unwrap(); let tip = "1".repeat(40); let transaction = |name: String| { - CapsuleTransaction::new( - root.record().digest(), - vec![CapsuleRefEdit::new(name, None, Some(tip.clone()), None)], + ordering_transaction( + &root, + CapsuleRefEdit::new(name, None, Some(tip.clone()), None), ) - .unwrap() }; let borrowed = (0..64) .map(|i| transaction(format!("refs/heads/borrow-{i}"))) - .find(|t| t.id().unwrap().as_bytes()[0] < b'8') + .find(|t| (b'4'..b'8').contains(&t.id().unwrap().as_bytes()[0])) .unwrap(); let borrowed_id = borrowed.id().unwrap(); let mut producers = (0..256) @@ -454,3 +461,124 @@ async fn independent_visibility_can_reorder_prior_borrowing_controls() { } } } + +#[tokio::test] +async fn deleting_a_visible_producer_can_delay_borrowing_controls() { + let mut fixture = borrowing_fixture().await; + let borrowed_id = fixture.borrowed.id().unwrap(); + let original_id = fixture.original.id().unwrap(); + let tip = fixture.original.edits()[0].new_oid().unwrap().to_owned(); + let (producer, deletion) = (0..16_384) + .find_map(|i| { + let name = format!("refs/heads/transient-{i}"); + let producer = ordering_transaction( + &fixture.root, + CapsuleRefEdit::new(name.clone(), None, Some(tip.clone()), None), + ); + let deletion = ordering_transaction( + &fixture.root, + CapsuleRefEdit::new(name, Some(tip.clone()), None, None), + ); + let producer_id = producer.id().unwrap(); + let deletion_id = deletion.id().unwrap(); + (producer_id < deletion_id && deletion_id < borrowed_id).then_some((producer, deletion)) + }) + .unwrap(); + let producer_id = producer.id().unwrap(); + let deletion_id = deletion.id().unwrap(); + for (transaction, edits) in [ + ( + &producer, + BTreeMap::from([( + producer.edits()[0].ref_name().to_owned(), + GitVisibilityEdit::from_replacement_objects(None, tip.clone(), vec![tip.clone()]), + )]), + ), + (&deletion, BTreeMap::new()), + ] { + let sections = if edits.is_empty() { + Vec::new() + } else { + vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + CapsuleVisibilityDelta::new(edits) + .unwrap() + .encode() + .unwrap(), + )] + }; + let run = + CapsuleRun::leaf(Capsule::build(transaction, Vec::new(), sections).unwrap()).unwrap(); + fixture + .layout + .store() + .create_strict( + &fixture.layout.capsule_path(run.hash()), + run.bytes().clone(), + ) + .await + .unwrap(); + let (_, mut controls) = run.verified_controls().unwrap(); + let id = transaction.id().unwrap(); + fixture.controls.insert(id.clone(), controls.remove(0)); + fixture.pointers.insert( + id, + CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(), + ); + } + fixture.controls.remove(&fixture.earlier.id().unwrap()); + let mut paged = BTreeMap::new(); + for id in fixture.controls.keys() { + let (_, mut controls) = + load_capsule_run_git_control(&fixture.layout, &fixture.pointers[id]) + .await + .unwrap(); + paged.insert(id.clone(), controls.remove(0)); + } + let mut observed = Vec::new(); + for (is_paged, controls) in [(false, fixture.controls), (true, paged)] { + let index = + GitVisibilityIndex::new(0, "0".repeat(64), "0".repeat(64), BTreeMap::new()).unwrap(); + let visibility = if is_paged { + VisibilityReconciliation::Paged(Box::new( + crate::capsule_protocol::CapturedVisibility::new( + fixture.layout.store().clone(), + index, + std::iter::empty(), + ), + )) + } else { + VisibilityReconciliation::Complete(index) + }; + let ordered = order_ref_controls( + &BTreeMap::new(), + &BTreeMap::new(), + visibility, + &BTreeMap::new(), + controls, + ) + .await + .unwrap(); + // The transient source disappears before borrowing wins admission. + // Treating visibility as monotone would incorrectly put borrowing + // before the surviving producer and reverse their file-binding winner. + observed.push( + ordered + .iter() + .map(|control| control.transaction_id().to_owned()) + .collect::>(), + ); + } + let expected = vec![producer_id, deletion_id, original_id, borrowed_id]; + assert_eq!(observed, vec![expected.clone(), expected]); +} diff --git a/crates/crab-metadata/src/capsule_protocol/store/tests.rs b/crates/crab-metadata/src/capsule_protocol/store/tests.rs index 45a3729dd..7443a559b 100644 --- a/crates/crab-metadata/src/capsule_protocol/store/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/store/tests.rs @@ -589,35 +589,23 @@ async fn full_catalog_matches_selected_closure_across_independent_ref_dependenci ) .await .unwrap(); - let descriptors = CapsuleTransaction::new( - root.record().digest(), - vec![CapsuleRefEdit::new( - "refs/tags/source", - None, - Some("1".repeat(40)), - None, - )], - ) - .unwrap(); - let descriptor_id = descriptors.id().unwrap(); - let mut binding = None; - for index in 0..256 { - let candidate = CapsuleTransaction::new( + let transaction = |name, tip| { + CapsuleTransaction::new( root.record().digest(), - vec![CapsuleRefEdit::new( - format!("refs/tags/consumer-{index}"), - None, - Some("2".repeat(40)), - None, - )], + vec![CapsuleRefEdit::new(name, None, Some(tip), None)], ) - .unwrap(); - if candidate.id().unwrap() < descriptor_id { - binding = Some(candidate); - break; - } - } - let binding = binding.expect("fixture must put its dependent binding first in hash order"); + .unwrap() + }; + let left = transaction("refs/tags/left", "1".repeat(40)); + let right = transaction("refs/tags/right", "2".repeat(40)); + // Assign roles after nonce generation; a bounded search against one random + // descriptor could fail before testing dependency assembly at all. + let (binding, descriptors) = if left.id().unwrap() < right.id().unwrap() { + (left, right) + } else { + (right, left) + }; + assert!(binding.id().unwrap() < descriptors.id().unwrap()); let file = "3".repeat(64); let shard = "4".repeat(64); let xorb = "5".repeat(64); diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index eb53e0b23..b7e996abd 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -55,7 +55,9 @@ capsule-relative, while transport ranges stay on the captured run. Shared transactions deduplicate without discarding conflicting content evidence. Paged closure layers preserve the inherited base when its source ref later moves, and dictionary retention -never supplies visibility for a new ref. Initial rooted additions use their +never supplies visibility for a new ref. Deletion-only capsules remove their +live ref closure even without an OID payload, in both full-body and control reads. +Initial rooted additions use their authenticated count; differing actual delta additions/removals remain admitted work. `git_visibility_index`, `candidate_git_visibility` and `tip_bound_transitions` asynchronously materialize captured controls only when diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 225d5b0c7..d155ac857 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -4663,9 +4663,7 @@ async fn assemble_view( } let mut visibility_index = visibility_index_for_checkpoint(checkpoint.as_ref())?; for capsule in &root_capsules { - if capsule.visibility_delta()?.is_some() { - apply_capsule_visibility_index(capsule, &mut visibility_index)?; - } + reconcile_capsule_visibility(capsule, &mut visibility_index)?; } let ordered = order_ref_capsules( snapshot.record().root().refs(), @@ -5174,9 +5172,7 @@ fn order_ref_capsules( .remove(&ready) .ok_or_else(|| ReadError::internal("ready capsule disappeared"))?; apply_capsule_refs(&capsule, &mut refs, &mut peeled)?; - if capsule.visibility_delta()?.is_some() { - apply_capsule_visibility_index(&capsule, &mut visibility)?; - } + reconcile_capsule_visibility(&capsule, &mut visibility)?; applied.insert(capsule.transaction_id().to_owned()); ordered.push(capsule); } @@ -5212,6 +5208,27 @@ fn apply_capsule_visibility_index( .map_err(metadata_control_error) } +fn reconcile_capsule_visibility( + capsule: &Capsule, + index: &mut crab_metadata::git_visibility::GitVisibilityIndex, +) -> Result<()> { + let transaction = capsule.transaction()?; + let delta = capsule.visibility_delta()?; + if delta.is_some() { + return apply_transaction_visibility_index(&transaction, delta.as_ref(), index) + .map_err(metadata_control_error); + } + // Reconciliation permits metadata-only live edits; deep inspection still + // requires their evidence. Deletions revoke closures without an OID payload, + // or a later borrower could inherit a deleted ref. + for edit in transaction.edits() { + if edit.new_oid().is_none() { + index.remove_ref(edit.ref_name()); + } + } + Ok(()) +} + fn apply_capsule_refs( capsule: &Capsule, refs: &mut BTreeMap, @@ -7402,6 +7419,86 @@ mod tests { ); } + #[test] + fn full_ref_ordering_cannot_borrow_from_a_deleted_ref() { + let tip = "1".repeat(40); + let mut pending = BTreeMap::new(); + for (key, name, old, new, evidence) in [ + ( + "a-delete", + "refs/heads/transient", + Some(tip.clone()), + None, + None, + ), + ( + "b-borrow", + "refs/heads/borrow", + None, + Some(tip.clone()), + Some(GitVisibilityEdit::from_delta_objects( + Some(tip.clone()), + tip.clone(), + Vec::new(), + Vec::new(), + )), + ), + ( + "z-source", + "refs/heads/source", + None, + Some(tip.clone()), + Some(GitVisibilityEdit::from_replacement_objects( + None, + tip.clone(), + vec![tip.clone()], + )), + ), + ] { + let transaction = CapsuleTransaction::new( + &"a".repeat(64), + vec![CapsuleRefEdit::new(name, old, new, None)], + ) + .unwrap(); + let sections = evidence + .map(|edit| { + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + CapsuleVisibilityDelta::new(BTreeMap::from([(name.to_owned(), edit)])) + .unwrap() + .encode() + .unwrap(), + ) + }) + .into_iter() + .collect(); + pending.insert( + key.to_owned(), + Capsule::build(&transaction, Vec::new(), sections).unwrap(), + ); + } + let expected = ["a-delete", "z-source", "b-borrow"] + .map(|key| pending[key].transaction_id().to_owned()); + let ordered = order_ref_capsules( + &BTreeMap::from([("refs/heads/transient".to_owned(), tip.clone())]), + &BTreeMap::new(), + visibility_index(BTreeMap::from([( + "refs/heads/transient".to_owned(), + vec![tip], + )])), + &BTreeMap::new(), + pending, + ) + .unwrap(); + assert_eq!( + ordered + .iter() + .map(|capsule| capsule.transaction_id().to_owned()) + .collect::>(), + expected, + ); + } + #[test] fn ref_capsules_follow_authenticated_predecessors_when_oid_repeats() { let oid_a = "1".repeat(40); diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 5c391f6d2..0364edf60 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -70,6 +70,9 @@ verified create/read identities with their source catalog, publishes rebuildable chunk hints only after proven ref visibility, and preserves a committed outcome if hint-index maintenance fails. Uncertain publication does not publish hints. Caller-owned ref leases and GC fences cover this complete operation. +New refs and explicit cross-ref visibility bases publish complete replacement +closures from the captured proof. Only same-ref updates retain additive evidence; +a different ref's later deletion must not invalidate an acknowledged branch. `objects` owns bounded tree reads, deterministic tree edits and encoding, commit encoding, and Git object identities. `prepare` owns normalized packs and diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 7210a1e20..b96de1ab0 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -910,6 +910,47 @@ impl Source<'_> { } } + fn prior_objects(&self, maximum: usize) -> Result> { + let name = self.prior.as_ref().map(|(name, _)| name.as_str()); + let Some(name) = name else { + return Err(Error::Request("Additive visibility has no captured base")); + }; + match &self.proof { + Some(VisibilityProof::Materialized(proof)) => { + if proof.object_count_for_refs([name]) > maximum { + return Err(Error::Request( + "Visibility base exceeds the graph object limit", + )); + } + proof.objects_for_ref(name).ok_or(Error::Request( + "Visibility base is absent from its captured proof", + )) + } + Some(VisibilityProof::Catalog(proof)) => { + if proof.object_count_for_refs([name]) > maximum { + return Err(Error::Request( + "Visibility base exceeds the graph object limit", + )); + } + let ordinals = proof.ordinals_for_refs([name]); + let objects = self + .handle + .block_on(self.operation.catalog_object_ids_by_ordinal(&ordinals))?; + if objects.len() != ordinals.len() || objects.iter().any(Option::is_none) { + return Err(Error::Request( + "Visibility base has missing catalog objects", + )); + } + Ok(objects + .into_iter() + .flatten() + .map(|oid| oid.to_string()) + .collect()) + } + None => Err(Error::Request("Visibility base has no captured proof")), + } + } + fn contains_ref(&self, name: &str) -> bool { match &self.proof { Some(VisibilityProof::Catalog(proof)) => proof.contains_ref(name), @@ -1173,6 +1214,24 @@ where || cancel.is_cancelled(), )?; let evidence = match proof { + RefVisibility::Additive { added, .. } + if update.old.is_none() + || source + .prior + .as_ref() + .is_some_and(|(name, _)| name != &update.name) => + { + // A different ref can disappear before canonical replay. + // Copy its pinned closure into replacement evidence so an + // acknowledged ref never borrows that mutable lifetime. + let mut objects = source.prior_objects(options.graph.max_graph_steps)?; + objects.extend(added.into_iter().map(|oid| oid.to_string())); + GitVisibilityEdit::from_replacement_objects( + update.old.map(|oid| oid.to_string()), + new.to_string(), + objects, + ) + } RefVisibility::Additive { base, added } => { GitVisibilityEdit::from_delta_objects( Some(base.to_string()), diff --git a/crates/crab-remote/tests/capsule_preparation.rs b/crates/crab-remote/tests/capsule_preparation.rs index 52d9db6fe..76539482e 100644 --- a/crates/crab-remote/tests/capsule_preparation.rs +++ b/crates/crab-remote/tests/capsule_preparation.rs @@ -195,12 +195,34 @@ async fn prepare( crab_read::capsule_protocol::CapsuleRepositoryView, Arc, crab_remote::publication::PublicationLeases, +) { + let oid = gix_hash::ObjectId::from_hex( + String::from_utf8(git(directory, &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .as_bytes(), + ) + .unwrap(); + prepare_update(layout, directory, content, "refs/heads/main", Some(oid)).await +} + +async fn prepare_update( + layout: &crab_storage::StoreLayout, + directory: &std::path::Path, + content: Option, + ref_name: &str, + new: Option, +) -> ( + crab_remote::prepare::Prepared, + crab_read::capsule_protocol::CapsuleRepositoryView, + Arc, + crab_remote::publication::PublicationLeases, ) { let cancel = CancellationToken::new(); let leases = crab_remote::publication::acquire_leases( layout.store(), layout, - ["refs/heads/main".to_owned()], + [ref_name.to_owned()], crab_remote::publication::LeaseOptions::renewing(std::time::Duration::from_secs(60)), &cancel, ) @@ -231,24 +253,24 @@ async fn prepare( ) .await .unwrap(); - let oid = gix_hash::ObjectId::from_hex( - String::from_utf8(git(directory, &["rev-parse", "HEAD"])) - .unwrap() - .trim() - .as_bytes(), - ) - .unwrap(); - let mut revisions = String::from("HEAD\n"); - if let Some(old) = view.refs().get("refs/heads/main") { - revisions.push_str(&format!("^{old}\n")); - } - let pack = git_input( - directory, - &["pack-objects", "--stdout", "--revs"], - revisions.as_bytes(), - ); - let incoming = directory.join("incoming.pack"); - std::fs::write(&incoming, pack).unwrap(); + let incoming = if let Some(oid) = new { + let mut revisions = format!("{oid}\n"); + for old in view.refs().values() { + revisions.push_str(&format!("^{old}\n")); + } + let pack = git_input( + directory, + &["pack-objects", "--stdout", "--revs"], + revisions.as_bytes(), + ); + let incoming = directory.join("incoming.pack"); + std::fs::write(&incoming, pack).unwrap(); + Some(std::io::BufReader::new( + std::fs::File::open(incoming).unwrap(), + )) + } else { + None + }; let limits = crab_git::incoming_pack::ReceiveLimits { max_pack_bytes: LIMIT, max_objects: 1024, @@ -260,16 +282,14 @@ async fn prepare( repository, view.git_visibility_index().await.unwrap(), crab_git::incoming_pack::ScratchSpace::new(directory, limits), - Some(std::io::BufReader::new( - std::fs::File::open(incoming).unwrap(), - )), + incoming, vec![crab_git::receive_plan::RefUpdate { - name: "refs/heads/main".to_owned(), + name: ref_name.to_owned(), old: view .refs() - .get("refs/heads/main") + .get(ref_name) .map(|oid| gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap()), - new: Some(oid), + new, }], std::collections::BTreeMap::new(), &cancel, @@ -619,3 +639,6 @@ async fn historical_pointer_lookup_preserves_declared_size_checks() { ); } } + +#[path = "capsule_preparation/ref_visibility.rs"] +mod ref_visibility; diff --git a/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs b/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs new file mode 100644 index 000000000..5a27243b2 --- /dev/null +++ b/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs @@ -0,0 +1,207 @@ +use super::*; + +#[tokio::test(flavor = "multi_thread")] +async fn acknowledged_branch_survives_source_deletion() { + branch_survives_source_deletion(false).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn prepared_branch_survives_source_deletion_before_commit() { + branch_survives_source_deletion(true).await; +} + +async fn branch_survives_source_deletion(delete_before_commit: bool) { + use crab_metadata::capsule_protocol::{CapsuleRefEdit, CapsuleTransaction}; + use crab_remote::prepare::CapsuleCommitOutcome; + let cancel = CancellationToken::new(); + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &cancel) + .await + .unwrap(); + let committed = artifacts + .commit(None, std::time::Duration::from_secs(60), &cancel) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!(matches!( + committed.unwrap(), + CapsuleCommitOutcome::Committed { .. } + )); + let tip = gix_hash::ObjectId::from_hex( + String::from_utf8(git(directory.path(), &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .as_bytes(), + ) + .unwrap(); + let mut expected_objects = String::from_utf8(git( + directory.path(), + &["rev-list", "--objects", "--no-object-names", "HEAD"], + )) + .unwrap() + .lines() + .map(str::to_owned) + .collect::>(); + expected_objects.sort_unstable(); + let borrowed = "refs/heads/borrowed"; + let (prepared, view, runtime, leases) = + prepare_update(&layout, directory.path(), None, borrowed, Some(tip)).await; + let edit = CapsuleRefEdit::new( + borrowed, + None, + Some(tip.to_string()), + prepared + .plan() + .peeled() + .get(borrowed) + .map(ToString::to_string), + ); + let (plan, expected_borrow) = (0..256) + .find_map(|i| { + let plan = blake3::hash(format!("borrow-lifetime:{i}").as_bytes()) + .to_hex() + .to_string(); + let transaction = CapsuleTransaction::for_plan( + view.root_snapshot().record().digest(), + &plan, + vec![edit.clone()], + ) + .unwrap(); + let id = transaction.id().unwrap(); + (id.as_bytes()[0] >= b'8').then_some((plan, id)) + }) + .unwrap(); + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &cancel) + .await + .unwrap(); + if delete_before_commit { + delete_source(&layout, directory.path(), tip, &expected_borrow).await; + } + let committed = artifacts + .commit(Some(&plan), std::time::Duration::from_secs(60), &cancel) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + let CapsuleCommitOutcome::Committed { transaction_id } = committed.unwrap() else { + panic!("borrowed ref publication must have a proven terminal outcome"); + }; + assert_eq!(transaction_id, expected_borrow); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let (heads, active) = + crab_metadata::capsule_protocol::capture_ref_heads(&layout, root.record().root()) + .await + .unwrap(); + let head = heads + .iter() + .find(|head| head.ref_name() == borrowed) + .unwrap(); + let pointer = &head.visible(&active).frontier()[0]; + let (_, controls) = + crab_metadata::capsule_protocol::load_capsule_run_git_control(&layout, pointer) + .await + .unwrap(); + let control = controls[0] + .materialize_visibility(layout.store()) + .await + .unwrap(); + let evidence = &control.visibility_delta().unwrap().unwrap().edits()[borrowed]; + assert_eq!( + (evidence.replaces, &evidence.old_oid, &evidence.added), + (true, &None, &expected_objects), + ); + if !delete_before_commit { + delete_source(&layout, directory.path(), tip, &expected_borrow).await; + } + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let limits = crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }; + for mode in ["full", "control", "paged"] { + let view = match mode { + "full" => { + crab_read::capsule_protocol::open_view_from_root(&layout, root.clone(), limits) + .await + } + "control" => { + crab_read::capsule_protocol::open_view_from_root_with_metadata( + &layout, + root.clone(), + limits, + ) + .await + } + _ => { + crab_read::capsule_protocol::open_git_view_from_root(&layout, root.clone(), limits) + .await + } + } + .unwrap_or_else(|error| panic!("acknowledged state must reopen through {mode}: {error}")); + let index = view.git_visibility_index().await.unwrap(); + assert_eq!( + ( + view.refs().get(borrowed).map(String::as_str), + view.refs().contains_key("refs/heads/main"), + index.objects_for_ref(borrowed) + ), + ( + Some(tip.to_string().as_str()), + false, + Some(expected_objects.clone()) + ), + ); + } +} + +async fn delete_source( + layout: &crab_storage::StoreLayout, + directory: &std::path::Path, + tip: gix_hash::ObjectId, + expected_borrow: &str, +) { + use crab_metadata::capsule_protocol::{CapsuleRefEdit, CapsuleTransaction}; + use crab_remote::prepare::CapsuleCommitOutcome; + let cancel = CancellationToken::new(); + let (prepared, view, runtime, leases) = + prepare_update(layout, directory, None, "refs/heads/main", None).await; + let edit = CapsuleRefEdit::new("refs/heads/main", Some(tip.to_string()), None, None); + // Reviewed plans provide stable transaction identities without a runtime + // nonce hook. Force deletion ahead of borrowing in canonical ID order. + let (plan, expected_delete) = (0..256) + .find_map(|i| { + let plan = blake3::hash(format!("delete-lifetime:{i}").as_bytes()) + .to_hex() + .to_string(); + let transaction = CapsuleTransaction::for_plan( + view.root_snapshot().record().digest(), + &plan, + vec![edit.clone()], + ) + .unwrap(); + let id = transaction.id().unwrap(); + (id.as_str() < expected_borrow).then_some((plan, id)) + }) + .unwrap(); + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &cancel) + .await + .unwrap(); + let committed = artifacts + .commit(Some(&plan), std::time::Duration::from_secs(60), &cancel) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + let CapsuleCommitOutcome::Committed { transaction_id } = committed.unwrap() else { + panic!("source deletion must have a proven terminal outcome"); + }; + assert_eq!(transaction_id, expected_delete); +} From d9759a21cfdeac98c1d4570e1bd1a01a8a273f75 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:13:04 -0700 Subject: [PATCH 098/127] test(remote): qualify deleted ref source lifetime Remove the source checkout before comparing all cold Git object bodies with native Git and verifying full native payload hashes at origin. Run both publication orderings through full, control and paged readers against RustFS. --- .../capsule_preparation/ref_visibility.rs | 99 +++++++++++++++++-- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs b/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs index 5a27243b2..841fcbe63 100644 --- a/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs +++ b/crates/crab-remote/tests/capsule_preparation/ref_visibility.rs @@ -2,21 +2,48 @@ use super::*; #[tokio::test(flavor = "multi_thread")] async fn acknowledged_branch_survives_source_deletion() { - branch_survives_source_deletion(false).await; + branch_survives_source_deletion(layout().await.0, false).await; } #[tokio::test(flavor = "multi_thread")] async fn prepared_branch_survives_source_deletion_before_commit() { - branch_survives_source_deletion(true).await; + branch_survives_source_deletion(layout().await.0, true).await; } -async fn branch_survives_source_deletion(delete_before_commit: bool) { +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] +async fn live_branch_survives_source_deletion_and_source_removal() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").expect("explicit scratch bucket required"); + let store = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + for before_commit in [false, true] { + let prefix = format!("ref-lifetime/{nonce}/{before_commit}"); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.clone(), + format!("{prefix}/repo"), + format!("{prefix}/global"), + ); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + branch_survives_source_deletion(layout, before_commit).await; + } +} + +async fn branch_survives_source_deletion( + layout: crab_storage::StoreLayout, + delete_before_commit: bool, +) { use crab_metadata::capsule_protocol::{CapsuleRefEdit, CapsuleTransaction}; use crab_remote::prepare::CapsuleCommitOutcome; let cancel = CancellationToken::new(); let (directory, content) = content_fixture().await; source_commit(directory.path(), &content); - let (layout, _) = layout().await; let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; let artifacts = prepared .upload_capsule(&view, proof_limits(), &cancel) @@ -47,6 +74,21 @@ async fn branch_survives_source_deletion(delete_before_commit: bool) { .map(str::to_owned) .collect::>(); expected_objects.sort_unstable(); + let expected_bodies = expected_objects + .iter() + .map(|oid| { + let kind = String::from_utf8(git(directory.path(), &["cat-file", "-t", oid])).unwrap(); + ( + gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap(), + git(directory.path(), &["cat-file", kind.trim(), oid]), + ) + }) + .collect::>(); + let crab_git::pointer_detect::PointerKind::Crab(pointer) = crab_git::pointer_detect::classify( + &std::fs::read(directory.path().join("model.bin")).unwrap(), + ) else { + panic!("fixture pointer"); + }; let borrowed = "refs/heads/borrowed"; let (prepared, view, runtime, leases) = prepare_update(&layout, directory.path(), None, borrowed, Some(tip)).await; @@ -102,9 +144,9 @@ async fn branch_survives_source_deletion(delete_before_commit: bool) { .iter() .find(|head| head.ref_name() == borrowed) .unwrap(); - let pointer = &head.visible(&active).frontier()[0]; + let capsule_pointer = &head.visible(&active).frontier()[0]; let (_, controls) = - crab_metadata::capsule_protocol::load_capsule_run_git_control(&layout, pointer) + crab_metadata::capsule_protocol::load_capsule_run_git_control(&layout, capsule_pointer) .await .unwrap(); let control = controls[0] @@ -122,6 +164,7 @@ async fn branch_survives_source_deletion(delete_before_commit: bool) { let root = crab_write::capsule_protocol::open_root(&layout) .await .unwrap(); + std::fs::remove_dir_all(directory.path()).unwrap(); let limits = crab_read::capsule_protocol::CapsuleReadLimits { max_capsule_bytes: LIMIT, max_frontier_bytes: LIMIT, @@ -159,6 +202,50 @@ async fn branch_survives_source_deletion(delete_before_commit: bool) { Some(expected_objects.clone()) ), ); + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let repository = view + .git_repository_from_store( + layout.clone(), + crab_remote_git::RepositoryIdentity::new("qualification", "branch", 1).unwrap(), + runtime.clone(), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap(); + let operation = repository + .operation(crab_remote_git::OperationKind::Repository, &cancel) + .await + .unwrap(); + let oids = expected_bodies + .iter() + .map(|(oid, _)| *oid) + .collect::>(); + let result = operation.read_objects(&oids).await; + let result = operation.finish(result).await; + runtime.shutdown().await; + assert_eq!( + result + .unwrap() + .into_iter() + .map(|object| (object.oid, object.data.to_vec())) + .collect::>(), + expected_bodies, + "cold {mode} object bytes after source removal", + ); + let catalog = view.pointer_catalog().await.unwrap(); + let file_hash = crab_xet::hash::merkle_hex_from_bytes(&pointer.file_hash); + let shard = MerkleHash::from_hex(catalog.files()[&file_hash].shard_hash()).unwrap(); + crab_read::pointer_proof::verify_crab_pointer( + &layout, + &pointer, + shard, + proof_limits().content, + &cancel, + ) + .await + .unwrap(); } } From 29e68486d914659510df0590fedcc4903a2bc2ad Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:13:55 -0700 Subject: [PATCH 099/127] fix(metadata): keep storage within feature budget Use the existing futures lock for read-ahead lanes and remove the test diagnostic rejected by the metadata scope gate. --- crates/crab-metadata/Cargo.toml | 2 +- .../src/capsule_protocol/catalog_capture.rs | 1 - crates/crab-metadata/src/pages/cache.rs | 29 ++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/crab-metadata/Cargo.toml b/crates/crab-metadata/Cargo.toml index 6b768e047..02bd000ab 100644 --- a/crates/crab-metadata/Cargo.toml +++ b/crates/crab-metadata/Cargo.toml @@ -17,7 +17,7 @@ default = [] file-index-reader = ["storage", "dep:futures-util", "dep:object_store", "dep:slatedb", "dep:tokio", "dep:tokio-util"] local-index = ["dep:rusqlite"] remote-index = ["storage", "dep:futures-util", "dep:object_store", "dep:slatedb"] -storage = ["dep:base64", "dep:crab-storage", "dep:futures-util", "dep:object_store", "dep:tokio"] +storage = ["dep:base64", "dep:crab-storage", "dep:futures-util", "dep:object_store"] [dependencies] base64 = { version = "0.22", optional = true } diff --git a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs index e14f7d6b1..6968b4c8c 100644 --- a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs +++ b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs @@ -298,7 +298,6 @@ mod tests { observations.push((count, reads.load(Ordering::SeqCst))); session.close().await.unwrap(); } - eprintln!("compacted catalog opening read attempts: {observations:?}"); assert!( observations.windows(2).all(|pair| pair[0].1 == pair[1].1), "provider reads should follow compacted proof blocks rather than logical edits: {observations:?}" diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs index f5be7b4d0..c26a98df1 100644 --- a/crates/crab-metadata/src/pages/cache.rs +++ b/crates/crab-metadata/src/pages/cache.rs @@ -13,7 +13,7 @@ const READ_AHEAD_BYTES: u64 = 64 * 1024; pub(crate) struct PageCache { state: Mutex, read_ahead: Option>, - block_reads: [tokio::sync::Mutex<()>; 4], + block_reads: [futures_util::lock::Mutex<()>; 4], } #[derive(Debug, Default)] @@ -435,6 +435,33 @@ mod tests { ); } + #[tokio::test] + async fn cancelled_queued_read_does_not_keep_the_cache_lane_locked() { + use futures_util::FutureExt; + let store = Store::new(Arc::new(object_store::memory::InMemory::new())); + let path = object_store::path::Path::from("proof"); + let expected = bytes::Bytes::from_static(b"block"); + store.create_strict(&path, expected.clone()).await.unwrap(); + let range = 0..expected.len() as u64; + let cache = PageCache::for_region(range.clone()); + let held = cache.block_reads[0].lock().await; + // Dropping a queued read must unregister its waiter; otherwise a + // cancelled query can strand later cold reads sharing this lane. + let queued = cache + .read_block(&store, &path, range.clone()) + .now_or_never() + .is_none(); + drop(held); + let actual = tokio::time::timeout( + std::time::Duration::from_secs(1), + cache.read_block(&store, &path, range), + ) + .await + .unwrap() + .unwrap(); + assert_eq!((queued, actual), (true, expected)); + } + #[tokio::test] async fn prefetched_pages_still_require_their_own_authentication() { let mut writer = Writer::default(); From 40d926902f2671d3ee877960c6fc7883982d1d1f Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:23:07 -0700 Subject: [PATCH 100/127] fix(metadata): bound catalog enum layouts --- crates/crab-metadata/src/capsule_protocol/layered.rs | 2 +- .../src/capsule_protocol/layered/catalog.rs | 4 ++-- crates/crab-remote-git/src/commit_graph.rs | 4 ++-- crates/crab-remote/src/checkpoint.rs | 9 +++++---- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index bdd82644e..8c2779f8b 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -1012,7 +1012,7 @@ impl LayeredCheckpoint { covered_generation, covered_root_digest, sources, - CatalogInput::Retained(catalog), + CatalogInput::Retained(catalog.map(Box::new)), visibility, ) } diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs index 0bcc63bc4..157a2edfc 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs @@ -79,7 +79,7 @@ impl CheckpointCatalog { pub(super) enum CatalogInput { New(PointerCatalog), - Retained(Option), + Retained(Option>), } impl CatalogInput { @@ -90,7 +90,7 @@ impl CatalogInput { if let Some(source) = &source { source.validate()?; } - return Ok((source, None)); + return Ok((source.map(|source| *source), None)); } }; if catalog.is_empty() { diff --git a/crates/crab-remote-git/src/commit_graph.rs b/crates/crab-remote-git/src/commit_graph.rs index 34d18d7cf..6180a3ad5 100644 --- a/crates/crab-remote-git/src/commit_graph.rs +++ b/crates/crab-remote-git/src/commit_graph.rs @@ -13,7 +13,7 @@ use crate::{BudgetDimension, CorruptionStage, Error, OperationContext, RemoteGit #[derive(Debug)] enum Graph { Manifest(SplitCommitGraph), - Capsule(CommitGraphReader), + Capsule(Box), } /// Validated positional graph bound to captured ref roots, never physical retention. @@ -60,7 +60,7 @@ impl CommitGraphIndex { if descriptor.version == 1 { load_split_commit_graph(store, layout, content_hash, max_bytes).await.map(Graph::Manifest) } else { - CommitGraphReader::load(store, layout, content_hash, max_bytes).await.map(Graph::Capsule) + CommitGraphReader::load(store, layout, content_hash, max_bytes).await.map(Box::new).map(Graph::Capsule) } } => result?, }; diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index bfe0af437..be1743a54 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -394,7 +394,7 @@ pub async fn publish_capsule_checkpoint_with_catalog_from_view( enum CatalogWrite { New(PointerCatalog), - Retained(Option), + Retained(Option>), } async fn catalog_for_checkpoint( @@ -408,7 +408,8 @@ async fn catalog_for_checkpoint( Ok(CatalogWrite::Retained( view.layered_checkpoint() .and_then(LayeredCheckpoint::catalog) - .cloned(), + .cloned() + .map(Box::new), )) } @@ -521,7 +522,7 @@ async fn repack_checkpoint_from_view( let Some(selected_start) = layered_suffix_start(checkpoint.sources())? else { return Ok(CheckpointPublication::default()); }; - let catalog = CatalogWrite::Retained(checkpoint.catalog().cloned()); + let catalog = CatalogWrite::Retained(checkpoint.catalog().cloned().map(Box::new)); let consolidation = consolidate_layered_suffix( layout, view, @@ -588,7 +589,7 @@ async fn publish_layered_sources( generation, root_digest, sources, - catalog, + catalog.map(|catalog| *catalog), Some(visibility), )?, }; From dbd97fc108c6fb35e3970b9a292c2a58660c9804 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:25:58 -0700 Subject: [PATCH 101/127] docs: consolidate indexed staging delivery --- crab/docs/architecture/indexed-remote-reads.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 16a7d4502..6075c2330 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -35,10 +35,13 @@ Xet files and unified local staging are subsequent workstreams. | 7 | Lifecycle and migration | Interrupted publication/builds, concurrent owners, GC retained roots/readers, stale caches and released-format migration qualification | Pending; also required in every earlier implementation | | 8 | Fuller Git blame semantics | Line-porcelain parity across merges/renames/repeated lines/recreation and each supported whitespace/ignored-revision option | Pending | -Each implementation PR owns one delivery gate or a complete vertical slice within -it. Tests alone are insufficient: user operations against a real S3-protocol store -must produce independently verified results. Representative cloud latency remains -a separate cell from local RustFS correctness. +Development slices are consolidated into one integration PR targeting `crab-v2`. +Earlier component PRs and revision-specific results below remain historical +evidence; they are not separate merge prerequisites or proof for the final tree. +The combined source must satisfy the outstanding gates before landing. Tests +alone are insufficient: user operations against a real S3-protocol store must +produce independently verified results. Representative cloud latency remains a +separate cell from local RustFS correctness. ## Measurement contract From 2f9e1972620a7f30a4e16dabe9965a084e41c07c Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:32:43 -0700 Subject: [PATCH 102/127] test(write): remove needless fence borrow --- crates/crab-write/src/capsule_protocol.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index 9dc4a619b..f1fb7ac9c 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -3639,13 +3639,9 @@ mod tests { .await .unwrap(); if restore { - base = begin_gc( - &seed, - base, - GcFence::new(&"e".repeat(64), u64::MAX).unwrap(), - ) - .await - .unwrap(); + base = begin_gc(&seed, base, GcFence::new("e".repeat(64), u64::MAX).unwrap()) + .await + .unwrap(); } let visibility = crab_metadata::git_visibility::GitVisibilityIndex::new( 0, From 937c6185aa96f1c5c1234fa11e45ba34d79c0737 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 15:48:49 -0700 Subject: [PATCH 103/127] test(gc): supply complete maintenance admission Keep Git query captures lazy. Build the GC fixture through the maintenance entry point and preserve query-view rejection, catalog retention and grace-period assertions. --- crab/src/cmd/gc/captured_index_tests.rs | 15 ++++++++++++++- crates/crab-remote/src/checkpoint.rs | 5 +++-- crates/crab-remote/tests/checkpoint.rs | 25 +++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index 0d7d5baa1..3485dcf1b 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -577,7 +577,20 @@ async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { catalog .insert_file("b".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) .unwrap(); - let before = view(&layout).await; + // Checkpoint publication needs exact member admission; query views leave it cold. + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let before = crab_read::capsule_protocol::open_view_from_root_with_metadata( + &layout, + root, + CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); crab_remote::checkpoint::publish_capsule_checkpoint_with_catalog_from_view( &layout, &before, diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index be1743a54..a855fb0ea 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -372,8 +372,9 @@ pub async fn publish_capsule_checkpoint_from_view( /// Publish one checkpoint from a pinned view with a replacement pointer catalog. /// /// The caller must make every external object named by `catalog` durable and -/// verify its complete dependency closure before calling. Publication retains -/// the captured ref frontier and succeeds only against the view's exact root. +/// verify its complete dependency closure before calling. The view must retain +/// complete pack-member admission from the metadata entry point. Publication +/// retains the captured ref frontier and succeeds only against its exact root. pub async fn publish_capsule_checkpoint_with_catalog_from_view( layout: &StoreLayout, view: &crab_read::capsule_protocol::CapsuleRepositoryView, diff --git a/crates/crab-remote/tests/checkpoint.rs b/crates/crab-remote/tests/checkpoint.rs index 93d8cba40..9a478c5e5 100644 --- a/crates/crab-remote/tests/checkpoint.rs +++ b/crates/crab-remote/tests/checkpoint.rs @@ -93,6 +93,31 @@ async fn maintenance_reuses_committed_checkpoint_without_reloading_authority() { ); } +#[tokio::test] +async fn query_view_cannot_publish_checkpoint_without_member_admission() { + let (layout, _) = fixture().await; + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let query = crab_read::capsule_protocol::open_git_view_from_root(&layout, root, LIMITS) + .await + .unwrap(); + let result = crab_remote::checkpoint::publish_capsule_checkpoint_with_catalog_from_view( + &layout, + &query, + crab_metadata::capsule_protocol::PointerCatalog::new(), + LIMIT, + &CancellationToken::new(), + ) + .await; + assert!(matches!( + result, + Err(crab_remote::checkpoint::CheckpointError::Repack( + crab_git::repack::RepackError::SourceIntegrity { .. } + )) + )); +} + #[tokio::test] async fn retained_checkpoint_view_requires_complete_metadata_and_exact_root_binding() { let (layout, _) = fixture().await; From 8442703a97d9807ffcbc723e0adebd539df6c38e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 16:01:36 -0700 Subject: [PATCH 104/127] test(cache): exercise push-time advisory lookup Stage the duplicate through Git-clean so push still needs advisory candidates. Keep every origin-proof and no-upload assertion, and cold-hydrate the newly published duplicate repository. --- .../e2e/run_cache_service_rustfs_smoke.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crab/scripts/e2e/run_cache_service_rustfs_smoke.py b/crab/scripts/e2e/run_cache_service_rustfs_smoke.py index e5a06b82e..aa3542dd5 100755 --- a/crab/scripts/e2e/run_cache_service_rustfs_smoke.py +++ b/crab/scripts/e2e/run_cache_service_rustfs_smoke.py @@ -4477,17 +4477,17 @@ def push_duplicate_repo_uses_cache_service_dedup(self, data: bytes) -> CliPushDe ) (repo / "model.bin").write_bytes(data) - # Deduplication now happens while `crab add` prepares the staged xorb. - # Measure the complete add-to-push pipeline so this qualification proves - # both the cache-service classification and push-time proof reuse. + # Native add can prove origin placements before push and skip advisory + # queries. Git-clean retains local bytes, so this case reaches push-time + # cache candidates without weakening their canonical-origin proof. before_stats = self.cache_admin_stats() before_gets = state.total_get_count() before_get_counts = state.counts_snapshot() before_puts = state.total_put_count() before_put_counts = state.put_counts_snapshot() self.run_cmd( - "cli dedup crab add model", - [self.crab_bin, "add", "--jobs", "0", "model.bin"], + "cli dedup git add model", + ["git", "add", "model.bin"], repo, env=env, timeout=self.args.push_timeout, @@ -4649,6 +4649,14 @@ def push_duplicate_repo_uses_cache_service_dedup(self, data: bytes) -> CliPushDe "mutable_actual": record.mutable_origin_get_key_delta, }, ) + expected_sha = hashlib.sha256(data).hexdigest() + self.clone_configure_and_hydrate( + "cli-dedup-cold-hydrate", + remote_url, + expected_sha, + cache_name="cli-dedup-cold-cache", + expect_origin_gets=False, + ) return record def clone_configure_and_hydrate( From e7e6cc3eb00724fa43a743791e456d629e6565ba Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 16:34:44 -0700 Subject: [PATCH 105/127] test(gc): admit members for catalog replacement --- crab/src/cmd/gc/captured_index_tests.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index 3485dcf1b..dd56d0c60 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -600,7 +600,19 @@ async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { ) .await .unwrap(); - let first = view(&layout).await; + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let first = crab_read::capsule_protocol::open_view_from_root_with_metadata( + &layout, + root, + CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); let source = first.layered_checkpoint().unwrap().catalog().unwrap(); let retained = layout.capsule_catalog_path(source.hash()); backend.paths.lock().unwrap().insert(retained.clone()); From d8b8bdafcf7f08b5b7274d57e353d1efd2d00fa7 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 16:36:40 -0700 Subject: [PATCH 106/127] test(cache): verify duplicate repo hydration traffic --- crab/scripts/verify-cache-service-smoke-report.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crab/scripts/verify-cache-service-smoke-report.py b/crab/scripts/verify-cache-service-smoke-report.py index 57a618a2c..5f723187a 100755 --- a/crab/scripts/verify-cache-service-smoke-report.py +++ b/crab/scripts/verify-cache-service-smoke-report.py @@ -1353,7 +1353,12 @@ def verify_direct_read_traffic(self) -> None: ) def verify_cli_hydrate_traffic(self) -> None: - for name in ("cli-cold-hydrate", "cli-warm-hydrate", "restart-cli-hydrate"): + for name in ( + "cli-dedup-cold-hydrate", + "cli-cold-hydrate", + "cli-warm-hydrate", + "restart-cli-hydrate", + ): record = self.record("cli_hydrates", name) origin_delta = self.int_value(record, "origin_gets_after") - self.int_value( record, "origin_gets_before" From 3ba4c51e1d1338dae25ba79c94d18834f5c2525a Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 16:45:49 -0700 Subject: [PATCH 107/127] test(cache): separate quiescent and retained WAL replacement --- crates/crab-cache-store/README.md | 6 ++++ crates/crab-cache-store/src/lib.rs | 58 +++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/crab-cache-store/README.md b/crates/crab-cache-store/README.md index 6fa3ae9de..ba8e5fe8e 100644 --- a/crates/crab-cache-store/README.md +++ b/crates/crab-cache-store/README.md @@ -42,6 +42,12 @@ entries are bypassed even when eviction fails; corrupt origin bytes return `CacheStoreError::OriginIntegrity` with their verification error retained. Transport retry policy remains owned by `crab-storage`. +Catalog replacement also fails closed: a live generation lease prevents +rebinding, and retained recovery files still prevent rebinding after that lease +closes. Verified origin reads remain available without changing the replacement +or replaying an unrelated WAL. A quiescent empty replacement can initialize +after the old lease closes; adapter reads do not repair ambiguous databases. + `git_pack::read_pack_ranges` supplies the file-backed native-pack path. It uses the caller's selected origin, even when the cache was constructed around another store. A verified local hit copies the immutable pack body and reads diff --git a/crates/crab-cache-store/src/lib.rs b/crates/crab-cache-store/src/lib.rs index a19de3ce3..2d11a9ed4 100644 --- a/crates/crab-cache-store/src/lib.rs +++ b/crates/crab-cache-store/src/lib.rs @@ -2073,8 +2073,8 @@ mod tests { } #[cfg(unix)] - #[tokio::test] - async fn replaced_catalog_bypasses_cache_until_old_generation_is_released() { + async fn replaced_catalog_readthrough(retain_recovery: bool) { + use std::os::unix::fs::OpenOptionsExt as _; let good_body = Bytes::from_static(b"origin survives replaced catalog"); let hash = crab_xet::hash::compute_data_hash(&good_body); let path = content_path("shards", &hash.hex()); @@ -2097,7 +2097,32 @@ mod tests { let main = root.join(".catalog.sqlite"); let retired = tmp.path().join("retired.sqlite"); std::fs::rename(&main, &retired).unwrap(); - std::fs::copy(retired, &main).unwrap(); + let mut recovery = Vec::new(); + for suffix in ["-journal", "-wal", "-shm"] { + let path = root.join(format!(".catalog.sqlite{suffix}")); + if path.exists() { + if retain_recovery { + recovery.push((path.clone(), std::fs::read(&path).unwrap())); + } else { + std::fs::rename(&path, tmp.path().join(path.file_name().unwrap())).unwrap(); + } + } + } + if retain_recovery { + assert!(recovery.iter().any(|(path, bytes)| { + path.file_name().unwrap() == ".catalog.sqlite-wal" && bytes.len() > 32 + })); + } + // An empty replacement is coherent. Copying only the live main would + // omit committed WAL state and cannot establish recovery ownership. + drop( + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&main) + .unwrap(), + ); let before = std::fs::read(&main).unwrap(); let key = CacheKey::Shard(hash); let store = @@ -2122,7 +2147,32 @@ mod tests { .await .unwrap(); assert_eq!(got, good_body); - assert!(cache.contains_verified(&key).await); + assert_eq!(counting_origin.counts().body_requests(), 2); + if retain_recovery { + assert!(!cache.contains_verified(&key).await); + assert!(matches!( + cache.put(&key, &good_body).await, + Err(CacheError::UnsafeRoot { .. }) + )); + assert_eq!(std::fs::read(&main).unwrap(), before); + for (path, bytes) in recovery { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + } else { + assert!(cache.contains_verified(&key).await); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn replaced_catalog_bypasses_cache_until_old_generation_is_released() { + replaced_catalog_readthrough(false).await; + } + + #[cfg(unix)] + #[tokio::test] + async fn replaced_catalog_with_recovery_files_keeps_bypassing_after_owner_release() { + replaced_catalog_readthrough(true).await; } #[cfg(unix)] From 73445c8c75ca17d2cdc3ff957daa39e89bb2d46c Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 18:58:19 -0700 Subject: [PATCH 108/127] fix(ci): scope async-trait lint allowances Newer Clippy flags must_use emitted by async-trait 0.1.89. Scope allowances to eight trait expansions so MSRV linting still works. Runtime code and dependency versions remain unchanged. --- crates/crab-coordination/src/cosmosdb_coordinator.rs | 4 ++++ crates/crab-coordination/src/dynamodb_coordinator.rs | 4 ++++ crates/crab-coordination/src/spanner_coordinator.rs | 4 ++++ crates/crab-coordination/src/write_coordinator.rs | 12 ++++++++++++ crates/crab-staging/src/add_push_plan.rs | 8 ++++++++ 5 files changed, 32 insertions(+) diff --git a/crates/crab-coordination/src/cosmosdb_coordinator.rs b/crates/crab-coordination/src/cosmosdb_coordinator.rs index d044c0005..163d86eba 100644 --- a/crates/crab-coordination/src/cosmosdb_coordinator.rs +++ b/crates/crab-coordination/src/cosmosdb_coordinator.rs @@ -69,6 +69,10 @@ pub struct CosmosDbCreateCoordinatorAccount { } /// Minimal Cosmos DB control-plane client needed by Crab-owned coordinator setup. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait CosmosDbCoordinatorControlPlaneClient { async fn describe_account( diff --git a/crates/crab-coordination/src/dynamodb_coordinator.rs b/crates/crab-coordination/src/dynamodb_coordinator.rs index d67564498..e1f5cc131 100644 --- a/crates/crab-coordination/src/dynamodb_coordinator.rs +++ b/crates/crab-coordination/src/dynamodb_coordinator.rs @@ -85,6 +85,10 @@ pub type DynamoDbRepoState = CoordinatorRepoState; pub type DynamoDbTransactionRecord = CoordinatorTransactionRecord; /// DynamoDB data-plane client for one serialized repo authority item. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait DynamoDbWriteCoordinatorClient { async fn read_repo_state( diff --git a/crates/crab-coordination/src/spanner_coordinator.rs b/crates/crab-coordination/src/spanner_coordinator.rs index cb2e7173a..aea5be24a 100644 --- a/crates/crab-coordination/src/spanner_coordinator.rs +++ b/crates/crab-coordination/src/spanner_coordinator.rs @@ -64,6 +64,10 @@ pub struct SpannerCreateCoordinator { } /// Minimal Spanner control-plane client needed by Crab-owned coordinator setup. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait SpannerCoordinatorControlPlaneClient { async fn describe_instance( diff --git a/crates/crab-coordination/src/write_coordinator.rs b/crates/crab-coordination/src/write_coordinator.rs index 12c9bcb11..bea9b35d1 100644 --- a/crates/crab-coordination/src/write_coordinator.rs +++ b/crates/crab-coordination/src/write_coordinator.rs @@ -373,6 +373,10 @@ pub struct CoordinatorControlPlaneStatus { } /// Management backend for a linearizable active-active coordinator. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait CoordinatorControlPlaneBackend: Send + Sync { fn provider(&self) -> ManagedCoordinatorProvider; @@ -601,6 +605,10 @@ fn coordinator_base_action(action: &str) -> &str { } /// Versioned CAS storage contract for managed coordinator data planes. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait VersionedCoordinatorStateStore: Send + Sync { async fn read_repo_state( @@ -645,6 +653,10 @@ where } /// Linearizable authority for active-active repository writes. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait WriteCoordinator: Send + Sync { async fn health(&self) -> Result; diff --git a/crates/crab-staging/src/add_push_plan.rs b/crates/crab-staging/src/add_push_plan.rs index d6e22ff5a..f29c78aa7 100644 --- a/crates/crab-staging/src/add_push_plan.rs +++ b/crates/crab-staging/src/add_push_plan.rs @@ -42,6 +42,10 @@ pub struct AddPushPlanSummary { } /// Looks up already-uploaded chunk placements for staged chunks. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait ExistingChunkLookup: Send + Sync { async fn lookup_existing_candidates( @@ -51,6 +55,10 @@ pub trait ExistingChunkLookup: Send + Sync { } /// Adds local prepared-xorb candidates to the staging cache. +#[allow( + clippy::double_must_use, + reason = "async-trait 0.1.89 injects must_use on boxed futures; MSRV Clippy does not report it" +)] #[async_trait] pub trait LocalXorbCandidateLookup: Send + Sync { async fn load_candidates( From f8d74b856b06bc10db147121dfef8f302bc95644 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 20:12:35 -0700 Subject: [PATCH 109/127] feat(metadata): persist changed catalog pages Reuse unchanged catalog subtrees so fixed edits avoid full history assembly and encoding. Verify new pages before checkpoint visibility; retain complete query closure, physical GC roots and accounting. Checkpoint format 9 rejects unshipped predecessors through 8. Audited published tags contain no capsule codec; no compatibility fallback. Proof: fixed edits through 100k bindings, corruption and amplification regressions, publication/GC consumers, CLI build and 63 real RustFS checks with independent current and historical hydration. Full maintenance streaming and final-format 10GiB qualification remain open; older format-7 large-file receipts do not prove this format. --- .../docs/architecture/indexed-remote-reads.md | 45 +- crab/src/cmd/gc/captured_index_tests.rs | 33 +- crab/src/cmd/gc/mod.rs | 142 +++++- crab/src/cmd/history_recovery_v2.rs | 10 +- .../cmd/history_recovery_v2/layered_tests.rs | 2 +- crab/src/cmd/metadb/capsule_tests.rs | 2 +- crates/crab-metadata/README.md | 32 +- .../src/capsule_protocol/browse_retention.rs | 21 + .../src/capsule_protocol/history.rs | 4 +- .../src/capsule_protocol/layered.rs | 36 +- .../src/capsule_protocol/layered/catalog.rs | 287 +++++++---- .../layered/visibility_pages/reader.rs | 6 +- .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/pointer.rs | 70 +++ .../src/capsule_protocol/pointer/object.rs | 254 ++++++++++ .../capsule_protocol/pointer/object/reader.rs | 464 ++++++++++++++++++ .../capsule_protocol/pointer/object/tests.rs | 283 +++++++++++ .../src/capsule_protocol/pointer/paged.rs | 55 +-- .../capsule_protocol/pointer/paged/reader.rs | 183 +++---- .../pointer/paged/reader/tests.rs | 78 +-- .../src/capsule_protocol/root.rs | 5 +- .../capsule_protocol/run/admission/reader.rs | 6 +- .../capsule_protocol/run/catalog/reader.rs | 13 +- .../visibility/paged/reader.rs | 19 +- .../src/file_index_lookup/catalog_tests.rs | 17 +- .../src/file_index_lookup/shared_tests.rs | 7 +- .../src/git_object_locator/paged/reader.rs | 6 +- crates/crab-metadata/src/pages.rs | 2 + crates/crab-metadata/src/pages/cache.rs | 61 ++- crates/crab-metadata/src/pages/object.rs | 283 +++++++++++ .../crab-metadata/src/pages/object/store.rs | 411 ++++++++++++++++ .../crab-metadata/src/pages/object/tests.rs | 326 ++++++++++++ crates/crab-metadata/src/path_state/reader.rs | 11 +- .../src/split_commit_graph/reader.rs | 6 +- crates/crab-read/src/capsule_protocol.rs | 38 +- crates/crab-remote/Cargo.toml | 1 + crates/crab-remote/src/checkpoint.rs | 37 +- crates/crab-remote/tests/checkpoint.rs | 16 +- .../tests/checkpoint/publication_catalog.rs | 65 ++- crates/crab-write/src/capsule_protocol.rs | 2 +- 40 files changed, 2891 insertions(+), 452 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs create mode 100644 crates/crab-metadata/src/pages/object.rs create mode 100644 crates/crab-metadata/src/pages/object/store.rs create mode 100644 crates/crab-metadata/src/pages/object/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 6075c2330..2f98897c2 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2441,7 +2441,7 @@ and affected-consumer validation must accompany this format change before push. ### Independent checkpoint catalog lifetime -Checkpoint format 8 (`CRBCKP08`) replaces the inline catalog section with an +The preceding checkpoint format 8 (`CRBCKP08`) replaces the inline catalog section with an immutable `v2/catalogs//` commitment containing the full body hash, encoded size and authenticated page roots. Selected queries keep the same bounded page reader and complete-or-error dependency closure. Checkpoint loading @@ -2477,3 +2477,46 @@ admission before catalog reads; missing/corrupt pages; ordinary/restore publication failures; and current/history/retained-only catalog survival through GC with the existing grace period and fences. No format-8 10 GiB or hosted-S3 performance acceptance is established by the earlier combined-format run. + + +### Copy-on-write checkpoint catalog: format 9 under qualification + +Checkpoint format 9 (`CRBCKP09`) replaces the monolithic format-8 catalog with +an authenticated descriptor (`CRBCAT02`, at most 4 KiB) and content-addressed +`CRABOP01` page objects. The existing `v2/catalogs/` namespace holds both; +the descriptor binds four independently paged trees: file bindings, shard +recipes, xorb recipes and reverse chunk placements. Nested recipes also use +paged trees. Directories carry bounded child references, including ordering, +cardinality, height, encoded size and hash commitments. + +The checkpoint writer folds selected capsule deltas in captured publication +order. It validates changed file sizes and immutable shard/xorb descriptors, +resolves selected dependencies, and rewrites only affected tree paths. An +unchanged catalog carries its existing descriptor with zero catalog object +I/O. New pages are persisted and verified before the new descriptor, then the +checkpoint/history records, then the visible root transition. The caller keeps +its GC fence across this sequence. Abandoned immutable pages remain ordinary +grace-period orphans; they never advertise a partially published catalog. + +One aggregate byte budget admits packed-delta reads, object-page reads and new +page writes. A separate logical-expansion counter allows four times the encoded ceiling +for vector/string overhead and simultaneous row/typed values. It charges every +visit/result copy, even when physical pages are deduplicated or cached. +Incoming catalog row payload is admitted before recipe/assembly duplication; +writers also enforce final encoded size. The object reader caps its decoded +cache at 8 MiB/256 entries and shares four I/O lanes across recursive edits. + +GC authenticates page closures for current checkpoints, historical checkpoints +and retained browse captures. A coordinator-protected descriptor key does not +count as a verified closure. Missing/corrupt descendants abort maintenance. +Full catalog maintenance still materializes logical metadata within its caller +budget; streaming full inspection remains outstanding. + +Release audit: all six upstream published tag objects were checked again before +this change; none contains the capsule codec. Unshipped checkpoint formats +through 8 are rejected rather than interpreted through an alternate reader. +Earlier format-7/8 benchmark receipts remain historical evidence. Format-9 +acceptance requires changed-catalog byte bounds, old/new cold reconstruction, +corruption/admission tests, GC/retention coverage and a new real RustFS run. +The earlier 10 GiB matrix does not qualify this new format. RustFS results do +not establish representative hosted-S3 latency. diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index dd56d0c60..2703dbfb7 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -615,7 +615,18 @@ async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { .unwrap(); let source = first.layered_checkpoint().unwrap().catalog().unwrap(); let retained = layout.capsule_catalog_path(source.hash()); - backend.paths.lock().unwrap().insert(retained.clone()); + let original_catalog = catalog.clone(); + let retained_pages = first + .layered_checkpoint() + .unwrap() + .catalog_objects(&layout, LIMIT) + .await + .unwrap(); + backend + .paths + .lock() + .unwrap() + .extend(retained_pages.into_iter().map(|(path, _)| path)); // Publish a different current catalog; the old one is now history-owned. catalog .insert_file("c".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) @@ -638,7 +649,17 @@ async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { .unwrap() .hash(), ); - backend.paths.lock().unwrap().insert(current_path.clone()); + let current_pages = current + .layered_checkpoint() + .unwrap() + .catalog_objects(&layout, LIMIT) + .await + .unwrap(); + backend + .paths + .lock() + .unwrap() + .extend(current_pages.into_iter().map(|(path, _)| path)); let mut orphans = Vec::new(); for (index, aged) in [(1, true), (2, false)] { let bytes = Bytes::from(format!("unreferenced-{index}")); @@ -678,4 +699,12 @@ async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { }); } assert_eq!(existing, [true, true, false, true]); + // All dependent pages are aged: cold reads prove traversal, not grace-period luck. + assert_eq!( + ( + first.pointer_catalog().await.unwrap(), + current.pointer_catalog().await.unwrap() + ), + (original_catalog, catalog), + ); } diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index a341b72e8..7f18d9a2e 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3191,13 +3191,15 @@ async fn sweep_capsule_objects( started: Instant, ) -> Result { let mut reachable = HashSet::new(); + let mut verified_catalogs = HashSet::new(); if let Some(checkpoint) = root.checkpoint() { reachable.insert( layout .capsule_checkpoint_path(checkpoint.hash()) .to_string(), ); - mark_layered_checkpoint_sources(layout, checkpoint, &mut reachable).await?; + mark_layered_checkpoint_sources(layout, checkpoint, &mut reachable, &mut verified_catalogs) + .await?; } reachable.extend( capsule_runs @@ -3224,7 +3226,13 @@ async fn sweep_capsule_objects( .capsule_checkpoint_path(segment.checkpoint().hash()) .to_string(), ); - mark_layered_checkpoint_sources(layout, segment.checkpoint(), &mut reachable).await?; + mark_layered_checkpoint_sources( + layout, + segment.checkpoint(), + &mut reachable, + &mut verified_catalogs, + ) + .await?; reachable.extend( segment .capsule_runs() @@ -3234,7 +3242,14 @@ async fn sweep_capsule_objects( } } reachable.extend(coordinator_protected_keys.iter().cloned()); - mark_derived_index_objects(store.as_storage(), layout, manifest, &mut reachable).await?; + mark_derived_index_objects( + store.as_storage(), + layout, + manifest, + &mut reachable, + &mut verified_catalogs, + ) + .await?; let capsule_prefix = layout.repo_path("v2/capsules/"); let checkpoint_prefix = layout.repo_path("v2/checkpoints/"); @@ -3399,6 +3414,7 @@ async fn mark_derived_index_objects( layout: &crab_storage::StoreLayout, manifest: &crab_metadata::manifests::Manifest, reachable: &mut HashSet, + verified_catalogs: &mut HashSet, ) -> Result<()> { let current = crab_metadata::capsule_protocol::load_browse_indexes(layout).await?; let roots = crab_metadata::capsule_protocol::load_retained_browse_indexes(layout).await?; @@ -3415,6 +3431,9 @@ async fn mark_derived_index_objects( // pages. Repacking cannot substitute a new source behind those ranges. for root in roots { reachable.insert(layout.browse_retained_root_path(root.hash()).to_string()); + if let Some(pointer) = root.checkpoint() { + mark_layered_checkpoint_sources(layout, pointer, reachable, verified_catalogs).await?; + } reachable.extend( root.source_paths(layout) .into_iter() @@ -3515,19 +3534,27 @@ async fn mark_layered_checkpoint_sources( layout: &crab_storage::StoreLayout, pointer: &crab_metadata::capsule_protocol::CheckpointPointer, reachable: &mut HashSet, + verified_catalogs: &mut HashSet, ) -> Result<()> { let checkpoint = - crab_metadata::capsule_protocol::load_layered_checkpoint(layout, pointer, u64::MAX) + crab_metadata::capsule_protocol::load_layered_checkpoint_control(layout, pointer) .await .map_err(CrabError::from)?; if let Some(catalog) = checkpoint.catalog() { let path = layout.capsule_catalog_path(catalog.hash()).to_string(); - // Ref-only checkpoints share one catalog. Authenticate it once per - // sweep, so retained checkpoint count does not multiply body decoding. - if reachable.insert(path) { - checkpoint - .pointer_catalog_from_store(layout, u64::MAX) - .await?; + // Checkpoints and captured readers share immutable catalog subtrees. + // Authenticate each root's complete closure before sweeping page objects. + reachable.insert(path.clone()); + // Coordinator protection may have marked only the descriptor key. + // Reachability alone does not prove its children were traversed. + if verified_catalogs.insert(path) { + reachable.extend( + checkpoint + .catalog_objects(layout, u64::MAX) + .await? + .into_iter() + .map(|(path, _)| path.to_string()), + ); } } for source in checkpoint.sources() { @@ -5461,6 +5488,87 @@ mod tests { assert_eq!(view.refs().get("refs/heads/main"), Some(&"2".repeat(40))); } + #[tokio::test] + async fn protected_catalog_descriptor_still_requires_verified_page_closure() { + use bytes::Bytes; + use crab_metadata::capsule_protocol::{ + CapsuleGitPack, FileCatalogEntry, LayeredCheckpoint, PackLayer, PointerCatalog, + ShardCatalogEntry, + }; + let store = + crab_storage::Store::new(std::sync::Arc::new(object_store::memory::InMemory::new())); + let layout = crab_storage::StoreLayout::new(store, "catalog-protection".to_owned()); + let pack = CapsuleGitPack::new( + Bytes::from_static(b"pack"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "1".repeat(40), + 1, + ) + .unwrap(); + let source = PackLayer::build(&pack) + .unwrap() + .source_descriptor() + .unwrap(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(1, vec![])) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(1, "a".repeat(64))) + .unwrap(); + let checkpoint = + LayeredCheckpoint::build(1, &"c".repeat(64), vec![source], catalog, None).unwrap(); + checkpoint.publish_catalog(&layout).await.unwrap(); + layout + .store() + .put_overwrite( + &layout.capsule_checkpoint_path(checkpoint.hash()), + checkpoint.bytes().clone(), + ) + .await + .unwrap(); + let root = layout + .capsule_catalog_path(checkpoint.catalog().unwrap().hash()) + .to_string(); + let expected = checkpoint + .catalog_objects(&layout, 1024 * 1024) + .await + .unwrap(); + let mut reachable = HashSet::from([root.clone()]); + let mut verified = HashSet::new(); + mark_layered_checkpoint_sources( + &layout, + &checkpoint.pointer().unwrap(), + &mut reachable, + &mut verified, + ) + .await + .unwrap(); + assert!( + expected + .iter() + .all(|(path, _)| reachable.contains(&path.to_string())) + ); + + let (child, _) = expected + .iter() + .find(|(path, _)| path.to_string() != root) + .unwrap(); + layout.store().delete(child).await.unwrap(); + assert!( + mark_layered_checkpoint_sources( + &layout, + &checkpoint.pointer().unwrap(), + &mut reachable, + &mut HashSet::new() + ) + .await + .is_err() + ); + } + #[tokio::test] async fn capsule_protocol_gc_retains_layered_checkpoint_history_and_sources() { use bytes::Bytes; @@ -5652,8 +5760,6 @@ mod tests { Err(CrabError::NotFound { .. }) )); } - assert_eq!(outcome.list_requests, 7); - assert_eq!(outcome.list_parallelism, 7); } async fn write_browse_indexes( @@ -5860,9 +5966,15 @@ mod tests { .unwrap(), }); let mut retained = HashSet::new(); - mark_derived_index_objects(store.as_storage(), &layout, &manifest, &mut retained) - .await - .unwrap(); + mark_derived_index_objects( + store.as_storage(), + &layout, + &manifest, + &mut retained, + &mut HashSet::new(), + ) + .await + .unwrap(); assert!( retained.contains( &layout diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index 2672217e3..d04677c8c 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -225,8 +225,9 @@ async fn verify_history( .map_err(|_| CrabError::Internal("history shard count overflowed".to_owned()))?; let xorb_count = u64::try_from(catalog.xorbs().len()) .map_err(|_| CrabError::Internal("history xorb count overflowed".to_owned()))?; + let catalog_objects = checkpoint.catalog_objects(layout, u64::MAX).await?; let dependency_objects = 2_u64 - .checked_add(u64::from(checkpoint.catalog().is_some())) + .checked_add(catalog_objects.len() as u64) .and_then(|total| total.checked_add(source_count)) .and_then(|total| total.checked_add(shard_count)) .and_then(|total| total.checked_add(xorb_count)) @@ -234,7 +235,12 @@ async fn verify_history( .ok_or_else(|| CrabError::Internal("history dependency count overflowed".to_owned()))?; let segment_bytes = u64::try_from(segment.bytes().len()) .map_err(|_| CrabError::Internal("history segment size overflowed".to_owned()))?; - let checkpoint_bytes = checkpoint.encoded_metadata_bytes()?; + let checkpoint_bytes = catalog_objects + .iter() + .try_fold(checkpoint.pointer()?.size(), |total, (_, bytes)| { + total.checked_add(*bytes) + }) + .ok_or_else(|| CrabError::Internal("history catalog byte count overflowed".to_owned()))?; let dependency_bytes = segment_bytes .checked_add(checkpoint_bytes) .and_then(|total| total.checked_add(source_bytes)) diff --git a/crab/src/cmd/history_recovery_v2/layered_tests.rs b/crab/src/cmd/history_recovery_v2/layered_tests.rs index a0d0027b8..9c7018fae 100644 --- a/crab/src/cmd/history_recovery_v2/layered_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_tests.rs @@ -129,7 +129,7 @@ async fn layered_history_verification_preserves_historical_refs_and_bytes() { let (layout, root) = fixture().await; let chain = history_chain(&layout, &root).await.unwrap(); let oldest = chain.last().unwrap(); - assert_eq!(oldest.checkpoint().format(), 8); + assert_eq!(oldest.checkpoint().format(), 9); let verified = verify_history( &layout, diff --git a/crab/src/cmd/metadb/capsule_tests.rs b/crab/src/cmd/metadb/capsule_tests.rs index df948c274..89104ee9e 100644 --- a/crab/src/cmd/metadb/capsule_tests.rs +++ b/crab/src/cmd/metadb/capsule_tests.rs @@ -109,7 +109,7 @@ async fn layered_diagnosis_and_rebuild_verify_nonempty_repository_without_rewrit .await .unwrap(); let before = root.record().digest().to_owned(); - assert_eq!(root.record().root().checkpoint().unwrap().format(), 8); + assert_eq!(root.record().root().checkpoint().unwrap().format(), 9); let diagnosis = diagnose_capsule(&layout, root.clone(), DbSelector::Both, true, &cancel) .await .unwrap(); diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index d0bff8872..cc95b1537 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -60,15 +60,29 @@ capsules can therefore appear in differently compacted ref frontiers without inflating pack counts or appearing to conflict; differing payload commitments remain rejected. -Checkpoint format 8 (`CRBCKP08`) commits to a separate immutable catalog at -`v2/catalogs//`, including its exact size and authenticated -page roots. Catalog durability precedes checkpoint/history writes and root CAS. -Ref-only checkpoint publication and physical repacking reuse that commitment -without decoding, re-encoding, reading or rewriting the catalog body. A visible -catalog delta still uses the canonical ordered full assembly; incremental delta -updates remain required. Full checkpoint loading verifies visibility and catalog -commitments; explicit maintenance materializes and verifies the catalog body. -GC follows catalog objects from current, historical and retained browse roots. +Checkpoint format 9 (`CRBCKP09`) commits to a small catalog descriptor and +immutable content-addressed pages under `v2/catalogs//`. +Descriptor version 2 (`CRBCAT02`) authenticates the roots of file bindings, +shards, xorbs and reverse chunk placements. Internal directories and nested +recipes use independent pages (`CRABOP01`); unchanged subtrees retain their +identities across checkpoint publication. New pages and the descriptor are +verified before checkpoint/history writes and root CAS. Ref-only publication +and physical repacking reuse the existing commitment without catalog object I/O. +A catalog delta resolves its selected dependency closure and rewrites affected +paths, preserving ordered file bindings and immutable descriptor conflicts. + +Readers share an aggregate page-byte budget across checkpoint and packed-run +sources. Logical expansion has a separate allowance of four times the encoded ceiling: shared +physical pages cannot authorize unbounded recipe/result copies. Cold and warm +reads consume admission; decoded caches have independent resident limits. +Full inspection authenticates every dependency and reverse placement; it still +materializes the logical catalog and is not yet a streaming-memory guarantee. +GC traverses catalog pages from current, historical and retained browse roots. +Protected keys and authenticated closure traversal are tracked separately. +Retired unshipped checkpoint formats through 8 are rejected without fallback. +The six published release tags do not contain this capsule codec. +This format is under qualification; earlier format-7/8 runtime receipts do not +qualify format 9 or establish hosted-S3 performance. Retained browse record format 2 carries the complete checkpoint pointer and catalog identity; loading authenticates the footer and rejects omitted or mismatched catalog/source dependencies. `LayeredCheckpoint::pointer_catalog_lookup` captures these roots; diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 773f76a7b..168d9480a 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -58,6 +58,12 @@ impl RetainedBrowseIndexes { &self.payload.indexes } + /// Return the immutable checkpoint whose dependency pages belong to this capture. + #[must_use] + pub fn checkpoint(&self) -> Option<&CheckpointPointer> { + self.payload.checkpoint.as_ref() + } + /// Return original source bodies, including a captured checkpoint when present. #[must_use] pub fn source_paths(&self, layout: &StoreLayout) -> Vec { @@ -338,6 +344,21 @@ pub async fn verify_retained_browse_indexes( } let mut sources = BTreeSet::new(); for record in records { + if let Some(pointer) = record.checkpoint() { + let checkpoint = super::load_layered_checkpoint_control(layout, pointer).await?; + if let Some(catalog) = checkpoint.catalog() + && !sources.contains(&layout.capsule_catalog_path(catalog.hash())) + { + sources.extend( + checkpoint + .catalog_objects(layout, maximum_bytes) + .await? + .into_iter() + .map(|(path, _)| path), + ); + } + } + let graph = crate::split_commit_graph::load_split_commit_graph( layout.store(), layout, diff --git a/crates/crab-metadata/src/capsule_protocol/history.rs b/crates/crab-metadata/src/capsule_protocol/history.rs index bfdc03249..86864e70e 100644 --- a/crates/crab-metadata/src/capsule_protocol/history.rs +++ b/crates/crab-metadata/src/capsule_protocol/history.rs @@ -467,8 +467,8 @@ mod tests { fn history_rejects_implicit_checkpoint_format() { let segment = HistorySegment::build(checkpoint(), None, state()).unwrap(); let body = std::str::from_utf8(segment.bytes()).unwrap(); - assert_eq!(body.matches("\"format\":8,").count(), 1); - let bytes = Bytes::from(body.replace("\"format\":8,", "")); + assert_eq!(body.matches("\"format\":9,").count(), 1); + let bytes = Bytes::from(body.replace("\"format\":9,", "")); let error = HistorySegment::decode(bytes) .expect_err("an implicit retired checkpoint format must fail"); diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index 8c2779f8b..0bc6f2e46 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -12,15 +12,17 @@ use crate::git_visibility::{ use crate::validation::{validate_content_hash, validate_sha1}; mod catalog; -use catalog::CatalogInput; pub use catalog::CheckpointCatalog; +#[cfg(feature = "storage")] +pub use catalog::fold_checkpoint_catalog; +use catalog::{CatalogInput, PreparedCatalog}; mod visibility_pages; #[cfg(feature = "storage")] pub use visibility_pages::LayeredVisibilityReader; -const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP08"; -pub(super) const CHECKPOINT_VERSION: u32 = 8; +const CHECKPOINT_MAGIC: &[u8; 8] = b"CRBCKP09"; +pub(super) const CHECKPOINT_VERSION: u32 = 9; const LAYER_MAGIC: &[u8; 8] = b"CRBPKL01"; const LAYER_VERSION: u32 = 1; const TRAILER_BYTES: usize = 8 + 32 + 8; @@ -961,12 +963,12 @@ pub struct LayeredCheckpoint { object_size: u64, control_offset: u64, control_only: bool, - catalog_bytes: Option, + catalog_objects: BTreeMap<[u8; 32], Bytes>, } impl PartialEq for LayeredCheckpoint { fn eq(&self, other: &Self) -> bool { - // Loaded/pending catalog bytes are runtime state. The footer already + // Pending catalog objects are runtime state. The footer already // commits to their exact identity, independently of cache residency. self.bytes == other.bytes && self.hash == other.hash @@ -998,10 +1000,11 @@ impl LayeredCheckpoint { ) } - /// Reuse a captured immutable catalog when no visible catalog delta changed it. + /// Build a checkpoint around an already durable immutable catalog root. /// - /// The caller retains the source checkpoint's catalog through publication. - pub fn build_reusing_catalog( + /// The caller verifies newly folded pages or retains a captured root, and + /// holds their GC fence until the replacement checkpoint becomes visible. + pub fn build_with_catalog_root( covered_generation: u64, covered_root_digest: &str, sources: Vec, @@ -1012,7 +1015,7 @@ impl LayeredCheckpoint { covered_generation, covered_root_digest, sources, - CatalogInput::Retained(catalog.map(Box::new)), + CatalogInput::Durable(catalog.map(Box::new)), visibility, ) } @@ -1085,7 +1088,10 @@ impl LayeredCheckpoint { } let mut body = Vec::new(); let mut sections = Vec::new(); - let (catalog, catalog_bytes) = catalog.prepare()?; + let PreparedCatalog { + source: catalog, + objects: catalog_objects, + } = catalog.prepare()?; if let Some((kind, visibility)) = visibility { append_section(&mut body, &mut sections, kind, visibility)?; } @@ -1131,7 +1137,7 @@ impl LayeredCheckpoint { body.extend_from_slice(blake3::hash(&footer_bytes).as_bytes()); body.extend_from_slice(CHECKPOINT_MAGIC); let mut checkpoint = Self::decode(Bytes::from(body))?; - checkpoint.catalog_bytes = catalog_bytes; + checkpoint.catalog_objects = catalog_objects; Ok(checkpoint) } @@ -1155,7 +1161,7 @@ impl LayeredCheckpoint { object_size, control_offset, control_only: false, - catalog_bytes: None, + catalog_objects: BTreeMap::new(), }) } @@ -1219,7 +1225,7 @@ impl LayeredCheckpoint { object_size, control_offset: footer_start, control_only: true, - catalog_bytes: None, + catalog_objects: BTreeMap::new(), }) } @@ -1794,12 +1800,12 @@ mod tests { .unwrap(); let checkpoint = LayeredCheckpoint::build(0, &"3".repeat(64), vec![source], catalog, None).unwrap(); - for field in ["hash", "bytes", "body_bytes"] { + for field in ["hash", "bytes", "page_bytes"] { let mut footer = serde_json::to_value(&checkpoint.footer).unwrap(); match field { "hash" => footer["catalog"]["hash"] = serde_json::json!("invalid"), "bytes" => footer["catalog"]["bytes"] = serde_json::json!(0), - _ => footer["catalog"]["pages"]["body_bytes"] = serde_json::json!(u64::MAX), + _ => footer["catalog"]["pages"]["files"]["bytes"] = serde_json::json!(0), } let footer = serde_json::from_value(footer).unwrap(); assert!( diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs index 157a2edfc..07ed6fef6 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs @@ -1,7 +1,10 @@ use super::*; -use crate::capsule_protocol::pointer::paged::{Descriptor, MAX_CATALOG_BYTES}; +#[cfg(feature = "storage")] +use crate::capsule_protocol::pointer::object::CatalogPages; +use crate::capsule_protocol::pointer::object::{Descriptor, MAX_DESCRIPTOR_BYTES}; +use std::collections::BTreeMap; -/// Authenticated page roots and identity of an immutable checkpoint catalog. +/// Authenticated roots and identity of an immutable object-paged checkpoint catalog. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CheckpointCatalog { @@ -11,54 +14,54 @@ pub struct CheckpointCatalog { } impl CheckpointCatalog { - /// Return the content identity retained by checkpoints and captured readers. #[must_use] pub fn hash(&self) -> &str { &self.hash } - /// Return the full encoded size used for read admission. + /// Return the descriptor byte length admitted during repository opening. #[must_use] pub const fn encoded_bytes(&self) -> u64 { self.bytes } - pub(super) fn validate(&self) -> Result<()> { - validate_content_hash(&self.hash, "checkpoint catalog", "layered checkpoint")?; - if self.bytes == 0 || self.bytes > MAX_CATALOG_BYTES { - return Err(corrupt("checkpoint catalog size is out of bounds")); - } - self.pages.validate(self.bytes) + fn new(pages: Descriptor) -> Result { + let body = pages.encode()?; + Ok(Self { + hash: blake3::hash(&body).to_hex().to_string(), + bytes: body.len() as u64, + pages, + }) } - #[cfg(feature = "storage")] - fn verify(&self, bytes: &Bytes) -> Result<()> { - self.validate()?; - if bytes.len() as u64 != self.bytes - || blake3::hash(bytes).to_hex().as_str() != self.hash - || Descriptor::decode(bytes)? != self.pages + pub(super) fn validate(&self) -> Result<()> { + validate_content_hash(&self.hash, "checkpoint catalog", "layered checkpoint")?; + let body = self.pages.encode()?; + if self.bytes == 0 + || self.bytes > MAX_DESCRIPTOR_BYTES + || body.len() as u64 != self.bytes + || blake3::hash(&body).to_hex().as_str() != self.hash { return Err(corrupt( - "checkpoint catalog differs from its committed roots", + "checkpoint catalog descriptor differs from its commitment", )); } Ok(()) } #[cfg(feature = "storage")] - async fn read(&self, layout: &crab_storage::StoreLayout) -> Result { - let hash = blake3::Hash::from_hex(&self.hash) - .map_err(|_| corrupt("checkpoint catalog has an invalid content identity"))?; - let bytes = layout - .store() - .verify_bounded( - &layout.capsule_catalog_path(&self.hash), - hash.as_bytes(), - self.bytes, - ) - .await?; - self.verify(&bytes)?; - Ok(bytes) + fn reader( + &self, + layout: &crab_storage::StoreLayout, + ) -> Result { + self.validate()?; + let identity = super::super::pointer::hash(&self.hash)?; + CatalogPages::new( + layout.capsule_catalog_prefix(), + self.pages.clone(), + identity, + self.bytes, + ) } #[cfg(feature = "storage")] @@ -67,46 +70,69 @@ impl CheckpointCatalog { layout: &crab_storage::StoreLayout, maximum: u64, ) -> Result { - self.validate()?; - super::super::PointerCatalogLookup::checkpoint( - layout.capsule_catalog_path(&self.hash), - PackRange::from_parts(0, self.bytes, self.hash.clone())?, - self.pages.clone(), + Ok(super::super::PointerCatalogLookup::checkpoint( + std::sync::Arc::new(self.reader(layout)?), maximum, - ) + )) } } pub(super) enum CatalogInput { New(PointerCatalog), - Retained(Option>), + Durable(Option>), +} + +pub(super) struct PreparedCatalog { + pub(super) source: Option, + pub(super) objects: BTreeMap<[u8; 32], Bytes>, } impl CatalogInput { - pub(super) fn prepare(self) -> Result<(Option, Option)> { + pub(super) fn prepare(self) -> Result { let catalog = match self { Self::New(catalog) => catalog, - Self::Retained(source) => { + Self::Durable(source) => { if let Some(source) = &source { source.validate()?; } - return Ok((source.map(|source| *source), None)); + return Ok(PreparedCatalog { + source: source.map(|source| *source), + objects: BTreeMap::new(), + }); } }; if catalog.is_empty() { - return Ok((None, None)); + return Ok(PreparedCatalog { + source: None, + objects: BTreeMap::new(), + }); } - let bytes = catalog.encode()?; - let source = CheckpointCatalog { - hash: blake3::hash(&bytes).to_hex().to_string(), - bytes: bytes.len() as u64, - pages: Descriptor::decode(&bytes)?, - }; - source.validate()?; - Ok((Some(source), Some(bytes))) + let (pages, objects) = Descriptor::build(&catalog)?; + Ok(PreparedCatalog { + source: Some(CheckpointCatalog::new(pages)?), + objects, + }) } } +#[cfg(feature = "storage")] +async fn publish_objects( + layout: &crab_storage::StoreLayout, + objects: &BTreeMap<[u8; 32], Bytes>, +) -> Result<()> { + for (identity, body) in objects { + if blake3::hash(body).as_bytes() != identity { + return Err(corrupt("pending catalog object identity differs")); + } + let identity = blake3::Hash::from_bytes(*identity).to_hex().to_string(); + layout + .store() + .put_if_absent_verified(&layout.capsule_catalog_path(&identity), body.clone()) + .await?; + } + Ok(()) +} + impl LayeredCheckpoint { /// Capture the authenticated checkpoint identity, including footer-only readers. pub fn pointer(&self) -> Result { @@ -123,72 +149,148 @@ impl LayeredCheckpoint { ) } - /// Return the aggregate encoded checkpoint and catalog size. + /// Return the checkpoint and root-descriptor bytes admitted when opening a view. pub fn encoded_metadata_bytes(&self) -> Result { self.object_size .checked_add(self.catalog().map_or(0, CheckpointCatalog::encoded_bytes)) .ok_or_else(|| corrupt("checkpoint dependency size overflows")) } - /// Return the catalog commitment without loading catalog pages. + /// Return all locally prepared immutable bytes before publication admission. + pub fn publication_metadata_bytes(&self) -> Result { + if self.catalog_objects.is_empty() { + return self.encoded_metadata_bytes(); + } + self.catalog_objects + .values() + .try_fold(self.object_size, |bytes, body| { + bytes + .checked_add(body.len() as u64) + .ok_or_else(|| corrupt("prepared catalog byte count overflows")) + }) + } + #[must_use] pub fn catalog(&self) -> Option<&CheckpointCatalog> { self.footer.catalog.as_ref() } - /// Persist newly built catalog bytes before publishing their checkpoint. + /// Verify and persist every newly prepared catalog object before its checkpoint. /// - /// A retained commitment must come from a captured published checkpoint; - /// the caller owns its GC fence until the replacement root is visible. + /// Retained roots must already be durable; the caller retains its GC fence + /// until the replacement root becomes visible. #[cfg(feature = "storage")] pub async fn publish_catalog( &self, layout: &crab_storage::StoreLayout, ) -> Result<()> { - if let (Some(source), Some(bytes)) = (&self.footer.catalog, &self.catalog_bytes) { - source.verify(bytes)?; - layout - .store() - .put_if_absent_verified(&layout.capsule_catalog_path(source.hash()), bytes.clone()) - .await?; + if let Some(source) = &self.footer.catalog + && !self.catalog_objects.is_empty() + { + let objects = &self.catalog_objects; + source.validate()?; + let identity = super::super::pointer::hash(&source.hash)?; + let body = objects + .get(&identity) + .ok_or_else(|| corrupt("pending catalog has no root descriptor"))?; + if Descriptor::decode(body)? != source.pages { + return Err(corrupt("pending catalog roots differ")); + } + publish_objects(layout, objects).await?; } Ok(()) } - /// Materialize only the catalog body using its captured origin and byte limit. + /// Materialize the catalog from its authenticated origin with one aggregate page budget. #[cfg(feature = "storage")] pub async fn pointer_catalog_from_store( &self, layout: &crab_storage::StoreLayout, maximum_bytes: u64, ) -> Result { - let Some(source) = &self.footer.catalog else { + let Some(source) = self.catalog() else { return Ok(PointerCatalog::new()); }; - if source.bytes > maximum_bytes { - return Err(MetadataError::CheckpointSize { - bytes: source.bytes, - maximum: maximum_bytes, - }); - } - if self.catalog_bytes.is_some() { - return self.pointer_catalog(); - } - PointerCatalog::decode(&source.read(layout).await?) + source + .reader(layout)? + .full( + layout.store(), + &mut crate::pages::PageWork::new(maximum_bytes, 0), + ) + .await } + /// Authenticate catalog objects and their encoded sizes for maintenance. #[cfg(feature = "storage")] - fn pointer_catalog(&self) -> Result { - let Some(source) = &self.footer.catalog else { - return Ok(PointerCatalog::new()); + pub async fn catalog_objects( + &self, + layout: &crab_storage::StoreLayout, + maximum_bytes: u64, + ) -> Result> { + let Some(source) = self.catalog() else { + return Ok(Vec::new()); }; - let bytes = self - .catalog_bytes - .as_ref() - .ok_or_else(|| corrupt("checkpoint catalog bytes have not been loaded"))?; - source.verify(bytes)?; - PointerCatalog::decode(bytes) + source + .reader(layout)? + .inspected_objects(layout.store(), maximum_bytes) + .await + } +} + +/// Fold selected ordered deltas into a durable catalog without scanning unrelated pages. +/// +/// The caller must retain its captured root and hold its publication GC fence. +/// New immutable pages and the descriptor are verified before this returns. +#[cfg(feature = "storage")] +pub async fn fold_checkpoint_catalog( + layout: &crab_storage::StoreLayout, + existing: Option<&CheckpointCatalog>, + deltas: &[PointerCatalog], + maximum_bytes: u64, +) -> Result> { + if deltas.iter().all(PointerCatalog::is_empty) { + return Ok(existing.cloned()); + } + let Some(existing) = existing else { + super::super::pointer::admit_catalog_rows(deltas, maximum_bytes)?; + let mut assembly = super::super::PointerCatalogAssembly::new(PointerCatalog::new()); + for delta in deltas { + assembly.apply(delta)?; + } + let prepared = CatalogInput::New(assembly.finish()?).prepare()?; + if !prepared.objects.is_empty() { + let objects = &prepared.objects; + let bytes = objects.values().try_fold(0u64, |bytes, body| { + bytes + .checked_add(body.len() as u64) + .ok_or_else(|| corrupt("pending catalog byte count overflows")) + })?; + if bytes > maximum_bytes { + return Err(MetadataError::CheckpointSize { + bytes, + maximum: maximum_bytes, + }); + } + publish_objects(layout, objects).await?; + } + return Ok(prepared.source); + }; + let mut work = crate::pages::PageWork::new(maximum_bytes, 0); + let descriptor = existing + .reader(layout)? + .apply(layout.store(), deltas, &mut work) + .await?; + if descriptor == existing.pages { + return Ok(Some(existing.clone())); } + let source = CheckpointCatalog::new(descriptor)?; + let body = source.pages.encode()?; + work.admit_object(*blake3::hash(&body).as_bytes(), body.len() as u64)?; + layout + .store() + .put_if_absent_verified(&layout.capsule_catalog_path(&source.hash), body) + .await?; + Ok(Some(source)) } #[cfg(all(test, feature = "storage"))] @@ -287,7 +389,7 @@ mod tests { async fn retained_catalog_survives_replacement_of_its_original_checkpoint() { let (layout, checkpoint, catalog) = fixture(); checkpoint.publish_catalog(&layout).await.unwrap(); - let next = LayeredCheckpoint::build_reusing_catalog( + let next = LayeredCheckpoint::build_with_catalog_root( 2, &"d".repeat(64), checkpoint.sources().to_vec(), @@ -321,4 +423,25 @@ mod tests { // catalog directly, rather than borrowing that old object's lifetime. assert_eq!(control.catalog(), checkpoint.catalog()); } + #[tokio::test] + async fn catalog_inspection_reports_every_physical_object_once() { + let (layout, checkpoint, _) = fixture(); + checkpoint.publish_catalog(&layout).await.unwrap(); + let inspected = checkpoint + .catalog_objects(&layout, 1024 * 1024) + .await + .unwrap(); + let stored = layout + .store() + .list_prefix(&layout.capsule_catalog_prefix()) + .await + .unwrap(); + assert_eq!( + inspected.into_iter().collect::>(), + stored + .into_iter() + .map(|meta| (meta.location, meta.size)) + .collect(), + ); + } } diff --git a/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs index 0f545b847..6dbde8534 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered/visibility_pages/reader.rs @@ -50,11 +50,7 @@ impl LayeredVisibilityReader { } fn work(&self) -> PageWork { - PageWork { - maximum: self.maximum, - bytes: 0, - seen: BTreeSet::new(), - } + PageWork::new(self.maximum, 0) } async fn rows( diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 904694a3f..7d2081a4e 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -54,14 +54,14 @@ pub use history::{ HistorySegment, HistorySegmentPointer, HistorySegmentState, MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, }; -#[cfg(feature = "storage")] -pub use layered::LayeredVisibilityReader; pub use layered::{ CheckpointCatalog, LayeredCheckpoint, LayeredObjectMember, LayeredVisibilitySnapshot, PackLayer, PackMemberDescriptor, PackRange, PackSourceDescriptor, PackSourceKind, source_catalog_digest, visibility_object_set_digest, }; #[cfg(feature = "storage")] +pub use layered::{LayeredVisibilityReader, fold_checkpoint_catalog}; +#[cfg(feature = "storage")] pub use plan::{ CapsulePlanReceipt, ensure_capsule_plan_unattempted, prepare_capsule_plan, publish_capsule_plan_receipt, publish_capsule_plan_repair_receipt, read_capsule_plan_intent, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 85255d410..d8f1f31b0 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -9,6 +9,7 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; +pub(in crate::capsule_protocol) mod object; pub(super) mod paged; pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] @@ -182,6 +183,12 @@ impl PointerCatalogAssembly { self.catalog.merge_delta(delta) } + /// Return an ordered delta whose missing dependencies must be resolved from its base. + pub fn finish_delta(self) -> Result { + self.catalog.validate(false, false)?; + Ok(self.catalog) + } + /// Return the captured catalog only if its complete dependency closure is valid. pub fn finish(self) -> Result { self.catalog.validate(true, false)?; @@ -189,6 +196,69 @@ impl PointerCatalogAssembly { } } +pub(in crate::capsule_protocol) fn hash(value: &str) -> Result<[u8; 32]> { + let bytes = value.as_bytes(); + // Binary keys cannot distinguish hex aliases. Reject them instead of silently + // merging two descriptors that were distinct in the caller's map. + if bytes.len() != 64 + || bytes + .iter() + .any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(byte)) + { + return Err(contract_error( + "catalog hashes must be lowercase canonical hex", + )); + } + // Encode text order directly: Xet's raw hash bytes reverse each u64 word. + // Using those bytes as keys would change the original BTreeMap precedence. + let digit = |byte: u8| match byte { + b'0'..=b'9' => byte - b'0', + _ => byte - b'a' + 10, + }; + Ok(std::array::from_fn(|index| { + (digit(bytes[index * 2]) << 4) | digit(bytes[index * 2 + 1]) + })) +} + +fn hex(bytes: &[u8]) -> Result { + if bytes.len() != 32 { + return Err(corrupt("catalog hash has an invalid byte length")); + } + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut value = String::with_capacity(64); + for byte in bytes { + value.push(DIGITS[(byte >> 4) as usize] as char); + value.push(DIGITS[(byte & 15) as usize] as char); + } + Ok(value) +} + +// Bound unavoidable row payload before cloning caller-owned catalogs or recipes. +// Both packed deltas and object-paged checkpoints share this admission rule; +// page framing and actual encoded bytes are checked again by their writers. +pub(in crate::capsule_protocol) fn admit_catalog_rows( + catalogs: &[PointerCatalog], + maximum: u64, +) -> Result<()> { + let bytes = catalogs.iter().try_fold(0u64, |sum, catalog| { + let sum = sum.checked_add((catalog.files.len() as u64).checked_mul(76)?)?; + let sum = catalog.xorbs.values().try_fold(sum, |sum, xorb| { + sum.checked_add(76)? + .checked_add((xorb.chunks.len() as u64).checked_mul(44)?) + })?; + catalog.shards.values().try_fold(sum, |sum, shard| { + sum.checked_add(44)? + .checked_add((shard.xorb_hashes.len() as u64).checked_mul(36)?) + }) + }); + if bytes.is_none_or(|bytes| bytes > maximum.min(MAX_CATALOG_BYTES)) { + return Err(contract_error( + "catalog rows exceed their encoded byte bound", + )); + } + Ok(()) +} + impl PointerCatalog { #[must_use] pub fn new() -> Self { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object.rs new file mode 100644 index 000000000..7836a9c5c --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object.rs @@ -0,0 +1,254 @@ +//! Copy-on-write checkpoint catalog roots and immutable dependency pages. +use super::*; +use crate::pages::object::{Builder, Reference}; +use crate::pages::{Kind, Rows}; + +#[cfg(feature = "storage")] +mod reader; +#[cfg(feature = "storage")] +pub(in crate::capsule_protocol) use reader::CatalogPages; + +const MAGIC: &[u8; 8] = b"CRBCAT02"; +pub(in crate::capsule_protocol) const MAX_DESCRIPTOR_BYTES: u64 = 4096; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(in crate::capsule_protocol) struct Descriptor { + version: u32, + pub(super) files: Option, + pub(super) shards: Option, + pub(super) xorbs: Option, + pub(super) chunks: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ShardRow { + pub(super) encoded_size: u64, + pub(super) xorbs: Option, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct XorbRow { + pub(super) encoded_size: u64, + pub(super) body_digest: [u8; 32], + pub(super) chunks: Reference, +} + +impl Descriptor { + pub(in crate::capsule_protocol) fn validate(&self) -> Result<()> { + if self.version != 2 { + return Err(corrupt("unsupported object catalog version")); + } + for (root, kind) in [ + (&self.files, Kind::CatalogFiles), + (&self.shards, Kind::CatalogShards), + (&self.xorbs, Kind::CatalogXorbs), + (&self.chunks, Kind::CatalogChunks), + ] { + if let Some(root) = root { + root.validate(kind)?; + } + } + Ok(()) + } + + pub(in crate::capsule_protocol) fn encode(&self) -> Result { + self.validate()?; + let mut bytes = json(self)?; + bytes.extend_from_slice(MAGIC); + if bytes.len() as u64 > MAX_DESCRIPTOR_BYTES { + return Err(contract_error( + "object catalog descriptor exceeds its byte limit", + )); + } + Ok(bytes.into()) + } + + #[cfg(feature = "storage")] + pub(in crate::capsule_protocol) fn decode(bytes: &[u8]) -> Result { + if bytes.len() < MAGIC.len() + || bytes.len() as u64 > MAX_DESCRIPTOR_BYTES + || bytes.get(bytes.len() - MAGIC.len()..) != Some(MAGIC.as_slice()) + { + return Err(corrupt("object catalog descriptor framing is invalid")); + } + let descriptor: Self = from_json(&bytes[..bytes.len() - MAGIC.len()])?; + if descriptor.encode()?.as_ref() != bytes { + return Err(corrupt("object catalog descriptor is not canonical")); + } + Ok(descriptor) + } + + pub(in crate::capsule_protocol) fn build( + catalog: &PointerCatalog, + ) -> Result<(Self, BTreeMap<[u8; 32], Bytes>)> { + catalog.validate(true, false)?; + admit_catalog_rows(std::slice::from_ref(catalog), MAX_CATALOG_BYTES)?; + let mut writer = Builder::default(); + let mut xorbs = Vec::new(); + let mut chunks = BTreeMap::new(); + for (identity, entry) in &catalog.xorbs { + let identity = hash(identity)?; + let rows = xorb_chunks(entry, &identity, &mut chunks)?; + let root = writer + .tree(Kind::CatalogXorbChunks, rows)? + .ok_or_else(|| contract_error("object catalog xorb contains no chunks"))?; + let row = XorbRow { + encoded_size: entry.encoded_size, + body_digest: hash(&entry.body_digest)?, + chunks: root, + }; + xorbs.push((identity.to_vec(), json(&row)?)); + } + let mut shards = Vec::new(); + for (identity, entry) in &catalog.shards { + let nested = shard_xorbs(entry)?; + let row = ShardRow { + encoded_size: entry.encoded_size, + xorbs: writer.tree(Kind::CatalogShardXorbs, nested)?, + }; + shards.push((hash(identity)?.to_vec(), json(&row)?)); + } + let descriptor = Self { + version: 2, + files: writer.tree(Kind::CatalogFiles, file_rows(catalog)?)?, + shards: writer.tree(Kind::CatalogShards, shards)?, + xorbs: writer.tree(Kind::CatalogXorbs, xorbs)?, + chunks: writer.tree(Kind::CatalogChunks, chunks.into_iter().collect())?, + }; + let root = descriptor.encode()?; + writer.objects.insert(*blake3::hash(&root).as_bytes(), root); + let bytes = writer.objects.values().try_fold(0u64, |sum, body| { + sum.checked_add(body.len() as u64) + .ok_or_else(|| contract_error("object catalog byte count overflows")) + })?; + if bytes > MAX_CATALOG_BYTES { + return Err(contract_error( + "initial object catalog exceeds its byte limit", + )); + } + Ok((descriptor, writer.objects)) + } +} + +fn json(value: &T) -> Result> { + serde_json::to_vec(value).map_err(|source| MetadataError::PointerCatalogRecord { source }) +} + +#[cfg(feature = "storage")] +fn from_json(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|source| MetadataError::PointerCatalogRecord { source }) +} + +fn file_rows(catalog: &PointerCatalog) -> Result { + catalog + .files + .iter() + .map(|(identity, entry)| { + let mut value = entry.size.to_be_bytes().to_vec(); + value.extend_from_slice(&hash(&entry.shard_hash)?); + Ok((hash(identity)?.to_vec(), value)) + }) + .collect() +} + +fn shard_xorbs(entry: &ShardCatalogEntry) -> Result { + entry + .xorb_hashes + .iter() + .map(|identity| Ok((hash(identity)?.to_vec(), Vec::new()))) + .collect() +} + +fn xorb_chunks( + entry: &XorbCatalogEntry, + identity: &[u8; 32], + chunks: &mut BTreeMap, Vec>, +) -> Result { + entry + .chunks + .iter() + .enumerate() + .map(|(index, chunk)| { + let index = u32::try_from(index) + .map_err(|_| contract_error("object xorb ordinal exceeds its bound"))?; + let key = hash(&chunk.hash)?; + let mut value = identity.to_vec(); + value.extend_from_slice(&index.to_be_bytes()); + value.extend_from_slice(&chunk.uncompressed_size.to_be_bytes()); + // Catalog iteration is text-ordered; the lowest xorb then its first + // occurrence owns the reverse placement, matching publication deltas. + chunks.entry(key.to_vec()).or_insert(value); + let mut value = key.to_vec(); + value.extend_from_slice(&chunk.uncompressed_size.to_be_bytes()); + Ok((index.to_be_bytes().to_vec(), value)) + }) + .collect() +} + +#[cfg(feature = "storage")] +pub(super) fn file_from_bytes(value: &[u8]) -> Result { + if value.len() != 40 { + return Err(corrupt("catalog file binding length differs")); + } + let size = u64::from_be_bytes( + value[..8] + .try_into() + .map_err(|_| corrupt("catalog file size is truncated"))?, + ); + Ok(FileCatalogEntry::new(size, hex(&value[8..])?)) +} + +#[cfg(feature = "storage")] +pub(super) fn shard_from_rows(encoded_size: u64, rows: Rows) -> Result { + if encoded_size == 0 { + return Err(corrupt("catalog shard is empty")); + } + let xorbs = rows + .into_iter() + .map(|(key, value)| { + if !value.is_empty() { + return Err(corrupt("catalog shard closure has unexpected row data")); + } + hex(&key) + }) + .collect::>>()?; + Ok(ShardCatalogEntry::new(encoded_size, xorbs)) +} + +#[cfg(feature = "storage")] +pub(super) fn xorb_from_rows( + encoded_size: u64, + digest: [u8; 32], + rows: Rows, +) -> Result { + if encoded_size == 0 || rows.is_empty() { + return Err(corrupt("catalog xorb is empty")); + } + let chunks = rows + .into_iter() + .enumerate() + .map(|(index, (key, value))| { + let index = u32::try_from(index) + .map_err(|_| corrupt("catalog chunk ordinal exceeds its bound"))?; + if key != index.to_be_bytes() || value.len() != 36 { + return Err(corrupt("catalog chunk position or size differs")); + } + let size = u32::from_be_bytes( + value[32..] + .try_into() + .map_err(|_| corrupt("catalog chunk size is truncated"))?, + ); + if size == 0 { + return Err(corrupt("catalog contains an empty chunk")); + } + Ok(XorbChunkEntry::new(hex(&value[..32])?, size)) + }) + .collect::>()?; + Ok(XorbCatalogEntry::new(encoded_size, hex(&digest)?, chunks)) +} + +#[cfg(all(test, feature = "storage"))] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs new file mode 100644 index 000000000..4ae396979 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs @@ -0,0 +1,464 @@ +use super::*; +use crate::pages::PageWork; +use crate::pages::object::ObjectPages; +use crab_storage::Store; +use object_store::path::Path; +use std::collections::BTreeSet; + +pub(in crate::capsule_protocol) struct CatalogPages { + pub(in crate::capsule_protocol) pages: ObjectPages, + pub(in crate::capsule_protocol) descriptor: Descriptor, + hash: [u8; 32], + bytes: u64, + opened: futures_util::lock::Mutex, +} + +impl std::fmt::Debug for CatalogPages { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CatalogPages") + .field("descriptor", &self.descriptor) + .finish_non_exhaustive() + } +} + +impl CatalogPages { + pub(in crate::capsule_protocol) fn new( + prefix: Path, + descriptor: Descriptor, + hash: [u8; 32], + bytes: u64, + ) -> Result { + let body = descriptor.encode()?; + if body.len() as u64 != bytes || blake3::hash(&body).as_bytes() != &hash { + return Err(corrupt("catalog commitment differs from its descriptor")); + } + Ok(Self { + pages: ObjectPages::new(prefix), + descriptor, + hash, + bytes, + opened: futures_util::lock::Mutex::new(false), + }) + } + + async fn open(&self, store: &Store, work: &mut PageWork) -> Result<()> { + work.admit_object(self.hash, self.bytes)?; + let mut opened = self.opened.lock().await; + if !*opened { + let body = store + .get_with_etag_bounded(&self.pages.path(&self.hash), self.bytes) + .await? + .0; + if blake3::hash(&body).as_bytes() != &self.hash + || Descriptor::decode(&body)? != self.descriptor + { + return Err(corrupt( + "stored catalog descriptor differs from its commitment", + )); + } + *opened = true; + } + Ok(()) + } + + fn root(&self, kind: Kind) -> Result> { + Ok(match kind { + Kind::CatalogFiles => self.descriptor.files.as_ref(), + Kind::CatalogShards => self.descriptor.shards.as_ref(), + Kind::CatalogXorbs => self.descriptor.xorbs.as_ref(), + Kind::CatalogChunks => self.descriptor.chunks.as_ref(), + _ => return Err(corrupt("catalog query selected a nested tree as its root")), + }) + } + + pub(in crate::capsule_protocol) async fn rows( + &self, + store: &Store, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + self.open(store, work).await?; + self.pages + .lookup(store, self.root(kind)?, kind, keys, work) + .await + } + + pub(in crate::capsule_protocol) async fn xorb( + &self, + store: &Store, + bytes: &[u8], + work: &mut PageWork, + ) -> Result { + let row: XorbRow = from_json(bytes)?; + if json(&row)? != bytes { + return Err(corrupt("catalog xorb row is not canonical")); + } + let rows = self + .pages + .scan(store, Some(&row.chunks), Kind::CatalogXorbChunks, work) + .await?; + xorb_from_rows(row.encoded_size, row.body_digest, rows) + } + + pub(in crate::capsule_protocol) async fn shard( + &self, + store: &Store, + bytes: &[u8], + work: &mut PageWork, + ) -> Result { + let row: ShardRow = from_json(bytes)?; + if json(&row)? != bytes { + return Err(corrupt("catalog shard row is not canonical")); + } + let rows = self + .pages + .scan(store, row.xorbs.as_ref(), Kind::CatalogShardXorbs, work) + .await?; + shard_from_rows(row.encoded_size, rows) + } + + pub(in crate::capsule_protocol) async fn full( + &self, + store: &Store, + work: &mut PageWork, + ) -> Result { + self.open(store, work).await?; + let mut catalog = PointerCatalog::new(); + let files = self + .pages + .scan( + store, + self.root(Kind::CatalogFiles)?, + Kind::CatalogFiles, + work, + ) + .await?; + for (key, value) in files { + catalog.insert_file(hex(&key)?, file_from_bytes(&value)?)?; + } + let shards = self + .pages + .scan( + store, + self.root(Kind::CatalogShards)?, + Kind::CatalogShards, + work, + ) + .await?; + for (key, value) in shards { + catalog.insert_shard(hex(&key)?, self.shard(store, &value, work).await?)?; + } + let xorbs = self + .pages + .scan( + store, + self.root(Kind::CatalogXorbs)?, + Kind::CatalogXorbs, + work, + ) + .await?; + let mut expected_chunks = BTreeMap::new(); + for (key, value) in xorbs { + let entry = self.xorb(store, &value, work).await?; + xorb_chunks( + &entry, + &key.as_slice() + .try_into() + .map_err(|_| corrupt("invalid xorb key"))?, + &mut expected_chunks, + )?; + catalog.insert_xorb(hex(&key)?, entry)?; + } + let chunks = self + .pages + .scan( + store, + self.root(Kind::CatalogChunks)?, + Kind::CatalogChunks, + work, + ) + .await?; + if chunks != expected_chunks.into_iter().collect::() { + return Err(corrupt("catalog reverse placements differ from xorb order")); + } + catalog.validate(true, true)?; + Ok(catalog) + } + + pub(in crate::capsule_protocol) async fn inspected_objects( + &self, + store: &Store, + maximum: u64, + ) -> Result> { + let mut work = PageWork::new(maximum, 0); + // A missing or inconsistent dependency must abort a sweep; mere root + // presence cannot authorize deletion of shared persistent pages. + self.full(store, &mut work).await?; + Ok(work + .object_dependencies() + .map(|(hash, bytes)| (self.pages.path(hash), bytes)) + .collect()) + } + + pub(in crate::capsule_protocol) async fn apply( + &self, + store: &Store, + deltas: &[PointerCatalog], + work: &mut PageWork, + ) -> Result { + admit_catalog_rows(deltas, work.maximum.saturating_sub(work.bytes))?; + self.open(store, work).await?; + let mut delta = PointerCatalog::new(); + for piece in deltas { + delta.merge_delta(piece)?; + } + let keys = |values: Vec<&String>| { + values + .into_iter() + .map(|value| Ok(hash(value)?.to_vec())) + .collect::>>() + }; + let xorb_keys = keys(delta.xorbs.keys().collect())?; + let existing = self + .rows(store, Kind::CatalogXorbs, &xorb_keys, work) + .await?; + let mut new_xorbs = BTreeSet::new(); + for ((identity, incoming), existing) in delta.xorbs.iter().zip(existing) { + match existing { + Some(existing) if self.xorb(store, &existing, work).await? != *incoming => { + return Err(contract_error(format!( + "xorb {identity} has conflicting descriptors" + ))); + } + None => { + new_xorbs.insert(identity.clone()); + } + _ => (), + } + } + let shard_keys = keys(delta.shards.keys().collect())?; + let existing = self + .rows(store, Kind::CatalogShards, &shard_keys, work) + .await?; + let mut new_shards = BTreeSet::new(); + for ((identity, incoming), existing) in delta.shards.iter().zip(existing) { + match existing { + Some(existing) if self.shard(store, &existing, work).await? != *incoming => { + return Err(contract_error(format!( + "shard {identity} has conflicting descriptors" + ))); + } + None => { + new_shards.insert(identity.clone()); + } + _ => (), + } + } + let file_keys = keys(delta.files.keys().collect())?; + let existing = self + .rows(store, Kind::CatalogFiles, &file_keys, work) + .await?; + for ((identity, incoming), existing) in delta.files.iter().zip(existing) { + if let Some(existing) = existing + && file_from_bytes(&existing)?.size != incoming.size + { + return Err(contract_error(format!( + "file {identity} has conflicting declared sizes" + ))); + } + } + let required_shards = delta + .files + .values() + .map(|entry| entry.shard_hash.clone()) + .filter(|hash| !delta.shards.contains_key(hash)) + .collect::>(); + let existing = self + .rows( + store, + Kind::CatalogShards, + &keys(required_shards.iter().collect())?, + work, + ) + .await?; + let mut required_xorbs = delta + .shards + .values() + .flat_map(|entry| &entry.xorb_hashes) + .cloned() + .collect::>(); + for row in existing { + let row = + row.ok_or_else(|| contract_error("catalog delta references an absent shard"))?; + required_xorbs.extend(self.shard(store, &row, work).await?.xorb_hashes); + } + required_xorbs.retain(|hash| !delta.xorbs.contains_key(hash)); + let existing = self + .rows( + store, + Kind::CatalogXorbs, + &keys(required_xorbs.iter().collect())?, + work, + ) + .await?; + for row in existing { + let row = + row.ok_or_else(|| contract_error("catalog delta references an absent xorb"))?; + self.xorb(store, &row, work).await?; + } + + let mut descriptor = self.descriptor.clone(); + let mut xorbs = Vec::new(); + let mut chunks = BTreeMap::new(); + for (identity, entry) in &delta.xorbs { + if !new_xorbs.contains(identity) { + continue; + } + let identity = hash(identity)?; + let rows = xorb_chunks(entry, &identity, &mut chunks)?; + let root = self + .pages + .edit(store, None, Kind::CatalogXorbChunks, rows, work) + .await? + .ok_or_else(|| contract_error("catalog xorb contains no chunks"))?; + xorbs.push(( + identity.to_vec(), + json(&XorbRow { + encoded_size: entry.encoded_size, + body_digest: hash(&entry.body_digest)?, + chunks: root, + })?, + )); + } + let mut shards = Vec::new(); + for (identity, entry) in &delta.shards { + if !new_shards.contains(identity) { + continue; + } + let nested = self + .pages + .edit( + store, + None, + Kind::CatalogShardXorbs, + shard_xorbs(entry)?, + work, + ) + .await?; + shards.push(( + hash(identity)?.to_vec(), + json(&ShardRow { + encoded_size: entry.encoded_size, + xorbs: nested, + })?, + )); + } + let chunk_keys = chunks.keys().cloned().collect::>(); + let existing = self + .rows(store, Kind::CatalogChunks, &chunk_keys, work) + .await?; + let mut changed = Vec::new(); + let mut prior = Vec::new(); + let mut prior_xorbs = BTreeSet::new(); + for ((key, incoming), existing) in chunks.into_iter().zip(existing) { + if let Some(existing) = existing { + if existing.len() != 40 { + return Err(corrupt("catalog chunk placement length differs")); + } + // Superseded placements remain loaded dependencies. Precedence + // must not hide a broken recipe behind a valid new placement. + let keep_existing = existing[..36] <= incoming[..36]; + prior_xorbs.insert(hex(&existing[..32])?); + prior.push((key.clone(), existing)); + if keep_existing { + continue; + } + } + changed.push((key, incoming)); + } + let existing = self + .rows( + store, + Kind::CatalogXorbs, + &keys(prior_xorbs.iter().collect())?, + work, + ) + .await?; + let mut prior_entries = BTreeMap::new(); + for (identity, row) in prior_xorbs.into_iter().zip(existing) { + let row = row.ok_or_else(|| corrupt("prior placement has no xorb"))?; + prior_entries.insert(identity, self.xorb(store, &row, work).await?); + } + for (key, placement) in prior { + let identity = hex(&placement[..32])?; + let entry = prior_entries + .get(&identity) + .ok_or_else(|| corrupt("prior placement lost its xorb"))?; + let index = u32::from_be_bytes( + placement[32..36] + .try_into() + .map_err(|_| corrupt("placement ordinal is truncated"))?, + ) as usize; + let size = u32::from_be_bytes( + placement[36..] + .try_into() + .map_err(|_| corrupt("placement size is truncated"))?, + ); + let hash = hex(&key)?; + if entry + .chunks + .get(index) + .is_none_or(|chunk| chunk.hash != hash || chunk.uncompressed_size != size) + || entry.chunks.iter().position(|chunk| chunk.hash == hash) != Some(index) + { + return Err(corrupt( + "prior placement differs from its canonical xorb recipe", + )); + } + } + descriptor.files = self + .pages + .edit( + store, + descriptor.files.as_ref(), + Kind::CatalogFiles, + file_rows(&delta)?, + work, + ) + .await?; + descriptor.shards = self + .pages + .edit( + store, + descriptor.shards.as_ref(), + Kind::CatalogShards, + shards, + work, + ) + .await?; + descriptor.xorbs = self + .pages + .edit( + store, + descriptor.xorbs.as_ref(), + Kind::CatalogXorbs, + xorbs, + work, + ) + .await?; + descriptor.chunks = self + .pages + .edit( + store, + descriptor.chunks.as_ref(), + Kind::CatalogChunks, + changed, + work, + ) + .await?; + descriptor.validate()?; + Ok(descriptor) + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs new file mode 100644 index 000000000..9d0a8a367 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -0,0 +1,283 @@ +use super::*; +use crate::pages::PageWork; +use crab_storage::{StorageObservation, StorageObserver, Store}; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct Observations(Mutex>); +impl StorageObserver for Observations { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, observation: StorageObservation) { + self.0.lock().unwrap().push(observation); + } +} + +async fn fixture(catalog: &PointerCatalog) -> (CatalogPages, Store, Arc) { + let observations = Arc::new(Observations::default()); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(observations.clone()); + let prefix = object_store::path::Path::from("catalog"); + let (descriptor, objects) = Descriptor::build(catalog).unwrap(); + let root = descriptor.encode().unwrap(); + let identity = *blake3::hash(&root).as_bytes(); + let pages = CatalogPages::new(prefix, descriptor, identity, root.len() as u64).unwrap(); + for (hash, body) in objects { + store + .put_if_absent_verified(&pages.pages.path(&hash), body) + .await + .unwrap(); + } + observations.0.lock().unwrap().clear(); + (pages, store, observations) +} + +fn catalog(files: u32) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(1, Vec::new())) + .unwrap(); + for index in 0..files { + catalog + .insert_file( + format!("{index:064x}"), + FileCatalogEntry::new(1, "a".repeat(64)), + ) + .unwrap(); + } + catalog +} + +#[tokio::test] +async fn fixed_binding_update_reuses_unrelated_catalog_pages() { + for count in [100, 10_000, 100_000] { + let expected = catalog(count); + let (pages, store, observations) = fixture(&expected).await; + let mut delta = PointerCatalog::new(); + delta + .insert_file( + format!("{count:064x}"), + FileCatalogEntry::new(1, "a".repeat(64)), + ) + .unwrap(); + let descriptor = pages + .apply(&store, &[delta.clone()], &mut PageWork::new(256 * 1024, 0)) + .await + .unwrap(); + let io = observations.0.lock().unwrap().clone(); + assert!(io.iter().map(|op| op.bytes_read).sum::() <= 256 * 1024); + assert!(io.iter().map(|op| op.bytes_written).sum::() <= 256 * 1024); + assert_eq!( + pages + .full(&store, &mut PageWork::new(64 * 1024 * 1024, 0)) + .await + .unwrap(), + expected + ); + let root = descriptor.encode().unwrap(); + let identity = *blake3::hash(&root).as_bytes(); + store + .put_if_absent_verified(&pages.pages.path(&identity), root.clone()) + .await + .unwrap(); + let next = CatalogPages::new( + object_store::path::Path::from("catalog"), + descriptor, + identity, + root.len() as u64, + ) + .unwrap(); + let mut expected = expected; + expected.apply(&delta).unwrap(); + assert_eq!( + next.full(&store, &mut PageWork::new(64 * 1024 * 1024, 0)) + .await + .unwrap(), + expected + ); + } +} + +#[tokio::test] +async fn ordered_deltas_resolve_later_dependencies_and_preserve_immutable_contracts() { + let (pages, store, _) = fixture(&catalog(1)).await; + let mut file = PointerCatalog::new(); + file.insert_file("1".repeat(64), FileCatalogEntry::new(7, "b".repeat(64))) + .unwrap(); + let mut shard = PointerCatalog::new(); + shard + .insert_shard("b".repeat(64), ShardCatalogEntry::new(2, Vec::new())) + .unwrap(); + assert!( + pages + .apply(&store, &[file.clone()], &mut PageWork::new(1024 * 1024, 0)) + .await + .is_err() + ); + let next = pages + .apply(&store, &[file, shard], &mut PageWork::new(1024 * 1024, 0)) + .await + .unwrap(); + assert!(next.files.is_some() && next.shards.is_some()); + let mut conflict = PointerCatalog::new(); + conflict + .insert_shard("a".repeat(64), ShardCatalogEntry::new(99, Vec::new())) + .unwrap(); + assert!( + pages + .apply(&store, &[conflict], &mut PageWork::new(1024 * 1024, 0)) + .await + .is_err() + ); +} + +#[tokio::test] +async fn new_lower_xorb_updates_shared_chunk_precedence() { + let chunk = "c".repeat(64); + let mut base = PointerCatalog::new(); + base.insert_xorb( + "9".repeat(64), + XorbCatalogEntry::new( + 80, + "d".repeat(64), + vec![ + XorbChunkEntry::new(chunk.clone(), 7), + XorbChunkEntry::new(chunk.clone(), 7), + ], + ), + ) + .unwrap(); + let (pages, store, _) = fixture(&base).await; + let mut delta = PointerCatalog::new(); + delta + .insert_xorb( + "1".repeat(64), + XorbCatalogEntry::new( + 90, + "e".repeat(64), + vec![XorbChunkEntry::new(chunk.clone(), 7)], + ), + ) + .unwrap(); + let descriptor = pages + .apply(&store, &[delta.clone()], &mut PageWork::new(1024 * 1024, 0)) + .await + .unwrap(); + let root = descriptor.encode().unwrap(); + let identity = *blake3::hash(&root).as_bytes(); + store + .put_if_absent_verified(&pages.pages.path(&identity), root.clone()) + .await + .unwrap(); + let next = CatalogPages::new( + object_store::path::Path::from("catalog"), + descriptor, + identity, + root.len() as u64, + ) + .unwrap(); + base.apply(&delta).unwrap(); + assert_eq!( + next.full(&store, &mut PageWork::new(1024 * 1024, 0)) + .await + .unwrap(), + base + ); +} + +#[tokio::test] +async fn reused_nested_pages_cannot_amplify_materialization_past_budget() { + let mut catalog = PointerCatalog::new(); + for identity in 1..=16 { + catalog + .insert_xorb( + format!("{identity:064x}"), + XorbCatalogEntry::new( + 80, + "d".repeat(64), + (0..1000) + .map(|_| XorbChunkEntry::new("c".repeat(64), 7)) + .collect(), + ), + ) + .unwrap(); + } + let (pages, store, _) = fixture(&catalog).await; + let mut work = PageWork::new(256 * 1024, 0); + let result = pages.full(&store, &mut work).await; + assert!( + result.is_err() && work.bytes <= 256 * 1024, + "shared physical pages must not grant unbounded materialization" + ); +} + +#[tokio::test] +async fn delta_row_admission_precedes_origin_reads_and_recipe_clones() { + let (pages, store, observations) = fixture(&catalog(1)).await; + let mut delta = PointerCatalog::new(); + for index in 0..1000 { + delta + .insert_shard(format!("{index:064x}"), ShardCatalogEntry::new(1, vec![])) + .unwrap(); + } + let result = pages + .apply(&store, &[delta], &mut PageWork::new(4096, 0)) + .await; + assert!(result.is_err() && observations.0.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn superseded_reverse_placement_still_validates_prior_dependency() { + let chunk = "c".repeat(64); + let mut base = PointerCatalog::new(); + base.insert_xorb( + "9".repeat(64), + XorbCatalogEntry::new( + 80, + "d".repeat(64), + vec![XorbChunkEntry::new(chunk.clone(), 7)], + ), + ) + .unwrap(); + let (pages, store, _) = fixture(&base).await; + let mut descriptor = pages.descriptor.clone(); + let mut placement = hash(&"f".repeat(64)).unwrap().to_vec(); + placement.extend_from_slice(&0u32.to_be_bytes()); + placement.extend_from_slice(&7u32.to_be_bytes()); + descriptor.chunks = pages + .pages + .edit( + &store, + descriptor.chunks.as_ref(), + Kind::CatalogChunks, + vec![(hash(&chunk).unwrap().to_vec(), placement)], + &mut PageWork::new(1024 * 1024, 0), + ) + .await + .unwrap(); + let root = descriptor.encode().unwrap(); + let identity = *blake3::hash(&root).as_bytes(); + store + .put_if_absent_verified(&pages.pages.path(&identity), root.clone()) + .await + .unwrap(); + let forged = CatalogPages::new( + object_store::path::Path::from("catalog"), + descriptor, + identity, + root.len() as u64, + ) + .unwrap(); + let mut delta = PointerCatalog::new(); + delta + .insert_xorb( + "1".repeat(64), + XorbCatalogEntry::new(90, "e".repeat(64), vec![XorbChunkEntry::new(chunk, 7)]), + ) + .unwrap(); + assert!( + forged + .apply(&store, &[delta], &mut PageWork::new(1024 * 1024, 0)) + .await + .is_err() + ); +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index 66e273430..02533acdd 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -112,43 +112,6 @@ pub(in crate::capsule_protocol) fn semantic_identity(kind: Kind, bytes: &[u8]) - *hash.finalize().as_bytes() } -fn hash(value: &str) -> Result<[u8; 32]> { - let bytes = value.as_bytes(); - // Binary keys cannot distinguish hex aliases. Reject them instead of silently - // merging two descriptors that were distinct in the caller's map. - if bytes.len() != 64 - || bytes - .iter() - .any(|byte| !byte.is_ascii_digit() && !(b'a'..=b'f').contains(byte)) - { - return Err(contract_error( - "catalog hashes must be lowercase canonical hex", - )); - } - // Encode text order directly: Xet's raw hash bytes reverse each u64 word. - // Using those bytes as keys would change the original BTreeMap precedence. - let digit = |byte: u8| match byte { - b'0'..=b'9' => byte - b'0', - _ => byte - b'a' + 10, - }; - Ok(std::array::from_fn(|index| { - (digit(bytes[index * 2]) << 4) | digit(bytes[index * 2 + 1]) - })) -} - -fn hex(bytes: &[u8]) -> Result { - if bytes.len() != 32 { - return Err(corrupt("catalog hash has an invalid byte length")); - } - const DIGITS: &[u8; 16] = b"0123456789abcdef"; - let mut value = String::with_capacity(64); - for byte in bytes { - value.push(DIGITS[(byte >> 4) as usize] as char); - value.push(DIGITS[(byte & 15) as usize] as char); - } - Ok(value) -} - fn tree(writer: &mut Writer, kind: Kind, rows: Rows) -> Result> { if rows.is_empty() { return Ok(None); @@ -161,23 +124,7 @@ fn tree(writer: &mut Writer, kind: Kind, rows: Rows) -> Result> } pub(super) fn encode(catalog: &PointerCatalog) -> Result { - // Admit the unavoidable row payload before duplicating any caller-owned - // lists. Page headers and directories are checked again after encoding. - let minimum = catalog.xorbs.values().try_fold(0u64, |bytes, xorb| { - bytes.checked_add((xorb.chunks.len() as u64).checked_mul(44)?) - }); - let minimum = minimum.and_then(|minimum| { - catalog.shards.values().try_fold(minimum, |bytes, shard| { - bytes.checked_add((shard.xorb_hashes.len() as u64).checked_mul(36)?) - }) - }); - let minimum = - minimum.and_then(|bytes| bytes.checked_add((catalog.files.len() as u64).checked_mul(76)?)); - if minimum.is_none_or(|bytes| bytes > MAX_CATALOG_BYTES) { - return Err(contract_error( - "catalog rows exceed their encoded byte bound", - )); - } + admit_catalog_rows(std::slice::from_ref(catalog), MAX_CATALOG_BYTES)?; let mut writer = Writer::default(); let mut chunks = BTreeMap::new(); let mut xorbs = Vec::new(); diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index 3b38e2e0d..2f58da3f6 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -1,7 +1,8 @@ //! Selected dependency closure from captured checkpoint and run roots. use super::*; +use crate::capsule_protocol::RunCatalogSource; +use crate::capsule_protocol::pointer::object::{CatalogPages, shard_from_rows, xorb_from_rows}; use crate::capsule_protocol::run::catalog::{CatalogOrigin, CatalogValue}; -use crate::capsule_protocol::{PackRange, RunCatalogSource}; use crate::pages::{PageCache, PageWork}; use crab_storage::Store; use std::collections::BTreeSet; @@ -18,42 +19,22 @@ const QUERY_KEY_BYTES: u64 = 96; /// empty selection; missing pages and conflicting dependencies return errors. #[derive(Debug)] pub struct PointerCatalogLookup { - checkpoint: Option, + checkpoint: Option>, runs: Vec, ranks: BTreeMap, cache: PageCache, maximum: u64, } -#[derive(Debug)] -struct CheckpointSource { - path: object_store::path::Path, - range: PackRange, - descriptor: Descriptor, -} - impl PointerCatalogLookup { - pub(in crate::capsule_protocol) fn checkpoint( - path: object_store::path::Path, - range: PackRange, - descriptor: Descriptor, - maximum: u64, - ) -> Result { - descriptor.validate(range.length())?; - if range.offset().checked_add(range.length()).is_none() { - return Err(corrupt("catalog source range overflows")); - } - Ok(Self { - checkpoint: Some(CheckpointSource { - path, - range, - descriptor, - }), + pub(in crate::capsule_protocol) fn checkpoint(pages: Arc, maximum: u64) -> Self { + Self { + checkpoint: Some(pages), runs: Vec::new(), ranks: BTreeMap::new(), cache: PageCache::default(), maximum, - }) + } } /// Start a lookup for a captured view that has no checkpoint catalog. @@ -94,41 +75,20 @@ impl PointerCatalogLookup { } fn work(&self) -> PageWork { - PageWork { - maximum: self.maximum, - bytes: 0, - seen: BTreeSet::new(), - } + PageWork::new(self.maximum, 0) } async fn rows( &self, store: &Store, - root: Option<&PageRef>, kind: Kind, keys: &[Vec], work: &mut PageWork, ) -> Result>> { let mut output = (0..keys.len()).map(|_| Vec::new()).collect::>(); - if let (Some(source), Some(root)) = (&self.checkpoint, root) { - let origin = Arc::new(CatalogOrigin { - path: source.path.clone(), - range: source.range.offset()..source.range.offset() + source.descriptor.body_bytes, - layer: 0, - }); - let rows = self - .cache - .lookup( - store, - &origin.path, - origin.layer, - origin.range.clone(), - root, - kind, - keys, - work, - ) - .await?; + if let Some(source) = &self.checkpoint { + let rows = source.rows(store, kind, keys, work).await?; + let origin = Arc::new(CatalogOrigin::Object(source.clone())); for (result, row) in output.iter_mut().zip(rows) { if let Some(bytes) = row { result.push(CatalogValue { @@ -162,16 +122,13 @@ impl PointerCatalogLookup { let Some(root) = root else { return Ok(Vec::new()); }; + let CatalogOrigin::Pack { path, range, layer } = origin else { + return Err(corrupt( + "offset scan requested for an object-backed catalog", + )); + }; self.cache - .scan( - store, - &origin.path, - origin.layer, - origin.range.clone(), - root, - kind, - work, - ) + .scan(store, path, *layer, range.clone(), root, kind, work) .await } @@ -182,42 +139,26 @@ impl PointerCatalogLookup { work: &mut PageWork, ) -> Result { let keys = keys(hashes)?; - let root = self - .checkpoint - .as_ref() - .and_then(|s| s.descriptor.xorbs.as_ref()); - let rows = self - .rows(store, root, Kind::CatalogXorbs, &keys, work) - .await?; + let rows = self.rows(store, Kind::CatalogXorbs, &keys, work).await?; let mut catalog = PointerCatalog::new(); for (hash, row) in hashes.iter().zip(rows) { for value in row { - let row: XorbRow = from_json(&value.bytes)?; - let chunks = self - .scan( - store, - &value.origin, - Some(&row.chunks), - Kind::CatalogXorbChunks, - work, - ) - .await?; - let chunks = chunks - .into_iter() - .enumerate() - .map(|(position, (key, value))| { - if key != (position as u32).to_be_bytes() || value.len() != 36 { - return Err(corrupt("xorb chunk row has invalid position or size")); - } - let size = u32::from_be_bytes( - value[32..] - .try_into() - .map_err(|_| corrupt("chunk size is truncated"))?, - ); - Ok(XorbChunkEntry::new(hex(&value[..32])?, size)) - }) - .collect::>()?; - let entry = XorbCatalogEntry::new(row.encoded_size, hex(&row.body_digest)?, chunks); + let entry = match value.origin.as_ref() { + CatalogOrigin::Object(source) => source.xorb(store, &value.bytes, work).await?, + CatalogOrigin::Pack { .. } => { + let row: XorbRow = from_json(&value.bytes)?; + let chunks = self + .scan( + store, + &value.origin, + Some(&row.chunks), + Kind::CatalogXorbChunks, + work, + ) + .await?; + xorb_from_rows(row.encoded_size, row.body_digest, chunks)? + } + }; verify_identity(&value, Kind::CatalogXorbs, &json(&entry)?)?; catalog.insert_xorb(hash.clone(), entry)?; } @@ -284,12 +225,8 @@ impl PointerCatalogLookup { if hashes.is_empty() && shards.is_empty() && xorbs.is_empty() { return Ok(PointerCatalog::new()); } - let root = self - .checkpoint - .as_ref() - .and_then(|s| s.descriptor.files.as_ref()); let rows = self - .rows(store, root, Kind::CatalogFiles, &keys(&hashes)?, &mut work) + .rows(store, Kind::CatalogFiles, &keys(&hashes)?, &mut work) .await?; let mut catalog = PointerCatalog::new(); for (hash, row) in hashes.iter().zip(rows) { @@ -349,36 +286,30 @@ impl PointerCatalogLookup { work: &mut PageWork, ) -> Result { let mut catalog = PointerCatalog::new(); - let root = self - .checkpoint - .as_ref() - .and_then(|s| s.descriptor.shards.as_ref()); let rows = self - .rows(store, root, Kind::CatalogShards, &keys(shards)?, work) + .rows(store, Kind::CatalogShards, &keys(shards)?, work) .await?; let mut xorbs = BTreeSet::new(); for (hash, row) in shards.iter().zip(rows) { for value in row { - let row: ShardRow = from_json(&value.bytes)?; - let closure = self - .scan( - store, - &value.origin, - row.xorbs.as_ref(), - Kind::CatalogShardXorbs, - work, - ) - .await?; - let closure = closure - .into_iter() - .map(|(key, value)| { - if !value.is_empty() { - return Err(corrupt("shard xorb row has unexpected data")); - } - hex(&key) - }) - .collect::>()?; - let entry = ShardCatalogEntry::new(row.encoded_size, closure); + let entry = match value.origin.as_ref() { + CatalogOrigin::Object(source) => { + source.shard(store, &value.bytes, work).await? + } + CatalogOrigin::Pack { .. } => { + let row: ShardRow = from_json(&value.bytes)?; + let closure = self + .scan( + store, + &value.origin, + row.xorbs.as_ref(), + Kind::CatalogShardXorbs, + work, + ) + .await?; + shard_from_rows(row.encoded_size, closure)? + } + }; verify_identity(&value, Kind::CatalogShards, &json(&entry)?)?; catalog.insert_shard(hash.clone(), entry)?; } @@ -401,12 +332,8 @@ impl PointerCatalogLookup { pub async fn chunks(&self, store: &Store, hashes: &[String]) -> Result { let mut work = self.work(); let hashes = requested(hashes, &mut work)?; - let root = self - .checkpoint - .as_ref() - .and_then(|s| s.descriptor.chunks.as_ref()); let rows = self - .rows(store, root, Kind::CatalogChunks, &keys(&hashes)?, &mut work) + .rows(store, Kind::CatalogChunks, &keys(&hashes)?, &mut work) .await?; let mut selected = BTreeMap::new(); for (hash, row) in hashes.iter().zip(rows) { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs index 2e386c733..485a442ec 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader/tests.rs @@ -57,7 +57,6 @@ async fn fixture( .unwrap() .source_descriptor() .unwrap(); - let catalog_bytes = catalog.encode().unwrap(); let checkpoint = LayeredCheckpoint::build(1, &"3".repeat(64), vec![source], catalog, None).unwrap(); let reads = Arc::new(AtomicUsize::new(0)); @@ -67,13 +66,7 @@ async fn fixture( observed.fetch_add(1, Ordering::SeqCst); })); let layout = StoreLayout::new(store, "catalog-lookup".to_owned()); - if let Some(source) = checkpoint.catalog() { - layout - .store() - .put_overwrite(&layout.capsule_catalog_path(source.hash()), catalog_bytes) - .await - .unwrap(); - } + checkpoint.publish_catalog(&layout).await.unwrap(); layout .store() .put_overwrite( @@ -93,6 +86,7 @@ async fn fixture( ) .unwrap(); let lookup = control.pointer_catalog_lookup(&layout, maximum).unwrap(); + reads.store(0, Ordering::SeqCst); (layout, checkpoint, lookup, reads) } @@ -301,7 +295,14 @@ async fn chunk_selection_uses_lowest_xorb_then_first_occurrence_across_sources() async fn missing_and_corrupted_pages_are_errors_instead_of_catalog_misses() { for missing in [false, true] { let (layout, checkpoint, lookup, _) = fixture(catalog(2), 1024 * 1024).await; - let path = layout.capsule_catalog_path(checkpoint.catalog().unwrap().hash()); + let source = lookup.checkpoint.as_ref().unwrap(); + let path = if missing { + layout.capsule_catalog_path(checkpoint.catalog().unwrap().hash()) + } else { + source + .pages + .path(&source.descriptor.files.as_ref().unwrap().hash) + }; if missing { layout.store().delete(&path).await.unwrap(); } else { @@ -312,9 +313,7 @@ async fn missing_and_corrupted_pages_are_errors_instead_of_catalog_misses() { .unwrap() .0 .to_vec(); - let source = lookup.checkpoint.as_ref().unwrap(); - let offset = source.range.offset() + source.descriptor.files.as_ref().unwrap().offset; - bytes[offset as usize] ^= 1; + bytes[0] ^= 1; layout .store() .put_overwrite(&path, bytes.into()) @@ -345,29 +344,46 @@ async fn loaded_dependencies_and_chunk_placements_fail_closed() { .is_err() ); // A re-signed reverse index must still agree with its authenticated xorb. - let bytes = catalog(1).encode().unwrap(); - let mut descriptor = Descriptor::decode(&bytes).unwrap(); - let root = descriptor.chunks.as_mut().unwrap(); - let mut body = bytes[..descriptor.body_bytes as usize].to_vec(); - let size = root.offset as usize + 14 + 32 + 4 + 36; - body[size] ^= 1; - root.hash = - *blake3::hash(&body[root.offset as usize..root.offset as usize + root.bytes as usize]) - .as_bytes(); - descriptor.body_hash = *blake3::hash(&body).as_bytes(); - let header = json(&descriptor).unwrap(); - body.extend_from_slice(&header); - crate::pages::put_u32(&mut body, header.len()).unwrap(); - body.extend_from_slice(MAGIC); - let descriptor = Descriptor::decode(&body).unwrap(); - let range = PackRange::new(0, &body).unwrap(); - let path = object_store::path::Path::from("resigned-catalog"); + let source = lookup.checkpoint.as_ref().unwrap(); + let mut descriptor = source.descriptor.clone(); + let mut value = blake3::Hash::from_hex(format!("{:064x}", 1_000_000)) + .unwrap() + .as_bytes() + .to_vec(); + value.extend_from_slice(&1u32.to_be_bytes()); + value.extend_from_slice(&7u32.to_be_bytes()); + descriptor.chunks = source + .pages + .edit( + layout.store(), + descriptor.chunks.as_ref(), + Kind::CatalogChunks, + vec![( + blake3::Hash::from_hex(format!("{:064x}", 2_000_000)) + .unwrap() + .as_bytes() + .to_vec(), + value, + )], + &mut PageWork::new(1024 * 1024, 0), + ) + .await + .unwrap(); + let body = descriptor.encode().unwrap(); + let identity = *blake3::hash(&body).as_bytes(); layout .store() - .put_overwrite(&path, body.into()) + .put_overwrite(&source.pages.path(&identity), body.clone()) .await .unwrap(); - let lookup = PointerCatalogLookup::checkpoint(path, range, descriptor, 1024 * 1024).unwrap(); + let pages = CatalogPages::new( + layout.capsule_catalog_prefix(), + descriptor, + identity, + body.len() as u64, + ) + .unwrap(); + let lookup = PointerCatalogLookup::checkpoint(Arc::new(pages), 1024 * 1024); assert!( lookup .chunks(layout.store(), &[format!("{:064x}", 2_000_000)]) diff --git a/crates/crab-metadata/src/capsule_protocol/root.rs b/crates/crab-metadata/src/capsule_protocol/root.rs index 43c59ae32..722047190 100644 --- a/crates/crab-metadata/src/capsule_protocol/root.rs +++ b/crates/crab-metadata/src/capsule_protocol/root.rs @@ -1330,7 +1330,7 @@ mod tests { fn root_rejects_retired_or_implicit_checkpoint_formats() { let record = checkpointed_record(); let payload = serde_json::to_string(record.root()).unwrap(); - assert_eq!(payload.matches("\"format\":8,").count(), 1); + assert_eq!(payload.matches("\"format\":9,").count(), 1); for replacement in [ "", "\"format\":3,", @@ -1338,8 +1338,9 @@ mod tests { "\"format\":5,", "\"format\":6,", "\"format\":7,", + "\"format\":8,", ] { - let payload = payload.replace("\"format\":8,", replacement); + let payload = payload.replace("\"format\":9,", replacement); // Recompute the envelope digest so only format admission can fail. let mut bytes = ROOT_MAGIC.to_vec(); bytes.extend_from_slice(&ROOT_VERSION.to_be_bytes()); diff --git a/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs b/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs index 52c3d5430..f4def878d 100644 --- a/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/run/admission/reader.rs @@ -151,10 +151,6 @@ impl CapsuleRunAdmissionReader { } fn work(&self) -> PageWork { - PageWork { - maximum: self.maximum, - bytes: 0, - seen: BTreeSet::new(), - } + PageWork::new(self.maximum, 0) } } diff --git a/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs b/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs index e5c65fb7d..6613ef828 100644 --- a/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/run/catalog/reader.rs @@ -5,10 +5,13 @@ use crab_storage::{Store, StoreLayout}; use std::sync::Arc; #[derive(Debug, Clone)] -pub(in crate::capsule_protocol) struct CatalogOrigin { - pub path: object_store::path::Path, - pub range: std::ops::Range, - pub layer: usize, +pub(in crate::capsule_protocol) enum CatalogOrigin { + Pack { + path: object_store::path::Path, + range: std::ops::Range, + layer: usize, + }, + Object(Arc), } pub(in crate::capsule_protocol) struct CatalogValue { @@ -133,7 +136,7 @@ impl RunCatalogSource { // Membership admits logical values, not all capsules in the // physical run. Equal variants may reuse an unselected source. result.push(CatalogValue { - origin: Arc::new(CatalogOrigin { + origin: Arc::new(CatalogOrigin::Pack { path: self.path.clone(), range, layer, diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs index f15a10bb5..8838c5692 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged/reader.rs @@ -2,7 +2,6 @@ use super::*; use crate::pages::{PageCache, PageWork}; use crab_storage::{Store, StoreLayout}; -use std::collections::BTreeSet; /// Change to an object's membership in one ref closure; never standalone authorization. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -61,11 +60,7 @@ impl CapsuleVisibilityReader { store: &Store, names: &[&str], ) -> Result>> { - let mut work = PageWork { - maximum: self.maximum, - bytes: 0, - seen: BTreeSet::new(), - }; + let mut work = PageWork::new(self.maximum, 0); self.read_headers(store, names, &mut work).await } @@ -115,11 +110,7 @@ impl CapsuleVisibilityReader { name: &str, direction: VisibilityChange, ) -> Result> { - let mut work = PageWork { - maximum: self.maximum, - bytes: 0, - seen: BTreeSet::new(), - }; + let mut work = PageWork::new(self.maximum, 0); let row = self .read_headers(store, &[name], &mut work) .await? @@ -199,11 +190,7 @@ impl CapsuleVisibilityReader { if !ref_name.starts_with("refs/") || !valid_ref_name(ref_name) { return Err(contract_error("visibility lookup ref name is invalid")); } - let mut work = PageWork { - maximum: self.maximum, - bytes: 0, - seen: BTreeSet::new(), - }; + let mut work = PageWork::new(self.maximum, 0); let key = blake3::hash(ref_name.as_bytes()).as_bytes().to_vec(); let mut rows = self .rows( diff --git a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs index 9634c7d6b..69ddc354e 100644 --- a/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/catalog_tests.rs @@ -73,7 +73,6 @@ pub(super) async fn capture( .unwrap(), ) .unwrap(); - let catalog_bytes = catalog.encode().unwrap(); let checkpoint = LayeredCheckpoint::build( base.root().generation(), base.digest(), @@ -82,15 +81,7 @@ pub(super) async fn capture( None, ) .unwrap(); - if let Some(source) = checkpoint.catalog() { - // Seed immutable fixture data without charging a publication readback - // to the reader-construction and selected-query measurements below. - layout - .store() - .put_overwrite(&layout.capsule_catalog_path(source.hash()), catalog_bytes) - .await - .unwrap(); - } + checkpoint.publish_catalog(layout).await.unwrap(); let pointer = CheckpointPointer::new_layered( checkpoint.hash(), checkpoint.bytes().len() as u64, @@ -200,6 +191,9 @@ async fn pinned_batches_preserve_order_misses_and_page_bounds_as_history_grows() })); let layout = crab_storage::StoreLayout::new(storage, "pinned-lookup".to_owned()); let (captured, _, _) = capture(&layout, catalog(count), 512 * 1024).await; + // Fixture publication verifies its writes; only capture construction and + // selected reads belong to this measurement window. + reads.store(0, Ordering::SeqCst); let lookup = SharedFileIndexLookup::for_pointer_catalog(layout.clone(), captured); assert_eq!( reads.load(Ordering::SeqCst), @@ -208,6 +202,7 @@ async fn pinned_batches_preserve_order_misses_and_page_bounds_as_history_grows() ); // A later authenticated catalog must not change an existing capture. capture(&layout, catalog(count + 1), 512 * 1024).await; + reads.store(0, Ordering::SeqCst); let file = super::tests::hash_from_seed(42); let later = super::tests::hash_from_seed(count as u64); let shard = super::tests::hash_from_seed(1_000_042); @@ -283,6 +278,8 @@ async fn current_state_opener_keeps_catalog_pages_selected_and_pins_its_first_ro let layout = crab_storage::StoreLayout::new(storage.clone(), "current-catalog".to_owned()); let (_, _, root) = capture(&layout, catalog(count), 512 * 1024).await; assert_eq!(root.root().generation(), 1); + reads.store(0, Ordering::SeqCst); + bytes.store(0, Ordering::SeqCst); let lookup = SharedFileIndexLookup::new_for_storage(&storage, layout.repo_prefix()); assert_eq!(reads.load(Ordering::SeqCst), 0); let file = super::tests::hash_from_seed(42); diff --git a/crates/crab-metadata/src/file_index_lookup/shared_tests.rs b/crates/crab-metadata/src/file_index_lookup/shared_tests.rs index 2f7c5a1b2..37ef63624 100644 --- a/crates/crab-metadata/src/file_index_lookup/shared_tests.rs +++ b/crates/crab-metadata/src/file_index_lookup/shared_tests.rs @@ -184,7 +184,7 @@ impl ObjectStore for PausedLookupStore { location: &ObjectPath, options: GetOptions, ) -> object_store::Result { - if location == &self.shard && !self.paused.swap(true, Ordering::AcqRel) { + if location.prefix_matches(&self.shard) && !self.paused.swap(true, Ordering::AcqRel) { self.entered.notify_one(); self.release.acquire().await.unwrap().forget(); } @@ -352,7 +352,7 @@ async fn captured_catalog_close_waits_for_active_reads_and_abandonment_releases_ let inner: Arc = Arc::new(InMemory::new()); let storage = crab_storage::Store::new(inner.clone()); let router = crab_storage::StoreLayout::new(storage, "pinned/close".to_owned()); - let (catalog, path, _) = super::catalog_tests::capture( + let (catalog, _, _) = super::catalog_tests::capture( &router, super::catalog_tests::catalog(10_000), 512 * 1024, @@ -360,7 +360,8 @@ async fn captured_catalog_close_waits_for_active_reads_and_abandonment_releases_ .await; let store = Arc::new(PausedLookupStore { inner, - shard: path, + // Pause a cold catalog object after the descriptor is warm. + shard: router.capsule_catalog_prefix(), paused: AtomicBool::new(true), pause_write: AtomicBool::new(false), entered: Notify::new(), diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index 2326d40e4..af51a3ae9 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -374,11 +374,7 @@ impl PagedGitObjectIndex { } fn work(&self) -> PageWork { - PageWork { - maximum: self.maximum, - bytes: self.descriptor_bytes, - seen: BTreeSet::new(), - } + PageWork::new(self.maximum, self.descriptor_bytes) } async fn lookup_rows( diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 63a1f40cb..1a0466e3f 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -1,6 +1,8 @@ //! Authenticated bounded pages shared by immutable metadata indexes. use serde::{Deserialize, Serialize}; +pub(crate) mod object; + #[cfg(feature = "storage")] mod cache; #[cfg(feature = "storage")] diff --git a/crates/crab-metadata/src/pages/cache.rs b/crates/crab-metadata/src/pages/cache.rs index c26a98df1..e1c3addfe 100644 --- a/crates/crab-metadata/src/pages/cache.rs +++ b/crates/crab-metadata/src/pages/cache.rs @@ -56,10 +56,57 @@ impl State { pub(crate) struct PageWork { pub(crate) maximum: u64, pub(crate) bytes: u64, - pub(crate) seen: BTreeSet<(usize, u64)>, + seen: BTreeSet<(usize, u64)>, + objects: BTreeMap<[u8; 32], u64>, + materialized: u64, } impl PageWork { + pub(crate) fn new(maximum: u64, bytes: u64) -> Self { + Self { + maximum, + bytes, + seen: BTreeSet::new(), + objects: BTreeMap::new(), + materialized: 0, + } + } + + pub(crate) fn admit_materialized(&mut self, bytes: u64) -> Result<()> { + let total = self + .materialized + .checked_add(bytes) + // Encoded admission also pays for vector/string headers, hex keys + // and simultaneous row/typed values. Bound that expansion separately + // so aliases cannot grant unlimited copies of one physical page. + .filter(|total| *total <= self.maximum.saturating_mul(4)) + .ok_or_else(|| corruption("metadata materialization exceeds its byte limit"))?; + self.materialized = total; + Ok(()) + } + + pub(crate) fn admit_object(&mut self, hash: [u8; 32], bytes: u64) -> Result<()> { + if let Some(prior) = self.objects.get(&hash) { + return if *prior == bytes { + Ok(()) + } else { + Err(corruption("object identity has conflicting byte lengths")) + }; + } + let total = self + .bytes + .checked_add(bytes) + .filter(|total| *total <= self.maximum) + .ok_or_else(|| corruption("metadata query exceeds its byte limit"))?; + self.bytes = total; + self.objects.insert(hash, bytes); + Ok(()) + } + + pub(crate) fn object_dependencies(&self) -> impl Iterator { + self.objects.iter().map(|(hash, bytes)| (hash, *bytes)) + } + fn admit(&mut self, layer: usize, offset: u64, bytes: u64) -> Result<()> { if self.seen.insert((layer, offset)) { self.bytes = self @@ -392,11 +439,7 @@ mod tests { body.extend_from_slice(&[0; 128]); store.create_strict(&path, body.into()).await.unwrap(); let cache = PageCache::for_region(region.clone()); - let mut work = PageWork { - maximum: u64::MAX, - bytes: 0, - seen: BTreeSet::new(), - }; + let mut work = PageWork::new(u64::MAX, 0); cache .lookup( &store, @@ -411,11 +454,7 @@ mod tests { .await .unwrap(); let cold = bytes.swap(0, Ordering::SeqCst); - let mut limited = PageWork { - maximum: cold - 1, - bytes: 0, - seen: BTreeSet::new(), - }; + let mut limited = PageWork::new(cold - 1, 0); let rejected = cache .lookup( &store, diff --git a/crates/crab-metadata/src/pages/object.rs b/crates/crab-metadata/src/pages/object.rs new file mode 100644 index 000000000..545be4a8d --- /dev/null +++ b/crates/crab-metadata/src/pages/object.rs @@ -0,0 +1,283 @@ +//! Content-addressed pages whose unchanged subtrees survive a new publication. +use super::*; +use bytes::Bytes; +use std::collections::BTreeMap; + +#[cfg(feature = "storage")] +mod store; +#[cfg(feature = "storage")] +pub(crate) use store::ObjectPages; + +const MAGIC: &[u8; 8] = b"CRABOP01"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Reference { + pub(crate) hash: [u8; 32], + pub(crate) bytes: u32, + pub(crate) height: u8, + pub(crate) count: u32, + pub(crate) first: Vec, + pub(crate) last: Vec, +} + +impl Reference { + pub(crate) fn validate(&self, kind: Kind) -> Result<()> { + if self.first.len() != kind.key_bytes() + || self.last.len() != kind.key_bytes() + || self.first > self.last + || self.count == 0 + || self.height > MAX_HEIGHT + || !(HEADER..=MAX_BYTES).contains(&(self.bytes as usize)) + { + return Err(corruption("invalid object page reference")); + } + if kind.is_positional() + && integer(&self.last)? + .checked_sub(integer(&self.first)?) + .and_then(|span| span.checked_add(1)) + != Some(self.count) + { + return Err(corruption("positional object page has incomplete coverage")); + } + Ok(()) + } +} + +#[derive(Debug)] +pub(crate) enum Node { + Leaf(Rows), + Branch(Vec), +} + +pub(crate) struct Encoded { + pub(crate) reference: Reference, + pub(crate) bytes: Bytes, +} + +#[derive(Default)] +pub(crate) struct Builder { + pub(crate) objects: BTreeMap<[u8; 32], Bytes>, +} + +impl Builder { + pub(crate) fn tree(&mut self, kind: Kind, rows: Rows) -> Result> { + if rows.is_empty() { + return Ok(None); + } + let mut level = self.retain(leaves(kind, rows)?); + while level.len() > 1 { + level = self.retain(branches(kind, level)?); + } + Ok(level.pop()) + } + + fn retain(&mut self, encoded: Vec) -> Vec { + encoded + .into_iter() + .map(|page| { + self.objects.insert(page.reference.hash, page.bytes); + page.reference + }) + .collect() + } +} + +fn encode(kind: Kind, node: Node) -> Result { + let (height, count, first, last, entries) = match &node { + Node::Leaf(rows) => { + validate_rows(kind, rows)?; + ( + 0, + u32::try_from(rows.len()).map_err(|_| corruption("leaf count overflows"))?, + rows[0].0.clone(), + rows[rows.len() - 1].0.clone(), + rows.len(), + ) + } + Node::Branch(children) => { + validate_children(kind, children)?; + let first = &children[0]; + ( + first.height + 1, + children.iter().try_fold(0u32, |total, child| { + total + .checked_add(child.count) + .ok_or_else(|| corruption("object directory count overflows")) + })?, + first.first.clone(), + children[children.len() - 1].last.clone(), + children.len(), + ) + } + }; + let mut bytes = MAGIC.to_vec(); + bytes.extend_from_slice(&[kind as u8, height]); + put_u32(&mut bytes, entries)?; + match node { + Node::Leaf(rows) => { + for (key, value) in rows { + bytes.extend_from_slice(&key); + put_u32(&mut bytes, value.len())?; + bytes.extend_from_slice(&value); + } + } + Node::Branch(children) => { + for child in children { + bytes.extend_from_slice(&child.hash); + bytes.extend_from_slice(&child.bytes.to_le_bytes()); + bytes.push(child.height); + bytes.extend_from_slice(&child.count.to_le_bytes()); + bytes.extend_from_slice(&child.first); + bytes.extend_from_slice(&child.last); + } + } + } + if bytes.len() > MAX_BYTES { + return Err(corruption("object page exceeds its byte limit")); + } + let reference = Reference { + hash: *blake3::hash(&bytes).as_bytes(), + bytes: bytes.len() as u32, + height, + count, + first, + last, + }; + reference.validate(kind)?; + Ok(Encoded { + reference, + bytes: bytes.into(), + }) +} + +fn validate_children(kind: Kind, children: &[Reference]) -> Result<()> { + let first = children + .first() + .ok_or_else(|| corruption("empty object directory"))?; + if first.height >= MAX_HEIGHT { + return Err(corruption("object directory exceeds its height limit")); + } + for child in children { + child.validate(kind)?; + if child.height != first.height { + return Err(corruption("object directory has mixed child heights")); + } + } + for pair in children.windows(2) { + if pair[0].last >= pair[1].first + || (kind.is_positional() + && integer(&pair[0].last)?.checked_add(1) != Some(integer(&pair[1].first)?)) + { + return Err(corruption( + "object directory has overlapping or incomplete routing", + )); + } + } + Ok(()) +} + +fn leaves(kind: Kind, rows: Rows) -> Result> { + validate_rows(kind, &rows)?; + let mut output = Vec::new(); + let mut pending = Vec::new(); + let mut size = HEADER; + for row in rows { + let row_bytes = kind + .key_bytes() + .checked_add(4) + .and_then(|bytes| bytes.checked_add(row.1.len())) + .filter(|bytes| *bytes <= MAX_BYTES - HEADER) + .ok_or_else(|| corruption("object row exceeds its byte limit"))?; + if !pending.is_empty() && size + row_bytes > TARGET_BYTES { + output.push(encode(kind, Node::Leaf(std::mem::take(&mut pending)))?); + size = HEADER; + } + size += row_bytes; + pending.push(row); + } + if !pending.is_empty() { + output.push(encode(kind, Node::Leaf(pending))?); + } + Ok(output) +} + +fn branches(kind: Kind, children: Vec) -> Result> { + validate_children(kind, &children)?; + let fanout = (TARGET_BYTES - HEADER) / (41 + 2 * kind.key_bytes()); + children + .chunks(fanout) + .map(|children| encode(kind, Node::Branch(children.to_vec()))) + .collect() +} + +#[cfg(any(test, feature = "storage"))] +fn decode(bytes: &[u8], reference: &Reference, kind: Kind) -> Result { + reference.validate(kind)?; + if bytes.len() != reference.bytes as usize + || blake3::hash(bytes).as_bytes() != &reference.hash + || bytes.get(..8) != Some(MAGIC.as_slice()) + || bytes.get(8) != Some(&(kind as u8)) + || bytes.get(9) != Some(&reference.height) + { + return Err(corruption("object page identity or header differs")); + } + let mut input = Input::new(&bytes[10..], "object metadata page"); + let count = input.u32()? as usize; + if count == 0 { + return Err(corruption("empty object page")); + } + let node = if reference.height == 0 { + if count > input.remaining() / (kind.key_bytes() + 4) { + return Err(corruption("object leaf count exceeds encoded bytes")); + } + let mut rows = Vec::with_capacity(count); + for _ in 0..count { + let key = input.take(kind.key_bytes())?.to_vec(); + let size = input.u32()? as usize; + rows.push((key, input.take(size)?.to_vec())); + } + validate_rows(kind, &rows)?; + if count != reference.count as usize + || rows[0].0 != reference.first + || rows[rows.len() - 1].0 != reference.last + { + return Err(corruption("object leaf coverage differs")); + } + Node::Leaf(rows) + } else { + if count > input.remaining() / (41 + 2 * kind.key_bytes()) { + return Err(corruption("object directory count exceeds encoded bytes")); + } + let mut children = Vec::with_capacity(count); + for _ in 0..count { + children.push(Reference { + hash: input.array()?, + bytes: input.u32()?, + height: input.array::<1>()?[0], + count: input.u32()?, + first: input.take(kind.key_bytes())?.to_vec(), + last: input.take(kind.key_bytes())?.to_vec(), + }); + } + validate_children(kind, &children)?; + let total = children.iter().try_fold(0u32, |total, child| { + total + .checked_add(child.count) + .ok_or_else(|| corruption("object directory count overflows")) + })?; + if children[0].height + 1 != reference.height + || total != reference.count + || children[0].first != reference.first + || children[children.len() - 1].last != reference.last + { + return Err(corruption("object directory coverage differs")); + } + Node::Branch(children) + }; + input.finish()?; + Ok(node) +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/pages/object/store.rs b/crates/crab-metadata/src/pages/object/store.rs new file mode 100644 index 000000000..0b7891f7c --- /dev/null +++ b/crates/crab-metadata/src/pages/object/store.rs @@ -0,0 +1,411 @@ +use super::*; +use crate::pages::PageWork; +use crab_storage::Store; +use futures_util::future::BoxFuture; +use futures_util::stream::{BoxStream, StreamExt as _, TryStreamExt as _}; +use object_store::path::Path; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +const CACHE_BYTES: usize = 8 * 1024 * 1024; +const CACHE_PAGES: usize = 256; + +#[derive(Default)] +struct Cache { + entries: BTreeMap<[u8; 32], (Kind, Reference, Arc)>, + order: VecDeque<[u8; 32]>, + bytes: usize, +} + +pub(crate) struct ObjectPages { + prefix: Path, + cache: Mutex, + // One shared set of lanes caps tree-wide reads and verified writes; warm + // cache hits avoid these locks and do not wait behind cold page requests. + io: [futures_util::lock::Mutex<()>; 4], +} + +impl std::fmt::Debug for ObjectPages { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ObjectPages") + .finish_non_exhaustive() + } +} + +impl ObjectPages { + pub(crate) fn new(prefix: Path) -> Self { + Self { + prefix, + cache: Mutex::default(), + io: std::array::from_fn(|_| futures_util::lock::Mutex::new(())), + } + } + + pub(crate) fn path(&self, hash: &[u8; 32]) -> Path { + let hex = blake3::Hash::from_bytes(*hash).to_hex().to_string(); + self.prefix.clone().join(&hex[..2]).join(hex) + } + + fn cached(&self, reference: &Reference, kind: Kind) -> Result>> { + let cache = self + .cache + .lock() + .map_err(|_| corruption("object page cache lock poisoned"))?; + let Some((prior_kind, prior, node)) = cache.entries.get(&reference.hash) else { + return Ok(None); + }; + if *prior_kind != kind || prior != reference { + return Err(corruption( + "object page identity has conflicting routing metadata", + )); + } + Ok(Some(node.clone())) + } + + async fn fetch(&self, store: &Store, reference: &Reference, kind: Kind) -> Result> { + reference.validate(kind)?; + if let Some(node) = self.cached(reference, kind)? { + return Ok(node); + } + let _read = self.io[usize::from(reference.hash[0]) % self.io.len()] + .lock() + .await; + if let Some(node) = self.cached(reference, kind)? { + return Ok(node); + } + let bytes = store + .get_with_etag_bounded(&self.path(&reference.hash), u64::from(reference.bytes)) + .await? + .0; + let node = Arc::new(decode(&bytes, reference, kind)?); + self.retain(reference, kind, node.clone())?; + Ok(node) + } + + fn retain(&self, reference: &Reference, kind: Kind, node: Arc) -> Result<()> { + let weight = resident_bytes(reference, &node); + if weight > CACHE_BYTES { + return Ok(()); + } + let mut cache = self + .cache + .lock() + .map_err(|_| corruption("object page cache lock poisoned"))?; + if cache.entries.contains_key(&reference.hash) { + return Ok(()); + } + while cache.bytes.saturating_add(weight) > CACHE_BYTES || cache.entries.len() >= CACHE_PAGES + { + let Some(hash) = cache.order.pop_front() else { + break; + }; + if let Some((_, reference, node)) = cache.entries.remove(&hash) { + cache.bytes = cache + .bytes + .saturating_sub(resident_bytes(&reference, &node)); + } + } + cache.bytes += weight; + cache.order.push_back(reference.hash); + cache + .entries + .insert(reference.hash, (kind, reference.clone(), node)); + Ok(()) + } + + pub(crate) async fn node( + &self, + store: &Store, + reference: &Reference, + kind: Kind, + work: &mut PageWork, + ) -> Result> { + work.admit_object(reference.hash, u64::from(reference.bytes))?; + self.fetch(store, reference, kind).await + } + + pub(crate) async fn lookup( + &self, + store: &Store, + root: Option<&Reference>, + kind: Kind, + keys: &[Vec], + work: &mut PageWork, + ) -> Result>>> { + if keys.iter().any(|key| key.len() != kind.key_bytes()) + || (keys.len() as u64) + .checked_mul((kind.key_bytes() + std::mem::size_of::>>()) as u64) + .is_none_or(|bytes| bytes > work.maximum) + { + return Err(corruption( + "object query keys exceed their shape or byte limit", + )); + } + work.admit_materialized((keys.len() as u64).saturating_mul( + (kind.key_bytes() + + std::mem::size_of::>>() + + 3 * std::mem::size_of::()) as u64, + ))?; + let mut output = vec![None; keys.len()]; + let Some(root) = root else { + return Ok(output); + }; + root.validate(kind)?; + let positions = keys + .iter() + .enumerate() + .filter_map(|(index, key)| (root.first <= *key && *key <= root.last).then_some(index)) + .collect::>(); + let mut pending = vec![(root.clone(), positions)]; + while !pending.is_empty() { + pending.retain(|(_, positions)| !positions.is_empty()); + for (reference, _) in &pending { + work.admit_object(reference.hash, u64::from(reference.bytes))?; + } + let nodes = futures_util::stream::iter(std::mem::take(&mut pending)) + .map(|(reference, positions)| async move { + Ok::<_, MetadataError>((self.fetch(store, &reference, kind).await?, positions)) + }) + .buffer_unordered(4) + .try_collect::>() + .await?; + for (node, positions) in nodes { + match node.as_ref() { + Node::Leaf(rows) => { + for index in positions { + if let Ok(position) = + rows.binary_search_by(|row| row.0.cmp(&keys[index])) + { + // Repeated keys clone values even when the physical page is shared. + work.admit_materialized(2 * (rows[position].1.len() as u64 + 48))?; + output[index] = Some(rows[position].1.clone()); + } + } + } + Node::Branch(children) => { + let mut grouped = BTreeMap::>::new(); + for index in positions { + let child = children.partition_point(|child| child.last < keys[index]); + if let Some(reference) = children.get(child) + && reference.first <= keys[index] + { + grouped.entry(child).or_default().push(index); + } + } + pending.extend( + grouped + .into_iter() + .map(|(index, positions)| (children[index].clone(), positions)), + ); + } + } + } + } + Ok(output) + } + + pub(crate) fn walk<'a>( + &'a self, + store: &'a Store, + root: Option<&Reference>, + kind: Kind, + work: &'a mut PageWork, + ) -> BoxStream<'a, Result<(Reference, Arc)>> { + let pending = root.cloned().into_iter().collect::>(); + futures_util::stream::try_unfold((pending, work), move |(mut pending, work)| async move { + let Some(reference) = pending.pop() else { + return Ok(None); + }; + let node = self.node(store, &reference, kind, work).await?; + match node.as_ref() { + // Count each logical visit, including repeated nested roots. The + // second copy admits row clones and their typed catalog values. + Node::Leaf(_) => work.admit_materialized( + (resident_bytes(&reference, &node) as u64).saturating_mul(2), + )?, + Node::Branch(children) => pending.extend(children.iter().rev().cloned()), + } + Ok(Some(((reference, node), (pending, work)))) + }) + .boxed() + } + + pub(crate) async fn scan( + &self, + store: &Store, + root: Option<&Reference>, + kind: Kind, + work: &mut PageWork, + ) -> Result { + self.walk(store, root, kind, work) + .try_fold(Vec::new(), |mut rows, (_, node)| async move { + if let Node::Leaf(leaf) = node.as_ref() { + rows.extend(leaf.iter().cloned()); + } + Ok(rows) + }) + .await + } + + pub(crate) async fn edit( + &self, + store: &Store, + root: Option<&Reference>, + kind: Kind, + rows: Rows, + work: &mut PageWork, + ) -> Result> { + if rows.is_empty() { + return Ok(root.cloned()); + } + validate_rows(kind, &rows)?; + let incoming_bytes = rows.iter().try_fold(0u64, |bytes, (key, value)| { + bytes + .checked_add(key.len() as u64)? + .checked_add(4)? + .checked_add(value.len() as u64) + }); + if incoming_bytes.is_none_or(|bytes| bytes > work.maximum.saturating_sub(work.bytes)) { + return Err(corruption("object edit input exceeds its byte limit")); + } + let work = futures_util::lock::Mutex::new(work); + let mut level = match root { + Some(root) => { + self.edit_node(store, root.clone(), kind, rows, &work) + .await? + } + None => { + self.persist(store, kind, leaves(kind, rows)?, &work) + .await? + } + }; + while level.len() > 1 { + level = self + .persist(store, kind, branches(kind, level)?, &work) + .await?; + } + Ok(level.pop()) + } + + fn edit_node<'a>( + &'a self, + store: &'a Store, + reference: Reference, + kind: Kind, + rows: Rows, + work: &'a futures_util::lock::Mutex<&mut PageWork>, + ) -> BoxFuture<'a, Result>> { + Box::pin(async move { + work.lock() + .await + .admit_object(reference.hash, u64::from(reference.bytes))?; + let node = self.fetch(store, &reference, kind).await?; + match node.as_ref() { + Node::Leaf(prior) => { + let mut merged = prior.iter().cloned().collect::>(); + merged.extend(rows); + let merged = merged.into_iter().collect::(); + if merged == *prior { + return Ok(vec![reference]); + } + self.persist(store, kind, leaves(kind, merged)?, work).await + } + Node::Branch(children) => { + let mut groups = BTreeMap::::new(); + for row in rows { + let index = children.partition_point(|child| child.last < row.0); + groups + .entry(index.min(children.len() - 1)) + .or_default() + .push(row); + } + let updated = futures_util::stream::iter(children.iter().cloned().enumerate()) + .map(|(index, child)| { + let rows = groups.remove(&index); + async move { + match rows { + Some(rows) => { + self.edit_node(store, child, kind, rows, work).await + } + None => Ok(vec![child]), + } + } + }) + .buffered(4) + .try_fold(Vec::new(), |mut updated, children| async move { + updated.extend(children); + Ok(updated) + }) + .await?; + if updated == *children { + return Ok(vec![reference]); + } + self.persist(store, kind, branches(kind, updated)?, work) + .await + } + } + }) + } + + async fn persist( + &self, + store: &Store, + kind: Kind, + pages: Vec, + work: &futures_util::lock::Mutex<&mut PageWork>, + ) -> Result> { + { + let mut work = work.lock().await; + for page in &pages { + work.admit_object(page.reference.hash, u64::from(page.reference.bytes))?; + } + } + futures_util::stream::iter(pages) + .map(|page| async move { + let _io = self.io[usize::from(page.reference.hash[0]) % self.io.len()] + .lock() + .await; + store + .put_if_absent_verified(&self.path(&page.reference.hash), page.bytes.clone()) + .await?; + let node = Arc::new(decode(&page.bytes, &page.reference, kind)?); + self.retain(&page.reference, kind, node)?; + Ok::<_, MetadataError>(page.reference) + }) + .buffered(4) + .try_collect() + .await + } +} + +fn resident_bytes(reference: &Reference, node: &Node) -> usize { + let heap = match node { + Node::Leaf(rows) => rows.iter().fold( + rows.capacity() + .saturating_mul(std::mem::size_of::<(Vec, Vec)>()), + |total, (key, value)| { + total + .saturating_add(key.capacity()) + .saturating_add(value.capacity()) + }, + ), + Node::Branch(children) => children.iter().fold( + children + .capacity() + .saturating_mul(std::mem::size_of::()), + |total, child| { + total + .saturating_add(child.first.capacity()) + .saturating_add(child.last.capacity()) + }, + ), + }; + std::mem::size_of::() + .saturating_add(std::mem::size_of::<([u8; 32], (Kind, Reference, Arc))>()) + .saturating_add(std::mem::size_of::<[u8; 32]>()) + .saturating_add(3 * std::mem::size_of::()) + .saturating_add(reference.first.capacity()) + .saturating_add(reference.last.capacity()) + .saturating_add(heap) +} diff --git a/crates/crab-metadata/src/pages/object/tests.rs b/crates/crab-metadata/src/pages/object/tests.rs new file mode 100644 index 000000000..f7d80ebcf --- /dev/null +++ b/crates/crab-metadata/src/pages/object/tests.rs @@ -0,0 +1,326 @@ +use super::*; +#[cfg(feature = "storage")] +use crate::pages::PageWork; + +fn rows(count: u32) -> Rows { + (0..count) + .map(|index| (index.to_be_bytes().to_vec(), vec![7; 96])) + .collect() +} + +#[test] +fn object_tree_authenticates_internal_routing_and_complete_order() { + let expected = (0..100_000u32) + .map(|index| { + let mut key = vec![0; 32]; + key[28..].copy_from_slice(&index.to_be_bytes()); + (key, vec![7; 96]) + }) + .collect::(); + let mut builder = Builder::default(); + let root = builder + .tree(Kind::CatalogFiles, expected.clone()) + .unwrap() + .unwrap(); + assert!(root.height >= 2); + let mut pending = vec![root]; + let mut actual = Vec::new(); + while let Some(reference) = pending.pop() { + let body = &builder.objects[&reference.hash]; + match decode(body, &reference, Kind::CatalogFiles).unwrap() { + Node::Leaf(rows) => actual.extend(rows), + Node::Branch(children) => pending.extend(children.into_iter().rev()), + } + } + assert_eq!(actual, expected); +} + +#[test] +fn object_page_rejects_corrupt_truncated_or_resigned_invalid_rows() { + let page = encode(Kind::CatalogXorbChunks, Node::Leaf(rows(3))).unwrap(); + for mode in ["corrupt", "truncated", "count", "order"] { + let mut body = page.bytes.to_vec(); + let mut reference = page.reference.clone(); + match mode { + "corrupt" => body[HEADER] ^= 1, + "truncated" => { + body.pop(); + } + "count" => body[10..14].copy_from_slice(&u32::MAX.to_le_bytes()), + "order" => body[HEADER..HEADER + 4].copy_from_slice(&9u32.to_be_bytes()), + _ => unreachable!(), + } + if mode != "corrupt" { + reference.hash = *blake3::hash(&body).as_bytes(); + reference.bytes = body.len() as u32; + } + assert!( + decode(&body, &reference, Kind::CatalogXorbChunks).is_err(), + "{mode}" + ); + } +} + +#[test] +fn object_directory_rejects_overlapping_children_and_false_cardinality() { + let mut builder = Builder::default(); + let root = builder + .tree(Kind::CatalogXorbChunks, rows(1000)) + .unwrap() + .unwrap(); + let Node::Branch(mut children) = + decode(&builder.objects[&root.hash], &root, Kind::CatalogXorbChunks).unwrap() + else { + panic!("fixture must contain an internal directory"); + }; + children[1].first = children[0].last.clone(); + assert!(encode(Kind::CatalogXorbChunks, Node::Branch(children)).is_err()); + let mut wrong = root.clone(); + wrong.count += 1; + wrong.last = (wrong.count - 1).to_be_bytes().to_vec(); + assert!( + decode( + &builder.objects[&root.hash], + &wrong, + Kind::CatalogXorbChunks + ) + .is_err() + ); +} + +#[cfg(feature = "storage")] +mod storage { + use super::*; + use crab_storage::{ImmutableWriteVerification, StorageObservation, StorageObserver, Store}; + use futures_util::TryStreamExt as _; + use std::sync::{Arc, Mutex}; + + #[derive(Default)] + struct Observations(Mutex>); + impl StorageObserver for Observations { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, observation: StorageObservation) { + self.0.lock().unwrap().push(observation); + } + } + fn fixture() -> (Store, ObjectPages, Arc) { + let observations = Arc::new(Observations::default()); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_immutable_write_verification(ImmutableWriteVerification::ReadbackRequired) + .with_storage_observer(observations.clone()); + ( + store.clone(), + ObjectPages::new(object_store::path::Path::from("catalog")), + observations, + ) + } + + #[tokio::test] + async fn small_edits_retain_history_and_reuse_unmodified_subtrees() { + for count in [1000, 100_000] { + let (store, writer, observations) = fixture(); + let original = rows(count); + let root = writer + .edit( + &store, + None, + Kind::CatalogXorbChunks, + original.clone(), + &mut PageWork::new(64 * 1024 * 1024, 0), + ) + .await + .unwrap() + .unwrap(); + let pages = ObjectPages::new(object_store::path::Path::from("catalog")); + observations.0.lock().unwrap().clear(); + let change = vec![(17u32.to_be_bytes().to_vec(), vec![9; 96])]; + let next = pages + .edit( + &store, + Some(&root), + Kind::CatalogXorbChunks, + change, + &mut PageWork::new(256 * 1024, 0), + ) + .await + .unwrap() + .unwrap(); + let io = observations.0.lock().unwrap().clone(); + assert!(io.iter().map(|op| op.bytes_read).sum::() < 128 * 1024); + assert!(io.iter().map(|op| op.bytes_written).sum::() < 128 * 1024); + let mut expected = original.clone(); + expected[17].1 = vec![9; 96]; + assert_eq!( + pages + .scan( + &store, + Some(&next), + Kind::CatalogXorbChunks, + &mut PageWork::new(64 * 1024 * 1024, 0) + ) + .await + .unwrap(), + expected + ); + assert_eq!( + pages + .scan( + &store, + Some(&root), + Kind::CatalogXorbChunks, + &mut PageWork::new(64 * 1024 * 1024, 0) + ) + .await + .unwrap(), + original + ); + } + } + + #[tokio::test] + async fn batched_queries_charge_warm_pages_and_preserve_duplicate_order() { + let (store, pages, observations) = fixture(); + let root = pages + .edit( + &store, + None, + Kind::CatalogXorbChunks, + rows(1000), + &mut PageWork::new(1024 * 1024, 0), + ) + .await + .unwrap() + .unwrap(); + let keys = [17, 500, 17, 2000].map(|i: u32| i.to_be_bytes().to_vec()); + observations.0.lock().unwrap().clear(); + let expected = vec![ + Some(vec![7; 96]), + Some(vec![7; 96]), + Some(vec![7; 96]), + None, + ]; + let mut work = PageWork::new(256 * 1024, 0); + assert_eq!( + pages + .lookup( + &store, + Some(&root), + Kind::CatalogXorbChunks, + &keys, + &mut work + ) + .await + .unwrap(), + expected + ); + assert!(observations.0.lock().unwrap().is_empty()); + assert!( + pages + .lookup( + &store, + Some(&root), + Kind::CatalogXorbChunks, + &keys, + &mut PageWork::new(100, 0) + ) + .await + .is_err() + ); + } + + #[tokio::test] + async fn missing_or_corrupt_objects_fail_before_returning_rows() { + for corrupt_body in [false, true] { + let (store, writer, _) = fixture(); + let root = writer + .edit( + &store, + None, + Kind::CatalogXorbChunks, + rows(1000), + &mut PageWork::new(1024 * 1024, 0), + ) + .await + .unwrap() + .unwrap(); + let path = writer.path(&root.hash); + if corrupt_body { + store + .put_overwrite(&path, Bytes::from(vec![0; root.bytes as usize])) + .await + .unwrap(); + } else { + store.delete(&path).await.unwrap(); + } + let cold = ObjectPages::new(object_store::path::Path::from("catalog")); + assert!( + cold.lookup( + &store, + Some(&root), + Kind::CatalogXorbChunks, + &[17u32.to_be_bytes().to_vec()], + &mut PageWork::new(256 * 1024, 0) + ) + .await + .is_err() + ); + } + } + + #[tokio::test] + async fn insertion_splits_preserve_every_key_and_streamed_dependency() { + let (store, pages, _) = fixture(); + let mut expected = BTreeMap::new(); + let mut root = None; + for pass in 0..8u32 { + let changes = (0..1000u32) + .map(|index| { + ( + (index * 8 + pass).to_be_bytes().to_vec(), + vec![pass as u8; 100], + ) + }) + .collect::(); + expected.extend(changes.clone()); + // Sparse keys exercise insertions before, inside and after old coverage. + root = pages + .edit( + &store, + root.as_ref(), + Kind::Children, + changes, + &mut PageWork::new(8 * 1024 * 1024, 0), + ) + .await + .unwrap(); + } + assert_eq!( + pages + .scan( + &store, + root.as_ref(), + Kind::Children, + &mut PageWork::new(8 * 1024 * 1024, 0) + ) + .await + .unwrap(), + expected.into_iter().collect::() + ); + let mut work = PageWork::new(8 * 1024 * 1024, 0); + let nodes = pages + .walk(&store, root.as_ref(), Kind::Children, &mut work) + .try_collect::>() + .await + .unwrap(); + assert_eq!( + nodes + .iter() + .filter_map(|(_, n)| match n.as_ref() { + Node::Leaf(rows) => Some(rows.len()), + _ => None, + }) + .sum::(), + 8000 + ); + } +} diff --git a/crates/crab-metadata/src/path_state/reader.rs b/crates/crab-metadata/src/path_state/reader.rs index 169a7a6b5..fd4fd24a0 100644 --- a/crates/crab-metadata/src/path_state/reader.rs +++ b/crates/crab-metadata/src/path_state/reader.rs @@ -1,9 +1,6 @@ //! Lazy lookups bound to an authenticated descriptor; full scans remain maintenance operations. use crab_storage::{Store, StoreLayout}; -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, -}; +use std::{collections::BTreeMap, sync::Arc}; use super::{ PathStateDescriptor, PathStateNodeRef, PathStateRecord, PathStateSummary, Result, codec, @@ -120,11 +117,7 @@ impl PathStateReader { .await? .remove(0) .ok_or_else(|| corruption("start commit is absent from graph"))?; - let mut work = PageWork { - maximum: self.max_bytes, - bytes: self.descriptor_bytes, - seen: BTreeSet::new(), - }; + let mut work = PageWork::new(self.max_bytes, self.descriptor_bytes); let start = self .records(store, graph, &[start_ordinal], &mut work) .await? diff --git a/crates/crab-metadata/src/split_commit_graph/reader.rs b/crates/crab-metadata/src/split_commit_graph/reader.rs index bec1ac05a..8a5c30134 100644 --- a/crates/crab-metadata/src/split_commit_graph/reader.rs +++ b/crates/crab-metadata/src/split_commit_graph/reader.rs @@ -329,11 +329,7 @@ impl CommitGraphReader { } fn work(&self) -> PageWork { - PageWork { - maximum: self.max_bytes, - bytes: self.descriptor_bytes, - seen: BTreeSet::new(), - } + PageWork::new(self.max_bytes, self.descriptor_bytes) } fn roots<'a>(&self, layer: &'a CommitGraphLayerRef) -> Result<&'a CommitGraphLayerPages> { diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index d155ac857..556399644 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -797,6 +797,30 @@ impl CapsuleRepositoryView { .any(CapsuleControl::has_catalog_delta)) } + /// Merge selected publication deltas in capture order without loading the checkpoint catalog. + /// + /// Dependencies may belong to the checkpoint. Fetch-only captures remain incomplete. + pub async fn visible_pointer_catalog_delta(&self) -> Result { + self.catalog_sources + .as_ref() + .ok_or(ReadError::IncompleteCatalogView)?; + let mut catalog = PointerCatalogAssembly::new(PointerCatalog::new()); + for capsule in &self.capsules { + if let Some(delta) = capsule.pointer_catalog_delta()? { + catalog.apply(&delta)?; + } + } + for capsule in &self.capsule_controls { + let layout = self.metadata_layout.as_ref().ok_or_else(|| { + ReadError::internal("catalog materialization has no captured origin") + })?; + if let Some(delta) = capsule.materialize_catalog(layout.store()).await? { + catalog.apply(&delta)?; + } + } + Ok(catalog.finish_delta()?) + } + /// Materialize the complete generation-pinned external pointer catalog. /// /// Fetch-only captures return `ReadError::IncompleteCatalogView`. @@ -819,19 +843,7 @@ impl CapsuleRepositoryView { None => PointerCatalog::new(), }; let mut catalog = PointerCatalogAssembly::new(base_catalog); - for capsule in &self.capsules { - if let Some(delta) = capsule.pointer_catalog_delta()? { - catalog.apply(&delta)?; - } - } - for capsule in &self.capsule_controls { - let layout = self.metadata_layout.as_ref().ok_or_else(|| { - ReadError::internal("catalog materialization has no captured origin") - })?; - if let Some(delta) = capsule.materialize_catalog(layout.store()).await? { - catalog.apply(&delta)?; - } - } + catalog.apply(&self.visible_pointer_catalog_delta().await?)?; Ok(catalog.finish()?) } diff --git a/crates/crab-remote/Cargo.toml b/crates/crab-remote/Cargo.toml index bf30ec429..615243933 100644 --- a/crates/crab-remote/Cargo.toml +++ b/crates/crab-remote/Cargo.toml @@ -41,6 +41,7 @@ futures-util = { workspace = true, optional = true } object_store = { workspace = true, optional = true } [dev-dependencies] +crab-storage = { workspace = true, features = ["test-support"] } slatedb = "=0.15.0" sha1 = "0.10" tracing-subscriber = "0.3" diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index a855fb0ea..25f49f7b2 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -201,7 +201,7 @@ pub async fn maintain_capsule_repository_from_view( publish_checkpoint( layout, view, - catalog_for_checkpoint(view).await?, + catalog_for_checkpoint(layout, view, maximum_bytes).await?, maximum_bytes, cancel, ) @@ -299,7 +299,7 @@ pub async fn repack_capsule_repository_from_view( view, consolidation.sources, member_oids, - catalog_for_checkpoint(view).await?, + catalog_for_checkpoint(layout, view, maximum_bytes).await?, maximum_bytes, cancel, ) @@ -315,7 +315,7 @@ pub async fn repack_capsule_repository_from_view( publication = publish_checkpoint( layout, view, - catalog_for_checkpoint(view).await?, + catalog_for_checkpoint(layout, view, maximum_bytes).await?, maximum_bytes, cancel, ) @@ -361,7 +361,7 @@ pub async fn publish_capsule_checkpoint_from_view( if view.capsule_count()? < u64::from(threshold) { return Ok(CheckpointOutcome::default()); } - let catalog = catalog_for_checkpoint(view).await?; + let catalog = catalog_for_checkpoint(layout, view, maximum_bytes).await?; Ok( publish_checkpoint(layout, view, catalog, maximum_bytes, cancel) .await? @@ -395,18 +395,35 @@ pub async fn publish_capsule_checkpoint_with_catalog_from_view( enum CatalogWrite { New(PointerCatalog), - Retained(Option>), + Durable(Option>), } async fn catalog_for_checkpoint( + layout: &StoreLayout, view: &crab_read::capsule_protocol::CapsuleRepositoryView, + maximum_bytes: u64, ) -> Result { if view.has_catalog_deltas()? { - return Ok(CatalogWrite::New(view.pointer_catalog().await?)); + let delta = view.visible_pointer_catalog_delta().await?; + let existing = view + .layered_checkpoint() + .and_then(LayeredCheckpoint::catalog); + let catalog = crab_metadata::capsule_protocol::fold_checkpoint_catalog( + layout, + existing, + std::slice::from_ref(&delta), + if maximum_bytes == 0 { + u64::MAX + } else { + maximum_bytes + }, + ) + .await?; + return Ok(CatalogWrite::Durable(catalog.map(Box::new))); } // The checkpoint freezes prior replay precedence. Reuse only when every // newly selected control lacks a catalog delta, even if other refs moved. - Ok(CatalogWrite::Retained( + Ok(CatalogWrite::Durable( view.layered_checkpoint() .and_then(LayeredCheckpoint::catalog) .cloned() @@ -523,7 +540,7 @@ async fn repack_checkpoint_from_view( let Some(selected_start) = layered_suffix_start(checkpoint.sources())? else { return Ok(CheckpointPublication::default()); }; - let catalog = CatalogWrite::Retained(checkpoint.catalog().cloned().map(Box::new)); + let catalog = CatalogWrite::Durable(checkpoint.catalog().cloned().map(Box::new)); let consolidation = consolidate_layered_suffix( layout, view, @@ -586,7 +603,7 @@ async fn publish_layered_sources( CatalogWrite::New(catalog) => { LayeredCheckpoint::build(generation, root_digest, sources, catalog, Some(visibility))? } - CatalogWrite::Retained(catalog) => LayeredCheckpoint::build_reusing_catalog( + CatalogWrite::Durable(catalog) => LayeredCheckpoint::build_with_catalog_root( generation, root_digest, sources, @@ -594,7 +611,7 @@ async fn publish_layered_sources( Some(visibility), )?, }; - if checkpoint.encoded_metadata_bytes()? > maximum_bytes && maximum_bytes > 0 { + if checkpoint.publication_metadata_bytes()? > maximum_bytes && maximum_bytes > 0 { return Err(CheckpointError::OutputLimit { maximum: maximum_bytes, }); diff --git a/crates/crab-remote/tests/checkpoint.rs b/crates/crab-remote/tests/checkpoint.rs index 9a478c5e5..18861ceb0 100644 --- a/crates/crab-remote/tests/checkpoint.rs +++ b/crates/crab-remote/tests/checkpoint.rs @@ -34,12 +34,24 @@ const LIMITS: CapsuleReadLimits = CapsuleReadLimits { max_frontier_bytes: LIMIT, }; -#[derive(Default)] struct Observations( Mutex>, Mutex>, + Arc, ); +impl Default for Observations { + fn default() -> Self { + Self( + Mutex::default(), + Mutex::default(), + Arc::new(crab_storage::test_support::CountingObjectStore::new( + Arc::new(object_store::memory::InMemory::new()), + )), + ) + } +} + impl StorageObserver for Observations { fn started(&self, _: StorageOperation) {} @@ -541,7 +553,7 @@ async fn compacted_frontier_uses_resident_indexes_and_verifies_payloads() { async fn empty_fixture() -> (StoreLayout, Arc) { let observations = Arc::new(Observations::default()); - let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + let store = Store::new(observations.2.clone()) .with_immutable_write_verification(ImmutableWriteVerification::Sha256Checksum) .with_storage_observer(observations.clone()); let layout = StoreLayout::new(store, "repositories/checkpoint".to_owned()); diff --git a/crates/crab-remote/tests/checkpoint/publication_catalog.rs b/crates/crab-remote/tests/checkpoint/publication_catalog.rs index 0aa6569b4..6c16ce556 100644 --- a/crates/crab-remote/tests/checkpoint/publication_catalog.rs +++ b/crates/crab-remote/tests/checkpoint/publication_catalog.rs @@ -78,12 +78,11 @@ async fn native_content( (content, pointer, MerkleHash::from(hash).hex()) } -async fn publish_history(layout: &StoreLayout, catalog: PointerCatalog) { +async fn publish_history(layout: &StoreLayout, catalog: PointerCatalog, name: &str) { let root = crab_write::capsule_protocol::open_root(layout) .await .unwrap(); let (oid, pack) = blob_pack(b"history marker"); - let name = "refs/tags/history"; let transaction = CapsuleTransaction::new( root.record().digest(), vec![CapsuleRefEdit::new(name, None, Some(oid.clone()), None)], @@ -185,7 +184,7 @@ async fn fixed_pointer_publication_and_hydration_keep_unrelated_history_cold() { catalog.insert_file(hash.clone(), entry.clone()).unwrap(); } } - publish_history(&layout, catalog.clone()).await; + publish_history(&layout, catalog.clone(), "refs/tags/history").await; if checkpointed { assert!( crab_remote::checkpoint::publish_capsule_checkpoint( @@ -251,6 +250,7 @@ async fn fixed_pointer_publication_and_hydration_keep_unrelated_history_cold() { captured_lookup.close().await?; assert_eq!(captured_result?, if supplied { None } else { Some(expected_shard) }, "captured hydration must not follow a later publication"); observations.0.lock().unwrap().clear(); + observations.2.reset(); let lookup = crab_metadata::file_index_lookup::SharedFileIndexLookup::for_pointer_catalog( layout.clone(), after.pointer_catalog_lookup(&layout, 256 * 1024)?, ); @@ -263,7 +263,12 @@ async fn fixed_pointer_publication_and_hydration_keep_unrelated_history_cold() { let requests = observations.0.lock().unwrap().clone(); assert!(requests.len() <= 24 && requests.iter().map(|r| r.bytes_read).sum::() <= 256 * 1024, "fixed hydration lookup exceeds bounds: count={count}, checkpoint={checkpointed}, supplied={supplied}"); - assert!(requests.iter().all(|r| r.operation == StorageOperation::Range), "captured hydration must not read mutable state or write checkpoints"); + use crab_storage::test_support::ObjectReadKind; + let catalog_prefix = format!("{}/", layout.capsule_catalog_prefix()); + assert!(observations.2.put_requests() == 0 && observations.2.requests().iter().all(|r| { + r.kind == ObjectReadKind::Range || + (r.kind == ObjectReadKind::Full && r.location.starts_with(&catalog_prefix)) + }), "captured hydration must read only pinned ranges or immutable catalog pages, without writes"); drop(prepared); Ok::<_, Box>(()) }, @@ -306,7 +311,7 @@ async fn ref_only_checkpoints_reuse_catalog_without_body_io() { .unwrap(); } assert!(catalog.encode().unwrap().len() > 64 * 1024); - publish_history(&layout, catalog.clone()).await; + publish_history(&layout, catalog.clone(), "refs/tags/history").await; // Publication needs exact pack-member admission. The maintenance view // supplies it while checkpoint catalog bodies remain cold. let first = view(&layout).await; @@ -357,3 +362,53 @@ async fn ref_only_checkpoints_reuse_catalog_without_body_io() { .unwrap(); assert_eq!(after.pointer_catalog().await.unwrap(), catalog); } + +#[tokio::test(flavor = "multi_thread")] +async fn changed_catalog_checkpoints_do_not_rewrite_unrelated_history() { + for count in [100, 10_000] { + let (layout, observations) = empty_fixture().await; + let shard = "a".repeat(64); + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(1, Vec::new())) + .unwrap(); + for index in 0..count { + catalog + .insert_file(format!("{index:064x}"), FileCatalogEntry::new(1, &shard)) + .unwrap(); + } + publish_history(&layout, catalog.clone(), "refs/tags/history").await; + let captured = checkpoint(&layout).await; + let mut delta = PointerCatalog::new(); + delta + .insert_file(format!("{count:064x}"), FileCatalogEntry::new(1, &shard)) + .unwrap(); + publish_history(&layout, delta.clone(), "refs/tags/edit").await; + let before = view(&layout).await; + assert!(before.has_catalog_deltas().unwrap()); + observations.0.lock().unwrap().clear(); + let result = crab_remote::checkpoint::publish_capsule_checkpoint_from_view( + &layout, + &before, + 0, + 64 * LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + let io = observations.0.lock().unwrap().clone(); + assert!(result.published); + let bytes_read = io.iter().map(|op| op.bytes_read).sum::(); + let bytes_written = io.iter().map(|op| op.bytes_written).sum::(); + assert!( + bytes_read <= 256 * 1024 && bytes_written <= 256 * 1024, + "fixed catalog edit rewrote history: files={count}, read={bytes_read}, written={bytes_written}" + ); + assert_eq!(captured.pointer_catalog().await.unwrap(), catalog); + catalog.apply(&delta).unwrap(); + assert_eq!( + view(&layout).await.pointer_catalog().await.unwrap(), + catalog + ); + } +} diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index f1fb7ac9c..4eb49265a 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -3294,7 +3294,7 @@ mod tests { ) .await .unwrap(); - assert_eq!(restored.record().root().checkpoint().unwrap().format(), 8); + assert_eq!(restored.record().root().checkpoint().unwrap().format(), 9); assert_eq!( restored.record().root().generation(), fenced.record().root().generation() + 1 From f66277a761c42867a9c2ed88a3cf15148ab8bb57 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 20:16:54 -0700 Subject: [PATCH 110/127] test(metadata): reject retired checkpoint formats Format9 is admitted by both root validation and positive history tests. Reject format8 and future10 instead of listing9 as unsupported. All metadata index features: 623 passed, one existing stress case ignored. Production source is identical to the qualified COW build and RustFS run. --- crates/crab-metadata/src/capsule_protocol/history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/crab-metadata/src/capsule_protocol/history.rs b/crates/crab-metadata/src/capsule_protocol/history.rs index 86864e70e..7ef35239f 100644 --- a/crates/crab-metadata/src/capsule_protocol/history.rs +++ b/crates/crab-metadata/src/capsule_protocol/history.rs @@ -441,7 +441,7 @@ mod tests { #[test] fn history_rejects_unsupported_checkpoint_formats() { - for format in [0, 1, 2, 3, 4, 5, 6, 7, 9, u32::MAX] { + for format in [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, u32::MAX] { let segment = history_with_unvalidated_pointers(format, 1); let built = HistorySegment::build( segment.checkpoint().clone(), From d7a865c2363e8c1b53dcb8a4b68f7bf0a0b72b00 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 21:17:22 -0700 Subject: [PATCH 111/127] perf(metadata): stream catalog maintenance joins Scan authenticated pages once and reduce bounded relation batches in private local spills. Preserve complete dependency and canonical reverse placement validation without full recipes or random origin joins. The rejected point-join design read 2.70 GB for a 15.8 MB catalog. This path reads 15.8 MB, with bounded row windows and admitted live disk bytes. Blocking workers retain workspace ownership through cancellation and close drains them before cleanup. No dependency or config changes. Proof: 629 metadata tests, strict component Clippy, publication/checkpoint and GC/history/fsck checks, CLI build and 63 real RustFS checks. The resource profile includes fixture/full-oracle work, not query-only RSS. Full logical-catalog consumer conversion and final-source 10 GiB gates remain open. --- .../docs/architecture/indexed-remote-reads.md | 15 +- crates/crab-metadata/README.md | 13 +- .../src/capsule_protocol/pointer/object.rs | 9 +- .../capsule_protocol/pointer/object/reader.rs | 23 +- .../pointer/object/reader/inspection.rs | 169 ++++++++ .../pointer/object/reader/inspection/join.rs | 404 ++++++++++++++++++ .../capsule_protocol/pointer/object/tests.rs | 294 +++++++++++++ .../crab-metadata/src/pages/object/store.rs | 71 ++- .../crab-metadata/src/pages/object/tests.rs | 26 +- crates/crab-storage/AGENTS.md | 6 + crates/crab-storage/README.md | 14 + crates/crab-storage/src/lib.rs | 1 + crates/crab-storage/src/scratch.rs | 242 +++++++++++ crates/crab-storage/src/scratch/tests.rs | 94 ++++ 14 files changed, 1302 insertions(+), 79 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs create mode 100644 crates/crab-storage/src/scratch.rs create mode 100644 crates/crab-storage/src/scratch/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 2f98897c2..163e0755c 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2509,8 +2509,19 @@ cache at 8 MiB/256 entries and shares four I/O lanes across recursive edits. GC authenticates page closures for current checkpoints, historical checkpoints and retained browse captures. A coordinator-protected descriptor key does not count as a verified closure. Missing/corrupt descendants abort maintenance. -Full catalog maintenance still materializes logical metadata within its caller -budget; streaming full inspection remains outstanding. +Catalog-object marking scans each authenticated page once and reduces bounded +relation batches through private local spill files. A point-join implementation +was rejected: a 1,000-xorb/128,000-distinct-chunk fixture read 2,698,130,639 bytes +versus 15,818,115 for a cold full scan. Spill reduction reads 15,818,115 bytes on +that same fixture while retaining bounded row windows and physical page identities. +These are in-memory backend byte counts; complete test durations (80.81 seconds +before, 13.34 after) include fixture construction and the full-scan comparison. +They do not establish hosted latency or query-only RSS. Disk admission precedes +spill growth; cleanup drains blocking workers. The existing temporary-directory +setting controls placement, with no new config/env surface or dependency. +Materialization admission still protects callers retaining recipe copies. Full +logical-catalog APIs and maintenance consumers still using them remain separate +conversion and runtime-qualification work. Release audit: all six upstream published tag objects were checked again before this change; none contains the capsule codec. Unshipped checkpoint formats diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index cc95b1537..49439536c 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -75,8 +75,17 @@ Readers share an aggregate page-byte budget across checkpoint and packed-run sources. Logical expansion has a separate allowance of four times the encoded ceiling: shared physical pages cannot authorize unbounded recipe/result copies. Cold and warm reads consume admission; decoded caches have independent resident limits. -Full inspection authenticates every dependency and reverse placement; it still -materializes the logical catalog and is not yet a streaming-memory guarantee. +Physical catalog-object inspection visits authenticated pages once and spills +bounded relation batches into a private local workspace. Sorted reductions check +file/shard/xorb dependencies and the lowest-xorb/first-occurrence reverse mapping +without random origin joins or a complete logical catalog in memory. Shared +positional pages retain their earliest canonical owner; inconsistent references +still fail. The existing temporary-directory setting selects the workspace parent; +its live disk ceiling is four times the caller's encoded-byte ceiling. Workers +retain file ownership through cancellation and are drained before normal/error +cleanup. No authoritative or cache object is written by inspection. Explicit +full-catalog callers still materialize it; converting remaining maintenance +consumers is separate work. GC traverses catalog pages from current, historical and retained browse roots. Protected keys and authenticated closure traversal are tracked separately. Retired unshipped checkpoint formats through 8 are rejected without fallback. diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object.rs index 7836a9c5c..0550e6c8b 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object.rs @@ -138,8 +138,13 @@ fn json(value: &T) -> Result> { } #[cfg(feature = "storage")] -fn from_json(bytes: &[u8]) -> Result { - serde_json::from_slice(bytes).map_err(|source| MetadataError::PointerCatalogRecord { source }) +fn from_json(bytes: &[u8]) -> Result { + let value = serde_json::from_slice(bytes) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if json(&value)? != bytes { + return Err(corrupt("catalog record is not canonical")); + } + Ok(value) } fn file_rows(catalog: &PointerCatalog) -> Result { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs index 4ae396979..85851daaa 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs @@ -1,3 +1,5 @@ +mod inspection; + use super::*; use crate::pages::PageWork; use crate::pages::object::ObjectPages; @@ -92,9 +94,6 @@ impl CatalogPages { work: &mut PageWork, ) -> Result { let row: XorbRow = from_json(bytes)?; - if json(&row)? != bytes { - return Err(corrupt("catalog xorb row is not canonical")); - } let rows = self .pages .scan(store, Some(&row.chunks), Kind::CatalogXorbChunks, work) @@ -109,9 +108,6 @@ impl CatalogPages { work: &mut PageWork, ) -> Result { let row: ShardRow = from_json(bytes)?; - if json(&row)? != bytes { - return Err(corrupt("catalog shard row is not canonical")); - } let rows = self .pages .scan(store, row.xorbs.as_ref(), Kind::CatalogShardXorbs, work) @@ -187,21 +183,6 @@ impl CatalogPages { Ok(catalog) } - pub(in crate::capsule_protocol) async fn inspected_objects( - &self, - store: &Store, - maximum: u64, - ) -> Result> { - let mut work = PageWork::new(maximum, 0); - // A missing or inconsistent dependency must abort a sweep; mere root - // presence cannot authorize deletion of shared persistent pages. - self.full(store, &mut work).await?; - Ok(work - .object_dependencies() - .map(|(hash, bytes)| (self.pages.path(hash), bytes)) - .collect()) - } - pub(in crate::capsule_protocol) async fn apply( &self, store: &Store, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs new file mode 100644 index 000000000..d4b6f8a25 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs @@ -0,0 +1,169 @@ +//! One physical catalog scan followed by bounded local relation reduction. +use super::*; +use crate::pages::object::Node; +use std::sync::Arc; + +mod join; +use join::{Join, Record}; + +type Seen = BTreeMap<[u8; 32], (Kind, Reference)>; + +impl CatalogPages { + pub(in crate::capsule_protocol) async fn inspected_objects( + &self, + store: &Store, + maximum: u64, + ) -> Result> { + let mut work = PageWork::new(maximum, 0); + self.open(store, &mut work).await?; + let mut join = Join::create(&std::env::temp_dir(), maximum).await?; + let checked = self.inspect(store, &mut work, &mut join).await; + join.finish(checked).await?; + Ok(work + .object_dependencies() + .map(|(hash, bytes)| (self.pages.path(hash), bytes)) + .collect()) + } + + async fn inspect(&self, store: &Store, work: &mut PageWork, join: &mut Join) -> Result<()> { + let mut seen = Seen::new(); + for kind in [ + Kind::CatalogFiles, + Kind::CatalogShards, + Kind::CatalogXorbs, + Kind::CatalogChunks, + ] { + let mut pending = self.root(kind)?.cloned().into_iter().collect(); + while let Some(node) = self + .inspection_page(store, kind, &mut pending, work, &mut seen) + .await? + { + let Node::Leaf(rows) = node.as_ref() else { + continue; + }; + match kind { + Kind::CatalogFiles => { + for (_, value) in rows { + let file = file_from_bytes(value)?; + join.add(Record::reference(0, &hash(file.shard_hash())?, false)?) + .await?; + } + } + Kind::CatalogShards => { + for (identity, value) in rows { + let row: ShardRow = from_json(value)?; + if row.encoded_size == 0 { + return Err(corrupt("catalog shard is empty")); + } + join.add(Record::reference(0, identity, true)?).await?; + let mut pending = row.xorbs.into_iter().collect(); + while let Some(node) = self + .inspection_page( + store, + Kind::CatalogShardXorbs, + &mut pending, + work, + &mut seen, + ) + .await? + { + if let Node::Leaf(rows) = node.as_ref() { + for (xorb, value) in rows { + if !value.is_empty() { + return Err(corrupt( + "catalog shard closure has unexpected row data", + )); + } + join.add(Record::reference(1, xorb, false)?).await?; + } + } + } + } + } + Kind::CatalogXorbs => { + for (identity, value) in rows { + let row: XorbRow = from_json(value)?; + if row.encoded_size == 0 || row.chunks.first != 0u32.to_be_bytes() { + return Err(corrupt( + "catalog xorb is empty or its chunk sequence starts late", + )); + } + join.add(Record::reference(1, identity, true)?).await?; + let mut pending = vec![row.chunks]; + while let Some(node) = self + .inspection_page( + store, + Kind::CatalogXorbChunks, + &mut pending, + work, + &mut seen, + ) + .await? + { + if let Node::Leaf(rows) = node.as_ref() { + for (index, value) in rows { + if value.len() != 36 || value[32..] == [0; 4] { + return Err(corrupt( + "catalog chunk length or size differs", + )); + } + let mut placement = [0; 40]; + placement[..32].copy_from_slice(identity); + placement[32..36].copy_from_slice(index); + placement[36..].copy_from_slice(&value[32..]); + join.add(Record::chunk(&value[..32], placement, false)?) + .await?; + } + } + } + } + } + Kind::CatalogChunks => { + for (hash, value) in rows { + let placement = value + .as_slice() + .try_into() + .map_err(|_| corrupt("catalog chunk placement length differs"))?; + join.add(Record::chunk(hash, placement, true)?).await?; + } + } + _ => return Err(corrupt("invalid inspection tree")), + } + } + } + Ok(()) + } + + async fn inspection_page( + &self, + store: &Store, + kind: Kind, + pending: &mut Vec, + work: &mut PageWork, + seen: &mut Seen, + ) -> Result>> { + while let Some(reference) = pending.last() { + reference.validate(kind)?; + if let Some((prior_kind, prior)) = seen.get(&reference.hash) { + if *prior_kind != kind || prior != reference { + return Err(corrupt( + "shared inspection page has conflicting routing metadata", + )); + } + // Xorbs arrive in canonical order. A shared positional page was + // already recorded for an earlier owner; later copies cannot win. + // Shard membership is a set, so repeated closure pages add no work. + pending.pop(); + continue; + } + let (reference, node) = self + .pages + .next_page(store, pending, kind, work) + .await? + .ok_or_else(|| corrupt("inspection lost its pending page"))?; + seen.insert(reference.hash, (kind, reference)); + return Ok(Some(node)); + } + Ok(None) + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs new file mode 100644 index 000000000..b5986d6a8 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs @@ -0,0 +1,404 @@ +use super::*; +use crab_storage::scratch::{ScratchFile, ScratchSpace}; +use std::collections::VecDeque; + +const MAGIC: &[u8; 8] = b"CRBCSJ01"; +const BLOCK_ROWS: usize = 4096; +const ROW_BYTES: usize = 114; +const MAX_BLOCK_BYTES: u64 = (12 + BLOCK_ROWS * ROW_BYTES) as u64; + +pub(super) struct Record { + key: [u8; 33], + required: bool, + defined: bool, + expected: Option<[u8; 40]>, + actual: Option<[u8; 40]>, +} + +impl Record { + pub(super) fn reference(kind: u8, hash: &[u8], defined: bool) -> Result { + if hash.len() != 32 { + return Err(corrupt("inspection reference hash length differs")); + } + let mut key = [0; 33]; + key[0] = kind; + key[1..].copy_from_slice(hash); + Ok(Self { + key, + required: !defined, + defined, + expected: None, + actual: None, + }) + } + + pub(super) fn chunk(hash: &[u8], placement: [u8; 40], actual: bool) -> Result { + if hash.len() != 32 { + return Err(corrupt("inspection chunk hash length differs")); + } + let mut key = [0; 33]; + key[0] = 2; + key[1..].copy_from_slice(hash); + Ok(Self { + key, + required: false, + defined: false, + expected: (!actual).then_some(placement), + actual: actual.then_some(placement), + }) + } + + fn merge(&mut self, other: Self) -> Result<()> { + self.required |= other.required; + self.defined |= other.defined; + self.expected = match (self.expected, other.expected) { + (Some(left), Some(right)) => Some(left.min(right)), + (left, right) => left.or(right), + }; + if self.actual.is_some() && other.actual.is_some() { + return Err(corrupt("catalog reverse placement occurs more than once")); + } + self.actual = self.actual.or(other.actual); + Ok(()) + } + + fn validate(&self) -> Result<()> { + match self.key[0] { + 0 | 1 if !self.required || self.defined => Ok(()), + 2 if self.expected.is_some() && self.actual == self.expected => Ok(()), + _ => Err(corrupt( + "catalog dependency or canonical reverse placement differs", + )), + } + } + + fn encode(&self, bytes: &mut Vec) { + bytes.extend_from_slice(&self.key); + bytes.push( + u8::from(self.required) + | (u8::from(self.defined) << 1) + | (u8::from(self.expected.is_some()) << 2) + | (u8::from(self.actual.is_some()) << 3), + ); + bytes.extend_from_slice(&self.expected.unwrap_or([0; 40])); + bytes.extend_from_slice(&self.actual.unwrap_or([0; 40])); + } +} + +struct Run { + files: Vec, +} + +struct Cursor { + files: VecDeque, + rows: std::vec::IntoIter, + previous: Option<[u8; 33]>, +} + +impl Cursor { + fn new(run: Run) -> Self { + Self { + files: run.files.into(), + rows: Vec::new().into_iter(), + previous: None, + } + } + + async fn next(&mut self, space: &ScratchSpace) -> Result> { + loop { + if let Some(row) = self.rows.next() { + if self.previous.is_some_and(|previous| previous >= row.key) { + return Err(corrupt("inspection spill rows are not strictly ordered")); + } + self.previous = Some(row.key); + return Ok(Some(row)); + } + let Some(file) = self.files.pop_front() else { + return Ok(None); + }; + let body = space.read(&file, MAX_BLOCK_BYTES).await?; + self.rows = decode(&body)?.into_iter(); + } + } +} + +fn encode(rows: &[Record]) -> Result { + if rows.is_empty() || rows.len() > BLOCK_ROWS { + return Err(corrupt("inspection spill block count exceeds its bound")); + } + let mut bytes = Vec::with_capacity(12 + rows.len() * ROW_BYTES); + bytes.extend_from_slice(MAGIC); + bytes.extend_from_slice(&(rows.len() as u32).to_be_bytes()); + for row in rows { + row.encode(&mut bytes); + } + Ok(bytes.into()) +} + +fn decode(bytes: &[u8]) -> Result> { + if bytes.len() < 12 || bytes.get(..8) != Some(MAGIC.as_slice()) { + return Err(corrupt("inspection spill block framing differs")); + } + let count = u32::from_be_bytes( + bytes[8..12] + .try_into() + .map_err(|_| corrupt("spill count is truncated"))?, + ) as usize; + if count == 0 || count > BLOCK_ROWS || bytes.len() != 12 + count * ROW_BYTES { + return Err(corrupt("inspection spill block count differs")); + } + bytes[12..] + .chunks_exact(ROW_BYTES) + .map(|row| { + let key: [u8; 33] = row[..33] + .try_into() + .map_err(|_| corrupt("spill key is truncated"))?; + let flags = row[33]; + if flags == 0 + || flags > 15 + || key[0] > 2 + || (key[0] < 2 && flags & 12 != 0) + || (key[0] == 2 && flags & 3 != 0) + || (flags & 4 == 0 && row[34..74].iter().any(|byte| *byte != 0)) + || (flags & 8 == 0 && row[74..].iter().any(|byte| *byte != 0)) + { + return Err(corrupt("inspection spill row shape differs")); + } + Ok(Record { + key, + required: flags & 1 != 0, + defined: flags & 2 != 0, + expected: if flags & 4 != 0 { + Some( + row[34..74] + .try_into() + .map_err(|_| corrupt("spill placement is truncated"))?, + ) + } else { + None + }, + actual: if flags & 8 != 0 { + Some( + row[74..] + .try_into() + .map_err(|_| corrupt("spill placement is truncated"))?, + ) + } else { + None + }, + }) + }) + .collect() +} + +pub(super) struct Join { + space: ScratchSpace, + buffer: Vec, + levels: Vec>, +} + +impl Join { + pub(super) async fn create(parent: &std::path::Path, maximum: u64) -> Result { + Ok(Self { + space: ScratchSpace::create(parent, maximum.saturating_mul(4)).await?, + buffer: Vec::with_capacity(BLOCK_ROWS), + levels: Vec::new(), + }) + } + + pub(super) async fn add(&mut self, row: Record) -> Result<()> { + self.buffer.push(row); + if self.buffer.len() == BLOCK_ROWS { + self.flush().await?; + } + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + if self.buffer.is_empty() { + return Ok(()); + } + let mut rows = std::mem::replace(&mut self.buffer, Vec::with_capacity(BLOCK_ROWS)); + rows.sort_unstable_by_key(|row| row.key); + let mut grouped: Vec = Vec::with_capacity(rows.len()); + for row in rows { + match grouped.last_mut().filter(|prior| prior.key == row.key) { + Some(prior) => prior.merge(row)?, + None => grouped.push(row), + } + } + let mut run = Run { + files: vec![self.space.write(encode(&grouped)?).await?], + }; + let mut level = 0; + loop { + if level == self.levels.len() { + self.levels.push(None); + } + match self.levels[level].take() { + Some(prior) => run = self.merge(prior, run).await?, + None => { + self.levels[level] = Some(run); + return Ok(()); + } + } + level += 1; + } + } + + async fn merge(&self, left: Run, right: Run) -> Result { + let retired = left + .files + .iter() + .chain(&right.files) + .cloned() + .collect::>(); + let mut left = Cursor::new(left); + let mut right = Cursor::new(right); + let mut a = left.next(&self.space).await?; + let mut b = right.next(&self.space).await?; + let mut rows = Vec::with_capacity(BLOCK_ROWS); + let mut output = Run { files: Vec::new() }; + while a.is_some() || b.is_some() { + let take_left = match (&a, &b) { + (Some(a), Some(b)) => a.key <= b.key, + (Some(_), None) => true, + _ => false, + }; + let row = if take_left { + let mut row = a + .take() + .ok_or_else(|| corrupt("spill merge lost its left row"))?; + a = left.next(&self.space).await?; + if b.as_ref().is_some_and(|other| other.key == row.key) { + row.merge( + b.take() + .ok_or_else(|| corrupt("spill merge lost its right row"))?, + )?; + b = right.next(&self.space).await?; + } + row + } else { + let row = b + .take() + .ok_or_else(|| corrupt("spill merge lost its right row"))?; + b = right.next(&self.space).await?; + row + }; + rows.push(row); + if rows.len() == BLOCK_ROWS { + output.files.push(self.space.write(encode(&rows)?).await?); + rows.clear(); + } + } + if !rows.is_empty() { + output.files.push(self.space.write(encode(&rows)?).await?); + } + for file in retired { + self.space.remove(file).await?; + } + Ok(output) + } + + pub(super) async fn finish(mut self, prior: Result<()>) -> Result<()> { + let result: Result<()> = async { + prior?; + self.flush().await?; + let mut combined = None; + for run in std::mem::take(&mut self.levels).into_iter().flatten() { + combined = Some(match combined.take() { + Some(prior) => self.merge(prior, run).await?, + None => run, + }); + } + if let Some(run) = combined { + let mut cursor = Cursor::new(run); + while let Some(row) = cursor.next(&self.space).await? { + row.validate()?; + } + } + Ok(()) + } + .await; + let cleanup = self.space.close().await.map_err(MetadataError::from); + if let (Err(_), Err(source)) = (&result, &cleanup) { + tracing::warn!(%source, "inspection spill cleanup failed after validation error"); + } + result?; + cleanup + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn spilled_join_uses_the_lowest_owner_across_separate_runs() { + for correct in [false, true] { + let parent = tempfile::tempdir().unwrap(); + let mut join = Join::create(parent.path(), 8 * 1024 * 1024).await.unwrap(); + let mut high = [9; 40]; + let mut low = high; + low[..32].fill(1); + high[32..36].copy_from_slice(&0u32.to_be_bytes()); + low[32..36].copy_from_slice(&0u32.to_be_bytes()); + join.add(Record::chunk(&[3; 32], high, false).unwrap()) + .await + .unwrap(); + join.add(Record::chunk(&[3; 32], if correct { low } else { high }, true).unwrap()) + .await + .unwrap(); + for index in 0..10_000u32 { + let mut key = [0; 32]; + key[28..].copy_from_slice(&index.to_be_bytes()); + join.add(Record::reference(0, &key, true).unwrap()) + .await + .unwrap(); + } + join.add(Record::chunk(&[3; 32], low, false).unwrap()) + .await + .unwrap(); + assert_eq!(join.finish(Ok(())).await.is_ok(), correct); + assert_eq!(std::fs::read_dir(parent.path()).unwrap().count(), 0); + } + } + + #[tokio::test] + async fn prior_scan_failure_drains_and_removes_written_runs() { + let parent = tempfile::tempdir().unwrap(); + let mut join = Join::create(parent.path(), 8 * 1024 * 1024).await.unwrap(); + for index in 0..5000u32 { + let mut key = [0; 32]; + key[28..].copy_from_slice(&index.to_be_bytes()); + join.add(Record::reference(0, &key, true).unwrap()) + .await + .unwrap(); + } + assert!( + join.finish(Err(corrupt("source page missing"))) + .await + .is_err() + ); + assert_eq!(std::fs::read_dir(parent.path()).unwrap().count(), 0); + } + + #[test] + fn spill_decoder_rejects_unbounded_counts_and_noncanonical_rows() { + let row = Record::reference(0, &[1; 32], true).unwrap(); + let canonical = encode(&[row]).unwrap(); + for scenario in ["count", "truncated", "flags", "absent-value"] { + let mut bytes = canonical.to_vec(); + match scenario { + "count" => bytes[8..12].copy_from_slice(&u32::MAX.to_be_bytes()), + "truncated" => { + bytes.pop(); + } + "flags" => bytes[12 + 33] = 255, + "absent-value" => bytes[12 + 34] = 1, + _ => unreachable!(), + } + assert!(decode(&bytes).is_err(), "{scenario}"); + } + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs index 9d0a8a367..1b3044520 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -281,3 +281,297 @@ async fn superseded_reverse_placement_still_validates_prior_dependency() { .is_err() ); } + +#[tokio::test] +async fn maintenance_streams_shared_recipes_without_materializing_the_catalog() { + let mut catalog = PointerCatalog::new(); + for identity in 1..=64 { + catalog + .insert_xorb( + format!("{identity:064x}"), + XorbCatalogEntry::new( + 80, + "d".repeat(64), + (0..2048) + .map(|_| XorbChunkEntry::new("c".repeat(64), 7)) + .collect(), + ), + ) + .unwrap(); + } + let (_, expected) = Descriptor::build(&catalog).unwrap(); + let (pages, store, _) = fixture(&catalog).await; + drop(catalog); + let maximum = 2 * 1024 * 1024; + assert!( + pages + .full(&store, &mut PageWork::new(maximum, 0)) + .await + .is_err() + ); + let body = pages.descriptor.encode().unwrap(); + let pages = CatalogPages::new( + object_store::path::Path::from("catalog"), + pages.descriptor.clone(), + *blake3::hash(&body).as_bytes(), + body.len() as u64, + ) + .unwrap(); + let actual = pages.inspected_objects(&store, maximum).await.unwrap(); + let expected = expected + .into_iter() + .map(|(hash, body)| (pages.pages.path(&hash), body.len() as u64)) + .collect::>(); + assert_eq!(actual.into_iter().collect::>(), expected); +} + +#[tokio::test] +async fn streamed_inspection_rejects_resigned_broken_dependency_edges() { + for scenario in [ + "file-shard", + "shard-xorb", + "shard-data", + "empty-shard", + "extra-placement", + "absent-placement-xorb", + "later-occurrence", + "placement-size", + "late-recipe", + "empty-xorb", + "empty-chunk", + "missing-recipe-page", + "shared-routing", + ] { + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + "9".repeat(64), + XorbCatalogEntry::new( + 80, + "d".repeat(64), + vec![ + XorbChunkEntry::new("c".repeat(64), 7), + XorbChunkEntry::new("c".repeat(64), 7), + ], + ), + ) + .unwrap(); + catalog + .insert_shard( + "a".repeat(64), + ShardCatalogEntry::new(1, vec!["9".repeat(64)]), + ) + .unwrap(); + catalog + .insert_file("f".repeat(64), FileCatalogEntry::new(14, "a".repeat(64))) + .unwrap(); + if scenario == "shared-routing" { + catalog + .insert_xorb("1".repeat(64), catalog.xorbs()[&"9".repeat(64)].clone()) + .unwrap(); + } + let (pages, store, _) = fixture(&catalog).await; + let mut descriptor = pages.descriptor.clone(); + let mut work = PageWork::new(1024 * 1024, 0); + let xorb_values = pages + .rows( + &store, + Kind::CatalogXorbs, + &[hash(&"9".repeat(64)).unwrap().to_vec()], + &mut work, + ) + .await + .unwrap(); + let mut xorb: XorbRow = from_json(xorb_values[0].as_ref().unwrap()).unwrap(); + let recipe_path = pages.pages.path(&xorb.chunks.hash); + let mut kind = Kind::CatalogChunks; + let mut key = hash(&"c".repeat(64)).unwrap().to_vec(); + let mut value = hash(&"9".repeat(64)).unwrap().to_vec(); + value.extend_from_slice(&0u32.to_be_bytes()); + value.extend_from_slice(&7u32.to_be_bytes()); + match scenario { + "file-shard" => { + kind = Kind::CatalogFiles; + key = hash(&"f".repeat(64)).unwrap().to_vec(); + value = 14u64.to_be_bytes().to_vec(); + value.extend_from_slice(&hash(&"b".repeat(64)).unwrap()); + } + "shard-xorb" | "shard-data" | "empty-shard" => { + let nested_key = (if scenario == "shard-xorb" { "b" } else { "9" }).repeat(64); + let nested_value = if scenario == "shard-data" { + vec![1] + } else { + Vec::new() + }; + let nested = pages + .pages + .edit( + &store, + None, + Kind::CatalogShardXorbs, + vec![(hash(&nested_key).unwrap().to_vec(), nested_value)], + &mut work, + ) + .await + .unwrap(); + kind = Kind::CatalogShards; + key = hash(&"a".repeat(64)).unwrap().to_vec(); + value = json(&ShardRow { + encoded_size: u64::from(scenario != "empty-shard"), + xorbs: nested, + }) + .unwrap(); + } + "extra-placement" => key = hash(&"b".repeat(64)).unwrap().to_vec(), + "absent-placement-xorb" => value[..32].copy_from_slice(&hash(&"1".repeat(64)).unwrap()), + "later-occurrence" => value[32..36].copy_from_slice(&1u32.to_be_bytes()), + "placement-size" => value[36..].copy_from_slice(&9u32.to_be_bytes()), + "late-recipe" | "empty-xorb" | "empty-chunk" | "shared-routing" => { + if scenario == "empty-xorb" { + xorb.encoded_size = 0; + } else if scenario == "shared-routing" { + xorb.chunks.count = 3; + xorb.chunks.last = 2u32.to_be_bytes().to_vec(); + } else { + let mut chunk = hash(&"c".repeat(64)).unwrap().to_vec(); + chunk.extend_from_slice(&u32::from(scenario != "empty-chunk").to_be_bytes()); + xorb.chunks = pages + .pages + .edit( + &store, + None, + Kind::CatalogXorbChunks, + vec![( + u32::from(scenario == "late-recipe").to_be_bytes().to_vec(), + chunk, + )], + &mut work, + ) + .await + .unwrap() + .unwrap(); + } + kind = Kind::CatalogXorbs; + key = hash(&"9".repeat(64)).unwrap().to_vec(); + value = json(&xorb).unwrap(); + } + "missing-recipe-page" => (), + _ => unreachable!(), + } + if scenario != "missing-recipe-page" { + let previous = match kind { + Kind::CatalogFiles => descriptor.files.as_ref(), + Kind::CatalogShards => descriptor.shards.as_ref(), + Kind::CatalogXorbs => descriptor.xorbs.as_ref(), + Kind::CatalogChunks => descriptor.chunks.as_ref(), + _ => unreachable!(), + }; + let next = pages + .pages + .edit(&store, previous, kind, vec![(key, value)], &mut work) + .await + .unwrap(); + match kind { + Kind::CatalogFiles => descriptor.files = next, + Kind::CatalogShards => descriptor.shards = next, + Kind::CatalogXorbs => descriptor.xorbs = next, + Kind::CatalogChunks => descriptor.chunks = next, + _ => unreachable!(), + } + } + let root = descriptor.encode().unwrap(); + let identity = *blake3::hash(&root).as_bytes(); + store + .put_if_absent_verified(&pages.pages.path(&identity), root.clone()) + .await + .unwrap(); + if scenario == "missing-recipe-page" { + store.delete(&recipe_path).await.unwrap(); + } + // A fresh reader must authenticate the forged closure, not reuse fixture nodes. + let forged = CatalogPages::new( + object_store::path::Path::from("catalog"), + descriptor, + identity, + root.len() as u64, + ) + .unwrap(); + assert!( + forged + .full(&store, &mut PageWork::new(1024 * 1024, 0)) + .await + .is_err(), + "full: {scenario}" + ); + assert!( + forged.inspected_objects(&store, 1024 * 1024).await.is_err(), + "stream: {scenario}" + ); + } +} + +#[tokio::test] +async fn streamed_inspection_bounds_io_for_distinct_recipes_beyond_the_page_cache() { + let mut catalog = PointerCatalog::new(); + for identity in 0..1000u32 { + let chunks = (0..128u32) + .map(|index| { + let mut input = identity.to_be_bytes().to_vec(); + input.extend_from_slice(&index.to_be_bytes()); + XorbChunkEntry::new(blake3::hash(&input).to_hex().to_string(), 7) + }) + .collect(); + catalog + .insert_xorb( + format!("{identity:064x}"), + XorbCatalogEntry::new(80, "d".repeat(64), chunks), + ) + .unwrap(); + } + let (pages, store, observations) = fixture(&catalog).await; + drop(catalog); + let maximum = 64 * 1024 * 1024; + drop( + pages + .full(&store, &mut PageWork::new(maximum, 0)) + .await + .unwrap(), + ); + let full = observations + .0 + .lock() + .unwrap() + .iter() + .map(|op| op.bytes_read) + .sum::(); + observations.0.lock().unwrap().clear(); + let body = pages.descriptor.encode().unwrap(); + let cold = CatalogPages::new( + object_store::path::Path::from("catalog"), + pages.descriptor.clone(), + *blake3::hash(&body).as_bytes(), + body.len() as u64, + ) + .unwrap(); + cold.inspected_objects(&store, maximum).await.unwrap(); + let streamed = observations + .0 + .lock() + .unwrap() + .iter() + .map(|op| op.bytes_read) + .sum::(); + assert!( + observations + .0 + .lock() + .unwrap() + .iter() + .all(|op| op.bytes_written == 0) + ); + eprintln!("catalog_maintenance_io full_bytes={full} streamed_bytes={streamed}"); + assert!( + streamed <= 4 * full, + "streaming must not trade catalog-sized memory for repeated cold joins: {streamed} versus {full}" + ); +} diff --git a/crates/crab-metadata/src/pages/object/store.rs b/crates/crab-metadata/src/pages/object/store.rs index 0b7891f7c..de59eb21d 100644 --- a/crates/crab-metadata/src/pages/object/store.rs +++ b/crates/crab-metadata/src/pages/object/store.rs @@ -2,7 +2,7 @@ use super::*; use crate::pages::PageWork; use crab_storage::Store; use futures_util::future::BoxFuture; -use futures_util::stream::{BoxStream, StreamExt as _, TryStreamExt as _}; +use futures_util::stream::{StreamExt as _, TryStreamExt as _}; use object_store::path::Path; use std::collections::VecDeque; use std::sync::{Arc, Mutex}; @@ -163,14 +163,15 @@ impl ObjectPages { for (reference, _) in &pending { work.admit_object(reference.hash, u64::from(reference.bytes))?; } - let nodes = futures_util::stream::iter(std::mem::take(&mut pending)) + let mut nodes = futures_util::stream::iter(std::mem::take(&mut pending)) .map(|(reference, positions)| async move { Ok::<_, MetadataError>((self.fetch(store, &reference, kind).await?, positions)) }) - .buffer_unordered(4) - .try_collect::>() - .await?; - for (node, positions) in nodes { + .buffer_unordered(4); + // Drain each decoded page before retaining the next wave; a wide + // batch must not hold every cold leaf outside the bounded cache. + while let Some(result) = nodes.next().await { + let (node, positions) = result?; match node.as_ref() { Node::Leaf(rows) => { for index in positions { @@ -205,30 +206,21 @@ impl ObjectPages { Ok(output) } - pub(crate) fn walk<'a>( - &'a self, - store: &'a Store, - root: Option<&Reference>, + pub(crate) async fn next_page( + &self, + store: &Store, + pending: &mut Vec, kind: Kind, - work: &'a mut PageWork, - ) -> BoxStream<'a, Result<(Reference, Arc)>> { - let pending = root.cloned().into_iter().collect::>(); - futures_util::stream::try_unfold((pending, work), move |(mut pending, work)| async move { - let Some(reference) = pending.pop() else { - return Ok(None); - }; - let node = self.node(store, &reference, kind, work).await?; - match node.as_ref() { - // Count each logical visit, including repeated nested roots. The - // second copy admits row clones and their typed catalog values. - Node::Leaf(_) => work.admit_materialized( - (resident_bytes(&reference, &node) as u64).saturating_mul(2), - )?, - Node::Branch(children) => pending.extend(children.iter().rev().cloned()), - } - Ok(Some(((reference, node), (pending, work)))) - }) - .boxed() + work: &mut PageWork, + ) -> Result)>> { + let Some(reference) = pending.pop() else { + return Ok(None); + }; + let node = self.node(store, &reference, kind, work).await?; + if let Node::Branch(children) = node.as_ref() { + pending.extend(children.iter().rev().cloned()); + } + Ok(Some((reference, node))) } pub(crate) async fn scan( @@ -238,14 +230,19 @@ impl ObjectPages { kind: Kind, work: &mut PageWork, ) -> Result { - self.walk(store, root, kind, work) - .try_fold(Vec::new(), |mut rows, (_, node)| async move { - if let Node::Leaf(leaf) = node.as_ref() { - rows.extend(leaf.iter().cloned()); - } - Ok(rows) - }) - .await + let mut pending = root.cloned().into_iter().collect(); + let mut rows = Vec::new(); + while let Some((reference, node)) = self.next_page(store, &mut pending, kind, work).await? { + if let Node::Leaf(leaf) = node.as_ref() { + // Materializing callers retain row clones and typed values. + // Streaming visitors retain only a bounded page, not the catalog. + work.admit_materialized( + (resident_bytes(&reference, &node) as u64).saturating_mul(2), + )?; + rows.extend(leaf.iter().cloned()); + } + } + Ok(rows) } pub(crate) async fn edit( diff --git a/crates/crab-metadata/src/pages/object/tests.rs b/crates/crab-metadata/src/pages/object/tests.rs index f7d80ebcf..9fe417ec4 100644 --- a/crates/crab-metadata/src/pages/object/tests.rs +++ b/crates/crab-metadata/src/pages/object/tests.rs @@ -92,7 +92,6 @@ fn object_directory_rejects_overlapping_children_and_false_cardinality() { mod storage { use super::*; use crab_storage::{ImmutableWriteVerification, StorageObservation, StorageObserver, Store}; - use futures_util::TryStreamExt as _; use std::sync::{Arc, Mutex}; #[derive(Default)] @@ -307,20 +306,17 @@ mod storage { expected.into_iter().collect::() ); let mut work = PageWork::new(8 * 1024 * 1024, 0); - let nodes = pages - .walk(&store, root.as_ref(), Kind::Children, &mut work) - .try_collect::>() + let mut pending = root.into_iter().collect(); + let mut count = 0; + while let Some((_, node)) = pages + .next_page(&store, &mut pending, Kind::Children, &mut work) .await - .unwrap(); - assert_eq!( - nodes - .iter() - .filter_map(|(_, n)| match n.as_ref() { - Node::Leaf(rows) => Some(rows.len()), - _ => None, - }) - .sum::(), - 8000 - ); + .unwrap() + { + if let Node::Leaf(rows) = node.as_ref() { + count += rows.len(); + } + } + assert_eq!(count, 8000); } } diff --git a/crates/crab-storage/AGENTS.md b/crates/crab-storage/AGENTS.md index 5cbc2f959..0af308c91 100644 --- a/crates/crab-storage/AGENTS.md +++ b/crates/crab-storage/AGENTS.md @@ -27,6 +27,7 @@ dependency source before asserting provider behavior. | CAS behavior | `crates/crab-storage/src/cas.rs` | `crates/crab-metadata/src/manifest_store.rs` | | Provider or target identity | `crates/crab-storage/src/provider_store.rs` | `crates/crab-auth-store/src/lib.rs` | | Object key layout | `crates/crab-storage/src/layout.rs` | `crates/crab-cache/src/path_class.rs` | +| Private local spills | `crates/crab-storage/src/scratch.rs` | `crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs` | ## Invariants @@ -50,6 +51,11 @@ dependency source before asserting provider behavior. - Logical bucket identity and credential-free transport target identity serve different consumers; do not derive the latter from display text. Source: `crates/crab-storage/src/identity.rs`. +- Private spill writes admit live bytes first; reads verify committed length/hash. + Blocking workers pin directory ownership through cancelled futures. Await close + on normal and returned-error paths; drop defers cleanup until the last worker owner + exits. Source: `crates/crab-storage/src/scratch.rs`. + ## Features and platform Empty default; `test-support` exposes shared test helpers. S3/GCS/Azure/fs capabilities are dependency features, not crate flags. Read the locked object_store ObjectStore/PutMode/UpdateVersion contract before changing conditions or error mapping. Live provider behavior needs dedicated credentials and CI, not just in-memory tests. diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index c9726cc4d..0e0932d29 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -194,3 +194,17 @@ chain selection and retains the built store's signing, multipart and target identity capabilities. Azure logical identity retains account and container. The existing environment-selection and grant-endpoint constructors retain their caller policies; all share the provider construction implementation. + +## Private maintenance spills + +`scratch::ScratchSpace` owns bounded immutable local spill files below an existing +private temporary parent. It admits live bytes before creation and verifies exact +length, EOF and Blake3 on reads. References cannot cross workspace identities. +Catalog maintenance uses it to replace repeated remote joins with sorted local +reductions. This does not change foreground metadata reads or provider routing. + +Every blocking operation pins its directory until the worker exits, even when its +awaiting future is dropped. `close` drains those workers before removing the root; +callers retain primary validation errors while reporting cleanup failures. Drop +performs best-effort off-thread cleanup after the last owner exits. No SlateDB, +SQLite, new dependency or additional configuration is introduced by this transport. diff --git a/crates/crab-storage/src/lib.rs b/crates/crab-storage/src/lib.rs index f68dc76ef..ddeec44f3 100644 --- a/crates/crab-storage/src/lib.rs +++ b/crates/crab-storage/src/lib.rs @@ -14,6 +14,7 @@ pub mod provider_store; mod read_admission; pub use read_admission::ReadAdmission; pub mod retry; +pub mod scratch; #[cfg(test)] mod signed_read_tests; pub mod store; diff --git a/crates/crab-storage/src/scratch.rs b/crates/crab-storage/src/scratch.rs new file mode 100644 index 000000000..93a1b0c8d --- /dev/null +++ b/crates/crab-storage/src/scratch.rs @@ -0,0 +1,242 @@ +//! Private bounded spill files with cancellation-safe blocking-I/O ownership. +use crate::{Result, StorageError}; +use bytes::Bytes; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use tokio_util::task::TaskTracker; + +struct Directory { + path: PathBuf, + identity: [u8; 16], + next: AtomicU64, + live: AtomicU64, + maximum: u64, + closed: AtomicBool, +} + +/// One immutable spill file, bound to its private workspace and verified bytes. +#[derive(Clone)] +pub struct ScratchFile { + identity: [u8; 16], + slot: u64, + bytes: u64, + hash: [u8; 32], +} + +/// Caller-scoped local spill storage; operations retain ownership until their worker exits. +pub struct ScratchSpace { + directory: Arc, + tasks: TaskTracker, +} + +fn io(source: impl Into) -> StorageError { + StorageError::Io { + source: source.into(), + } +} + +fn worker(source: impl std::error::Error + Send + Sync + 'static) -> StorageError { + io(std::io::Error::other(source)) +} + +impl ScratchSpace { + /// Create a private child of an existing caller-owned directory with a live-byte ceiling. + /// + /// The parent must remain private to the caller. Await `close` on normal and + /// returned-error paths; dropping retains cleanup behind any in-flight worker. + pub async fn create(parent: &Path, maximum: u64) -> Result { + let parent = parent.to_owned(); + let handle = tokio::runtime::Handle::try_current().map_err(worker)?; + let directory = handle + .spawn_blocking(move || { + let identity = rand::random::<[u8; 16]>(); + let name = blake3::hash(&identity).to_hex().to_string(); + let path = parent.join(format!("crab-spill-{name}")); + let builder = std::fs::DirBuilder::new(); + #[cfg(unix)] + let builder = { + use std::os::unix::fs::DirBuilderExt as _; + let mut builder = builder; + builder.mode(0o700); + builder + }; + builder.create(&path).map_err(io)?; + Ok::<_, StorageError>(Arc::new(Directory { + path, + identity, + next: AtomicU64::new(0), + live: AtomicU64::new(0), + maximum, + closed: AtomicBool::new(false), + })) + }) + .await + .map_err(worker)??; + Ok(Self { + directory, + tasks: TaskTracker::new(), + }) + } + + async fn blocking( + &self, + operation: impl FnOnce(&Directory) -> Result + Send + 'static, + ) -> Result { + let directory = self.directory.clone(); + let handle = tokio::runtime::Handle::try_current().map_err(worker)?; + self.tasks + .spawn_blocking_on(move || operation(&directory), &handle) + .await + .map_err(worker)? + } + + /// Write one immutable file, admitting its bytes before creating it. + pub async fn write(&self, bytes: Bytes) -> Result { + self.blocking(move |directory| { + let size = bytes.len() as u64; + let slot = directory + .next + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + value.checked_add(1) + }) + .map_err(|_| io(std::io::Error::other("spill file count overflowed")))?; + directory + .live + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { + live.checked_add(size) + .filter(|total| *total <= directory.maximum) + }) + .map_err(|live| { + io(std::io::Error::other(format!( + "spill workspace requires {} live bytes, exceeding {} capacity", + live.saturating_add(size), + directory.maximum + ))) + })?; + let path = directory.path.join(format!("{slot:016x}")); + let mut output = match OpenOptions::new().write(true).create_new(true).open(&path) { + Ok(output) => output, + Err(source) => { + directory.live.fetch_sub(size, Ordering::AcqRel); + return Err(io(source)); + } + }; + if let Err(source) = output.write_all(&bytes) { + drop(output); + if std::fs::remove_file(&path).is_ok() { + directory.live.fetch_sub(size, Ordering::AcqRel); + } + return Err(io(source)); + } + Ok(ScratchFile { + identity: directory.identity, + slot, + bytes: size, + hash: *blake3::hash(&bytes).as_bytes(), + }) + }) + .await + } + + /// Read one admitted file, rejecting changed lengths or content before returning bytes. + pub async fn read(&self, file: &ScratchFile, maximum: u64) -> Result { + let file = file.clone(); + self.blocking(move |directory| { + let path = directory.file_path(&file)?; + let input = File::open(&path).map_err(io)?; + let size = input.metadata().map_err(io)?.len(); + if size != file.bytes || size > maximum { + return Err(corrupt( + "spill file length exceeds its commitment or read limit", + )); + } + let capacity = usize::try_from(size).map_err(worker)?; + let limit = size + .checked_add(1) + .ok_or_else(|| corrupt("spill file length overflows"))?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(capacity).map_err(worker)?; + input.take(limit).read_to_end(&mut bytes).map_err(io)?; + if bytes.len() as u64 != size || blake3::hash(&bytes).as_bytes() != &file.hash { + return Err(corrupt("spill file differs from its committed bytes")); + } + Ok(bytes.into()) + }) + .await + } + + /// Delete one owned file and release its live-byte admission. + pub async fn remove(&self, file: ScratchFile) -> Result<()> { + self.blocking(move |directory| { + std::fs::remove_file(directory.file_path(&file)?).map_err(io)?; + directory.live.fetch_sub(file.bytes, Ordering::AcqRel); + Ok(()) + }) + .await + } + + /// Drain workers before removing the private directory; preserve cleanup errors. + pub async fn close(self) -> Result<()> { + self.tasks.close(); + self.tasks.wait().await; + self.blocking(Directory::remove).await + } +} + +impl Directory { + fn file_path(&self, file: &ScratchFile) -> Result { + if file.identity != self.identity { + return Err(corrupt("spill file belongs to another workspace")); + } + Ok(self.path.join(format!("{:016x}", file.slot))) + } + + fn remove(&self) -> Result<()> { + match std::fs::remove_dir_all(&self.path) { + Ok(()) => (), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => (), + Err(source) => return Err(io(source)), + } + self.closed.store(true, Ordering::Release); + Ok(()) + } +} + +impl Drop for Directory { + fn drop(&mut self) { + if self.closed.load(Ordering::Acquire) { + return; + } + // Every blocking operation owns this directory. Cleanup cannot race a + // cancelled future's detached writer and resurrect an abandoned root. + let path = self.path.clone(); + let spawned = std::thread::Builder::new() + .name("crab-spill-cleanup".to_owned()) + .spawn(move || { + if let Err(source) = std::fs::remove_dir_all(path) + && source.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(%source, "private spill cleanup failed"); + } + }); + if let Err(source) = spawned { + tracing::warn!(%source, "private spill cleanup worker could not start"); + if let Err(source) = self.remove() { + tracing::warn!(%source, "private spill cleanup failed"); + } + } + } +} + +fn corrupt(reason: &str) -> StorageError { + StorageError::CorruptObject { + path: "private spill file".to_owned(), + reason: reason.to_owned(), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-storage/src/scratch/tests.rs b/crates/crab-storage/src/scratch/tests.rs new file mode 100644 index 000000000..d04c830d0 --- /dev/null +++ b/crates/crab-storage/src/scratch/tests.rs @@ -0,0 +1,94 @@ +use super::*; + +#[tokio::test] +async fn spill_capacity_is_released_only_after_owned_file_removal() { + let parent = tempfile::tempdir().unwrap(); + let space = ScratchSpace::create(parent.path(), 3).await.unwrap(); + let path = space.directory.path.clone(); + let file = space.write(Bytes::from_static(b"abc")).await.unwrap(); + assert_eq!( + space.read(&file, 3).await.unwrap(), + Bytes::from_static(b"abc") + ); + assert!(space.write(Bytes::from_static(b"d")).await.is_err()); + space.remove(file).await.unwrap(); + space.write(Bytes::from_static(b"xyz")).await.unwrap(); + space.close().await.unwrap(); + assert!(!path.exists()); +} + +#[tokio::test] +async fn spill_reads_reject_changed_bytes_lengths_limits_and_foreign_references() { + for scenario in ["hash", "length", "limit", "foreign"] { + let parent = tempfile::tempdir().unwrap(); + let space = ScratchSpace::create(parent.path(), 1024).await.unwrap(); + let other = ScratchSpace::create(parent.path(), 1024).await.unwrap(); + let file = space.write(Bytes::from_static(b"abc")).await.unwrap(); + let path = space.directory.file_path(&file).unwrap(); + let maximum = if scenario == "limit" { 2 } else { 1024 }; + match scenario { + "hash" => std::fs::write(path, b"xyz").unwrap(), + "length" => std::fs::write(path, b"abcd").unwrap(), + _ => (), + } + let reader = if scenario == "foreign" { + &other + } else { + &space + }; + assert!(reader.read(&file, maximum).await.is_err(), "{scenario}"); + space.close().await.unwrap(); + other.close().await.unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn cancellation_retains_directory_until_detached_writer_exits() { + for explicit_close in [false, true] { + let parent = tempfile::tempdir().unwrap(); + let space = ScratchSpace::create(parent.path(), 1024).await.unwrap(); + let path = space.directory.path.clone(); + let started = Arc::new(tokio::sync::Notify::new()); + let (release, receiver) = std::sync::mpsc::channel(); + let (done, finished) = tokio::sync::oneshot::channel(); + { + let started_job = started.clone(); + let future = space.blocking(move |directory| { + started_job.notify_one(); + receiver.recv().map_err(worker)?; + let result = std::fs::write(directory.path.join("late"), b"bytes").map_err(io); + let _ = done.send(result); + Ok(()) + }); + tokio::pin!(future); + tokio::select! { + _ = started.notified() => (), + result = &mut future => panic!("blocked writer completed early: {result:?}"), + } + } + if explicit_close { + let mut close = tokio::spawn(space.close()); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(25), &mut close) + .await + .is_err() + ); + release.send(()).unwrap(); + finished.await.unwrap().unwrap(); + close.await.unwrap().unwrap(); + } else { + drop(space); + assert!(path.exists()); + release.send(()).unwrap(); + finished.await.unwrap().unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while path.exists() { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + } + assert!(!path.exists()); + } +} From 0bbeadf565b342ba7824415a8530bd3949e9015d Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 21:49:49 -0700 Subject: [PATCH 112/127] perf(fsck): stream retained catalog validation Drain active inspectors after the first error so spill workers finish before history verification returns. Preserve historical page integrity checks after the current catalog is replaced. --- .../docs/architecture/indexed-remote-reads.md | 9 +- crab/src/cmd/fsck_store.rs | 54 ++++++--- .../cmd/fsck_store/capsule_history_tests.rs | 108 +++++++++++++++++- 3 files changed, 150 insertions(+), 21 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 163e0755c..e1c334bd1 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2519,9 +2519,12 @@ before, 13.34 after) include fixture construction and the full-scan comparison. They do not establish hosted latency or query-only RSS. Disk admission precedes spill growth; cleanup drains blocking workers. The existing temporary-directory setting controls placement, with no new config/env surface or dependency. -Materialization admission still protects callers retaining recipe copies. Full -logical-catalog APIs and maintenance consumers still using them remain separate -conversion and runtime-qualification work. +Historical fsck uses the same streaming integrity inspection before checking +pack sources, including catalogs replaced by an empty current catalog. It does +not retain complete historical recipe maps. Current fsck payload verification, +restore accounting, GC shard discovery and catalog-rewriting tools still use +full logical catalogs and remain separate conversion work. Materialization +admission continues to protect those callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before this change; none contains the capsule codec. Unshipped checkpoint formats diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index b1a3a55a2..f8aa72dd0 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -970,32 +970,52 @@ async fn verify_capsule_history( } } let mut sources = BTreeMap::new(); - let mut checkpoints = - futures_util::stream::iter(checkpoints.into_values().map(|pointer| async move { + let stop = std::sync::atomic::AtomicBool::new(false); + let mut checkpoints = futures_util::stream::iter(checkpoints.into_values()) + .take_while(|_| { + futures_util::future::ready(!stop.load(std::sync::atomic::Ordering::Relaxed)) + }) + .map(|pointer| async move { let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( layout, &pointer, u64::MAX, ) .await?; - checkpoint - .pointer_catalog_from_store(layout, u64::MAX) - .await?; + // Historical catalogs need complete integrity proof, not retained + // recipe maps. Streaming inspection also validates cross-page edges. + checkpoint.catalog_objects(layout, u64::MAX).await?; Ok::<_, CrabError>(checkpoint.sources().to_vec()) - })) + }) .buffer_unordered(16); - while let Some(checkpoint_sources) = checkpoints.try_next().await? { - for source in checkpoint_sources { - let path = match source.kind() { - crab_metadata::capsule_protocol::PackSourceKind::CapsuleRun => { - layout.capsule_path(source.object_hash()) - } - crab_metadata::capsule_protocol::PackSourceKind::PackLayer => { - layout.capsule_pack_layer_path(source.object_hash()) - } - }; - insert_history_pointer(&mut sources, path.as_ref(), source, "pack source")?; + let mut first_error = None; + while let Some(checkpoint_sources) = checkpoints.next().await { + if first_error.is_some() { + continue; } + let result = checkpoint_sources.and_then(|checkpoint_sources| { + for source in checkpoint_sources { + let path = match source.kind() { + crab_metadata::capsule_protocol::PackSourceKind::CapsuleRun => { + layout.capsule_path(source.object_hash()) + } + crab_metadata::capsule_protocol::PackSourceKind::PackLayer => { + layout.capsule_pack_layer_path(source.object_hash()) + } + }; + insert_history_pointer(&mut sources, path.as_ref(), source, "pack source")?; + } + Ok(()) + }); + if let Err(error) = result { + // Stop admission, then drain active inspections before returning. + // Dropping their futures would detach spill-worker cleanup. + stop.store(true, std::sync::atomic::Ordering::Relaxed); + first_error = Some(error); + } + } + if let Some(error) = first_error { + return Err(error); } // Stable sources can appear in many checkpoints and as retained run pointers. // Read each immutable body once, but prove both authenticated descriptions; diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index bf3fa7de8..311ecefb4 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -156,12 +156,13 @@ async fn checkpoint( root: RootSnapshot, source: PackSourceDescriptor, pointer: CapsulePointer, + catalog: PointerCatalog, ) -> RootSnapshot { let checkpoint = LayeredCheckpoint::build( root.record().root().generation(), root.record().digest(), vec![source], - PointerCatalog::new(), + catalog, None, ) .unwrap(); @@ -408,6 +409,7 @@ async fn retired_sources_share_full_body_verification_with_checkpoint_history() root, PackSourceDescriptor::from_capsule_run(&run).unwrap(), pointer, + PointerCatalog::new(), ) .await; let reads = Arc::new(SourceReads { @@ -447,6 +449,7 @@ async fn retained_checkpoints_and_run_pointers_read_shared_source_once() { root, PackSourceDescriptor::from_capsule_run(&run).unwrap(), run_pointer(&run, run.newest_base_root_digest()), + PointerCatalog::new(), ) .await; } @@ -484,6 +487,7 @@ async fn retained_checkpoints_read_shared_pack_layer_once() { root, layer.source_descriptor().unwrap(), run_pointer(&run, run.newest_base_root_digest()), + PointerCatalog::new(), ) .await; } @@ -512,6 +516,7 @@ async fn shared_source_body_corruption_is_not_hidden_by_reuse() { root, PackSourceDescriptor::from_capsule_run(&run).unwrap(), run_pointer(&run, run.newest_base_root_digest()), + PointerCatalog::new(), ) .await; } @@ -539,6 +544,7 @@ async fn source_reuse_still_validates_the_retained_run_pointer() { root, PackSourceDescriptor::from_capsule_run(&run).unwrap(), run_pointer(&run, &"e".repeat(64)), + PointerCatalog::new(), ) .await; @@ -558,6 +564,7 @@ async fn shared_sources_reject_conflicting_checkpoint_descriptors() { root, source.clone(), run_pointer(&run, run.newest_base_root_digest()), + PointerCatalog::new(), ) .await; let conflicting = PackSourceDescriptor::new( @@ -575,6 +582,7 @@ async fn shared_sources_reject_conflicting_checkpoint_descriptors() { root, conflicting, run_pointer(&run, run.newest_base_root_digest()), + PointerCatalog::new(), ) .await; @@ -584,3 +592,101 @@ async fn shared_sources_reject_conflicting_checkpoint_descriptors() { .is_err() ); } + +#[tokio::test] +async fn historical_catalog_pages_are_verified_after_catalog_is_replaced() { + use crab_metadata::capsule_protocol::{ + FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + }; + let (layout, root, run) = history_fixture().await; + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + "a".repeat(64), + XorbCatalogEntry::new( + 80, + "b".repeat(64), + vec![XorbChunkEntry::new("c".repeat(64), 7); 4], + ), + ) + .unwrap(); + catalog + .insert_shard( + "d".repeat(64), + ShardCatalogEntry::new(40, vec!["a".repeat(64)]), + ) + .unwrap(); + catalog + .insert_file("e".repeat(64), FileCatalogEntry::new(7, "d".repeat(64))) + .unwrap(); + let mut root = root; + for _ in 0..3 { + root = checkpoint( + &layout, + root, + PackSourceDescriptor::from_capsule_run(&run).unwrap(), + run_pointer(&run, run.newest_base_root_digest()), + catalog.clone(), + ) + .await; + } + let objects = layout + .store() + .list_prefix(&layout.capsule_catalog_prefix()) + .await + .unwrap(); + assert!( + objects.len() > 4, + "fixture must include nested catalog pages" + ); + // The latest catalog is empty: only authenticated history keeps these + // pages reachable, independently of current-catalog and source-body checks. + let root = checkpoint( + &layout, + root, + PackSourceDescriptor::from_capsule_run(&run).unwrap(), + run_pointer(&run, run.newest_base_root_digest()), + PointerCatalog::new(), + ) + .await; + verify_capsule_history(&layout, root.record().root()) + .await + .unwrap(); + for object in objects { + let (body, _) = layout + .store() + .get_with_etag_bounded(&object.location, object.size) + .await + .unwrap(); + layout.store().delete(&object.location).await.unwrap(); + assert!( + verify_capsule_history(&layout, root.record().root()) + .await + .is_err(), + "missing historical catalog object was accepted: {}", + object.location, + ); + let mut corrupt = body.to_vec(); + corrupt[0] ^= 1; + layout + .store() + .put(&object.location, Bytes::from(corrupt)) + .await + .unwrap(); + assert!( + verify_capsule_history(&layout, root.record().root()) + .await + .is_err(), + "corrupt historical catalog object was accepted: {}", + object.location, + ); + layout + .store() + .put_overwrite(&object.location, body) + .await + .unwrap(); + } + verify_capsule_history(&layout, root.record().root()) + .await + .unwrap(); +} From 588205128559c5f1e99fdec0ac9daddac8083694 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 22:33:52 -0700 Subject: [PATCH 113/127] fix(storage): use supported atomic update API --- crates/crab-storage/src/scratch.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/crab-storage/src/scratch.rs b/crates/crab-storage/src/scratch.rs index 93a1b0c8d..d25bd4407 100644 --- a/crates/crab-storage/src/scratch.rs +++ b/crates/crab-storage/src/scratch.rs @@ -99,13 +99,13 @@ impl ScratchSpace { let size = bytes.len() as u64; let slot = directory .next - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + .try_update(Ordering::AcqRel, Ordering::Acquire, |value| { value.checked_add(1) }) .map_err(|_| io(std::io::Error::other("spill file count overflowed")))?; directory .live - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { + .try_update(Ordering::AcqRel, Ordering::Acquire, |live| { live.checked_add(size) .filter(|total| *total <= directory.maximum) }) From a37098693e14ad80204d7572ed1a64ba85a0e09a Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 22:50:53 -0700 Subject: [PATCH 114/127] perf(metadata): page capsule chunk hints Replace capsule hint database initialization with authenticated copy-on-write pages. Preserve exact staged assertions and current-source/origin proof. Keep global GC admission through root publication and drain protected receive before fence release. --- .../docs/architecture/indexed-remote-reads.md | 52 +++ crab/src/cmd/gc/bucket.rs | 74 +++- crab/src/cmd/gc/bucket/tests/hint_tests.rs | 186 ++++++++ crab/src/cmd/gc/journal.rs | 1 + crab/src/core/error.rs | 3 +- crab/src/git/capsule_push.rs | 91 ++-- crab/src/git/push.rs | 192 +++++--- .../src/git/push/tests/capsule_chunk_tests.rs | 20 +- .../src/metadata/metadb/stores/chunk_index.rs | 85 ---- crates/crab-auth-server/AGENTS.md | 4 + crates/crab-auth-server/src/error.rs | 3 +- .../crab-auth-server/src/receive/capsule.rs | 106 ++++- .../crab-auth-server/src/receive/workflow.rs | 45 +- crates/crab-metadata/AGENTS.md | 8 + crates/crab-metadata/README.md | 9 +- .../src/capsule_protocol/chunk_candidates.rs | 11 +- .../chunk_candidates/store.rs | 418 ++++++++++++++++++ .../chunk_candidates/store/tests.rs | 212 +++++++++ .../chunk_candidates/store/tests/races.rs | 257 +++++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- crates/crab-metadata/src/error.rs | 5 + crates/crab-metadata/src/pages.rs | 2 + crates/crab-metadata/src/remote_index.rs | 96 +--- .../src/remote_index/capsule_tests.rs | 165 ------- .../src/remote_index/format_tests.rs | 46 ++ crates/crab-remote/src/prepare/capsule.rs | 12 +- crates/crab-sdk/src/remote_error.rs | 2 +- crates/crab-storage/AGENTS.md | 1 + crates/crab-storage/src/layout.rs | 14 + crates/crab-write/AGENTS.md | 4 + crates/crab-write/README.md | 6 +- crates/crab-write/src/capsule_protocol.rs | 6 +- 32 files changed, 1617 insertions(+), 523 deletions(-) create mode 100644 crab/src/cmd/gc/bucket/tests/hint_tests.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests/races.rs delete mode 100644 crates/crab-metadata/src/remote_index/capsule_tests.rs create mode 100644 crates/crab-metadata/src/remote_index/format_tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e1c334bd1..e17006324 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2534,3 +2534,55 @@ acceptance requires changed-catalog byte bounds, old/new cold reconstruction, corruption/admission tests, GC/retention coverage and a new real RustFS run. The earlier 10 GiB matrix does not qualify this new format. RustFS results do not establish representative hosted-S3 latency. + +## Capsule chunk discovery without the retired receipt database + +Capsule add and push use `CapsuleChunkIndex`, owned by `crab-metadata`. +The mutable `{global-prefix}/capsule-chunk-index` descriptor commits to an +immutable authenticated B+ tree under `capsule-chunk-pages/`. Internal routing +is paged. Current heads, immutable placement IDs and source anchors share this +one tree, so replacing a head preserves the exact assertion recorded by an +already-staged plan. Writers copy only affected paths and retry conditional root +conflicts against the newer root; concurrent publishers cannot lose each other's +records. Immutable pages are verified before the root can advertise them. + +These records remain advisory. A hit must still prove membership in a captured +current source catalog and match fresh canonical-origin identity. Index presence, +old exact IDs and cache-service observations grant no independent payload +lifetime. An absent index permits local packing. Corrupt or missing pages under +an unchanged root are errors, rather than negative hits. A read-only query whose +page disappeared may restart its entire advisory lookup only after observing a +different root, with three attempts and one aggregate byte budget. This does +not rebase a pinned repository or require write grants from managed readers. + +All three producers—CLI capsule push, shared remote preparation and protected +receive—use the same metadata writer after visibility and before releasing GC +writer admission. Cancellation before hint-root publication leaves collectible +immutable pages; it cannot advertise incomplete hints or reject a ref already +committed. Protected receive propagates GC-fence cancellation to publication, +drains its operation before releasing fences, and rechecks cancellation after +immutable preparation and before visibility. Namespace renewal loss joins that +same pre-publication check. Once publication starts, its exact outcome is awaited. + +Bucket GC authenticates the root's complete physical page closure, includes its +identity in durable plan revalidation, and retains every referenced page. It +collects only unreferenced pages under its existing grace and conditional-delete +policy. Missing current pages abort planning before any sweep. Global bucket +sweep covers the canonical `.crab` namespace; qualification of lifecycle cleanup +for managed view-local global scopes remains a release gate. + +The released v1.2.4 tag contains version-1 staged plans with opaque placement and +origin IDs. A target explicitly advertising the v1 layout therefore retains its +manifest receipt-reader path. Capsule targets and fresh repositories do not open +that database during discovery. Push resolves exact capsule IDs first; only +unresolved released staged assertions enter the v1 migration reader. This +boundary is a released local-data contract, not an error-driven fallback from a +corrupt capsule index. + +Deterministic tests cover selected edits against 100, 10,000 and 100,000 unrelated +chunks, read-only lookup concurrent with root replacement/page reclamation, +competing root creates, cancellation before root publication, and GC retention. +The fixed edit must stay below 512 KiB for both bytes read and written; its lookup +has a 256 KiB budget. These are encoded-request bounds, not latency claims. +Full-source RustFS, cache-service, protected receive and final 10 GiB acceptance +remain separate gates until their receipts pass. diff --git a/crab/src/cmd/gc/bucket.rs b/crab/src/cmd/gc/bucket.rs index 991eb75ea..cb8cb0965 100644 --- a/crab/src/cmd/gc/bucket.rs +++ b/crab/src/cmd/gc/bucket.rs @@ -35,6 +35,7 @@ use crate::core::config::GcListProfile; use crate::core::error::{CrabError, Result, check_cancelled}; use crate::storage::store::Store; use crab_metadata::ref_registry::RefRegistry; +use crab_storage::layout::CAPSULE_CHUNK_PAGE_KIND; use crab_storage::{ StorageProviderKind, StoreLayout, canonical_global_content_path, content_hash_from_path, global_content_partition_prefix, global_content_prefix, @@ -70,6 +71,7 @@ pub struct BucketGcOutcome { pub shards_deleted: u64, pub xorbs_deleted: u64, pub file_index_deleted: u64, + pub derived_index_objects_deleted: u64, pub bytes_reclaimed: u64, pub list_requests: u64, pub list_parallelism: usize, @@ -83,6 +85,7 @@ impl BucketGcOutcome { shards = self.shards_deleted, xorbs = self.xorbs_deleted, file_index = self.file_index_deleted, + derived_index_objects = self.derived_index_objects_deleted, bytes = self.bytes_reclaimed, list_requests = self.list_requests, list_parallelism = self.list_parallelism, @@ -93,6 +96,7 @@ impl BucketGcOutcome { shards = self.shards_deleted, xorbs = self.xorbs_deleted, file_index = self.file_index_deleted, + derived_index_objects = self.derived_index_objects_deleted, bytes = self.bytes_reclaimed, list_requests = self.list_requests, list_parallelism = self.list_parallelism, @@ -105,7 +109,7 @@ impl BucketGcOutcome { pub fn to_summary(&self) -> super::GcSummary { super::GcSummary { packs_deleted: 0, - derived_index_objects_deleted: 0, + derived_index_objects_deleted: self.derived_index_objects_deleted, xorbs_deleted: self.xorbs_deleted, shards_deleted: self.shards_deleted, file_index_entries_deleted: self.file_index_deleted, @@ -557,6 +561,7 @@ async fn run_bucket_gc_streaming( repo_prefixes, root_identity, capsule_shards, + hint_pages, } = roots; let snapshot_at = match resume_phase { Some(super::journal::GcRunPhase::Planning | super::journal::GcRunPhase::Deleting) => { @@ -743,7 +748,7 @@ async fn run_bucket_gc_streaming( .list_parallelism .max(closure_stats.parallelism.max(1)); - let mut closure_segment_planner = ClosureSegmentStreamingPlanner::new( + let mut closure_segment_planner = ImmutableObjectStreamingPlanner::new( DurableMarkReader::new_keys( store.clone(), journal.marks_prefix(), @@ -797,6 +802,40 @@ async fn run_bucket_gc_streaming( outcome.list_requests = outcome.list_requests.saturating_add(xorb_stats.requests); outcome.list_parallelism = outcome.list_parallelism.max(xorb_stats.parallelism.max(1)); + let mut hint_marks = DurableMarkWriter::new_keys( + store.clone(), + journal.marks_prefix(), + "referenced-capsule-hint-pages", + ); + for path in hint_pages { + hint_marks.add(&path).await?; + } + hint_marks.finish().await?; + let mut hint_planner = ImmutableObjectStreamingPlanner::new( + DurableMarkReader::new_keys( + store.clone(), + journal.marks_prefix(), + "referenced-capsule-hint-pages", + ), + coordinator_protected_keys, + cutoff, + args.force, + &mut journal, + ); + let hint_stats = scan_global_objects( + store, + CAPSULE_CHUNK_PAGE_KIND, + args.list_profile, + args.list_concurrency, + Arc::clone(&permits), + cancel, + &mut hint_planner, + ) + .await?; + hint_planner.finish().await?; + outcome.list_requests = outcome.list_requests.saturating_add(hint_stats.requests); + outcome.list_parallelism = outcome.list_parallelism.max(hint_stats.parallelism.max(1)); + if args.dry_run { let preview = async { let current_roots = bucket_root_snapshot_streaming( @@ -827,6 +866,7 @@ async fn run_bucket_gc_streaming( match object.key.split('/').nth(1) { Some("shards") => outcome.shards_deleted += 1, Some("xorbs") => outcome.xorbs_deleted += 1, + Some(CAPSULE_CHUNK_PAGE_KIND) => outcome.derived_index_objects_deleted += 1, _ => {} } outcome.bytes_reclaimed = outcome @@ -934,6 +974,7 @@ async fn execute_bucket_journal( match key.split('/').nth(1) { Some("shards") => outcome.shards_deleted += 1, Some("xorbs") => outcome.xorbs_deleted += 1, + Some(CAPSULE_CHUNK_PAGE_KIND) => outcome.derived_index_objects_deleted += 1, _ => {} } } @@ -992,6 +1033,7 @@ async fn execute_bucket_journal( match object.key.split('/').nth(1) { Some("shards") => outcome.shards_deleted += 1, Some("xorbs") => outcome.xorbs_deleted += 1, + Some(CAPSULE_CHUNK_PAGE_KIND) => outcome.derived_index_objects_deleted += 1, _ => {} } batch_bytes = match batch_bytes.checked_add(object.size) { @@ -1061,6 +1103,7 @@ struct BucketRootSnapshot { repo_prefixes: Vec, root_identity: String, capsule_shards: HashMap>, + hint_pages: Vec, } struct CapsuleRepositoryRoots { @@ -1240,10 +1283,20 @@ async fn bucket_root_snapshot_streaming( for key in coordinator_protected_keys { digest.add("coordinator-protected", key); } + let (hint_root, hint_pages) = crab_metadata::capsule_protocol::CapsuleChunkIndex::new(&router) + .objects(&storage, u64::MAX) + .await?; + if let Some(root) = hint_root { + digest.add("capsule-chunk-index", &root); + } Ok(BucketRootSnapshot { repo_prefixes, root_identity: digest.finish(), capsule_shards, + hint_pages: hint_pages + .into_iter() + .map(|(path, _)| path.to_string()) + .collect(), }) } @@ -2237,24 +2290,24 @@ impl GlobalObjectConsumer for ClosureStreamingPlanner<'_> { } } -struct ClosureSegmentStreamingPlanner<'a> { - live_segments: DurableMarkReader, +struct ImmutableObjectStreamingPlanner<'a> { + referenced: DurableMarkReader, coordinator_protected_keys: &'a HashSet, cutoff: SystemTime, force: bool, sink: CandidateBatchSink<'a>, } -impl<'a> ClosureSegmentStreamingPlanner<'a> { +impl<'a> ImmutableObjectStreamingPlanner<'a> { fn new( - live_segments: DurableMarkReader, + referenced: DurableMarkReader, coordinator_protected_keys: &'a HashSet, cutoff: SystemTime, force: bool, journal: &'a mut super::journal::GcRunJournal, ) -> Self { Self { - live_segments, + referenced, coordinator_protected_keys, cutoff, force, @@ -2268,10 +2321,10 @@ impl<'a> ClosureSegmentStreamingPlanner<'a> { } #[async_trait(?Send)] -impl GlobalObjectConsumer for ClosureSegmentStreamingPlanner<'_> { +impl GlobalObjectConsumer for ImmutableObjectStreamingPlanner<'_> { async fn consume(&mut self, object: ListedObject) -> Result<()> { if self.coordinator_protected_keys.contains(&object.location) - || self.live_segments.contains(&object.location).await? + || self.referenced.contains(&object.location).await? || (!self.force && object.last_modified >= self.cutoff) { return Ok(()); @@ -3020,7 +3073,7 @@ mod tests { .unwrap(); journal.set_root_identity("root").await.unwrap(); let protected = HashSet::new(); - let mut planner = ClosureSegmentStreamingPlanner::new( + let mut planner = ImmutableObjectStreamingPlanner::new( DurableMarkReader::new_keys( store.clone(), journal.marks_prefix(), @@ -3906,4 +3959,5 @@ mod tests { "workflow experiment stage-refs blob was touched by bucket GC", ); } + mod hint_tests; } diff --git a/crab/src/cmd/gc/bucket/tests/hint_tests.rs b/crab/src/cmd/gc/bucket/tests/hint_tests.rs new file mode 100644 index 000000000..e4fde8559 --- /dev/null +++ b/crab/src/cmd/gc/bucket/tests/hint_tests.rs @@ -0,0 +1,186 @@ +use super::*; +use crab_metadata::capsule_protocol::{ + CapsuleChunkIndex, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + write_capsule_chunk_candidates, +}; +use crab_metadata::receipts::OriginReceipt; +use std::collections::BTreeSet; + +async fn publish_hints(store: &Store, repo: &str) -> (MerkleHash, [u8; 32]) { + let layout = crab_storage::StoreLayout::new(store.as_storage().clone(), repo.to_owned()); + let xorb = MerkleHash::from([3; 4]); + let chunk = MerkleHash::from([1; 4]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 80, + "4".repeat(64), + vec![XorbChunkEntry::new(chunk.hex(), 7)], + ), + ) + .unwrap(); + catalog + .insert_shard("2".repeat(64), ShardCatalogEntry::new(10, vec![xorb.hex()])) + .unwrap(); + let origin = OriginReceipt::new( + "canonical-origin".to_owned(), + layout.xorb_path(&xorb).to_string(), + xorb.into(), + [5; 32], + 80, + Some("etag".to_owned()), + None, + ); + let cancel = CancellationToken::new(); + let fence = crate::maintenance::GcGlobalWriterLease::acquire(store, GLOBAL_PREFIX, &cancel) + .await + .unwrap(); + let result = write_capsule_chunk_candidates( + &layout, + &catalog, + &HashMap::from([(xorb, origin)]), + &|| cancel.is_cancelled(), + ) + .await; + fence.release().await.unwrap(); + result.unwrap(); + let candidates = CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &[chunk], 1024 * 1024) + .await + .unwrap(); + ( + chunk, + candidates.placements[&<[u8; 32]>::from(chunk)].placement_id(), + ) +} + +fn args(force: bool) -> BucketGcArgs { + BucketGcArgs { + bucket: "isolated-hint-test".to_owned(), + dry_run: false, + grace_period: Duration::from_secs(3600), + force, + yes: true, + list_concurrency: 4, + list_profile: GcListProfile::Adaptive, + delete_concurrency: 4, + resume_run_id: None, + } +} + +#[tokio::test] +async fn hint_gc_preserves_live_pages_and_honors_grace() { + for force in [false, true] { + let store = memory_store(); + seed_registry(&store, &RefRegistry::default()).await; + let (hash, old_id) = publish_hints(&store, "first").await; + publish_hints(&store, "second").await; + let layout = crab_storage::StoreLayout::new(store.as_storage().clone(), String::new()); + let (_, live) = CapsuleChunkIndex::new(&layout) + .objects(layout.store(), 1024 * 1024) + .await + .unwrap(); + let live = live + .into_iter() + .map(|(path, _)| path.to_string()) + .collect::>(); + let orphan = Bytes::from_static(b"unpublished hint page"); + let identity = blake3::hash(&orphan).to_hex().to_string(); + store + .put( + &canonical_global_content_path(CAPSULE_CHUNK_PAGE_KIND, &identity), + orphan, + ) + .await + .unwrap(); + let before = store + .list_prefix(&layout.capsule_chunk_pages_prefix()) + .await + .unwrap() + .into_iter() + .map(|meta| meta.location.to_string()) + .collect::>(); + assert!(before.len() > live.len()); + let outcome = run_bucket_gc( + &args(force), + &store, + &HashSet::new(), + &HashSet::new(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let after = store + .list_prefix(&layout.capsule_chunk_pages_prefix()) + .await + .unwrap() + .into_iter() + .map(|meta| meta.location.to_string()) + .collect::>(); + assert_eq!(after, if force { live } else { before.clone() }); + assert_eq!( + outcome.derived_index_objects_deleted, + (before.len() - after.len()) as u64 + ); + assert_eq!( + outcome.to_summary().derived_index_objects_deleted, + outcome.derived_index_objects_deleted + ); + let exact = CapsuleChunkIndex::new(&layout) + .candidates_by_id(layout.store(), &[(hash, old_id)], 1024 * 1024) + .await + .unwrap(); + assert_eq!( + exact.placements[&<[u8; 32]>::from(hash)].placement_id(), + old_id + ); + } +} + +#[tokio::test] +async fn missing_hint_page_aborts_sweep_before_payload_deletion() { + let store = memory_store(); + seed_registry(&store, &RefRegistry::default()).await; + publish_hints(&store, "first").await; + let layout = crab_storage::StoreLayout::new(store.as_storage().clone(), String::new()); + let (_, pages) = CapsuleChunkIndex::new(&layout) + .objects(layout.store(), 1024 * 1024) + .await + .unwrap(); + store.delete(&pages[0].0).await.unwrap(); + let orphan = canonical_global_content_path("xorbs", &"a".repeat(64)); + store + .put(&orphan, Bytes::from_static(b"unreferenced payload")) + .await + .unwrap(); + assert!( + run_bucket_gc( + &args(true), + &store, + &HashSet::new(), + &HashSet::new(), + &CancellationToken::new() + ) + .await + .is_err() + ); + assert!(store.get_with_etag(&orphan).await.is_ok()); +} + +#[tokio::test] +async fn hint_root_changes_invalidate_bucket_root_identity() { + let store = memory_store(); + seed_registry(&store, &RefRegistry::default()).await; + let registry = load_ref_registry(&store, false).await.unwrap(); + publish_hints(&store, "first").await; + let before = bucket_root_snapshot_streaming(&store, ®istry, 4, &HashSet::new()) + .await + .unwrap(); + publish_hints(&store, "second").await; + let after = bucket_root_snapshot_streaming(&store, ®istry, 4, &HashSet::new()) + .await + .unwrap(); + assert_ne!(before.root_identity, after.root_identity); +} diff --git a/crab/src/cmd/gc/journal.rs b/crab/src/cmd/gc/journal.rs index 9fd8554c6..514433e6b 100644 --- a/crab/src/cmd/gc/journal.rs +++ b/crab/src/cmd/gc/journal.rs @@ -833,6 +833,7 @@ fn validate_candidate_key(state: &GcRunState, key: &str) -> Result<()> { ".crab/xorbs/", ".crab/gc/closures/", ".crab/gc/closure-segments/", + ".crab/capsule-chunk-pages/", ] .iter() .any(|prefix| key.starts_with(prefix)), diff --git a/crab/src/core/error.rs b/crab/src/core/error.rs index 7f443d104..e42e5d258 100644 --- a/crab/src/core/error.rs +++ b/crab/src/core/error.rs @@ -2038,7 +2038,8 @@ impl From for CrabError { impl From for CrabError { fn from(error: crab_metadata::error::MetadataError) -> Self { match error { - crab_metadata::error::MetadataError::RefJournalCancelled => Self::Cancelled, + crab_metadata::error::MetadataError::RefJournalCancelled + | crab_metadata::error::MetadataError::CapsuleHintCancelled => Self::Cancelled, crab_metadata::error::MetadataError::FileLookupLimit { resource, maximum } => { Self::Protocol(format!("file lookup exceeds {resource} limit ({maximum})")) } diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index fc58d03c3..8b3b8d27e 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -534,6 +534,28 @@ async fn run_inner( } } .await; + // Root visibility precedes advisory discovery, while global admission + // still pins immutable hint pages until their own root becomes durable. + let hint_indexed = if publication.as_ref().is_ok_and(Option::is_some) + && layout.store().staging_write_prefix().is_none() + { + match crab_metadata::capsule_protocol::write_capsule_chunk_candidates( + &layout, + &pointer_delta, + &xet_stats.origins, + &|| cancel.is_cancelled(), + ) + .await + { + Ok(()) => true, + Err(error) => { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + false + } + } + } else { + false + }; let release = match gc_writer { Some(writer) => writer.release().await, None => Ok(()), @@ -550,55 +572,26 @@ async fn run_inner( } }; if let Some(active_active_commit) = committed { - if layout.store().staging_write_prefix().is_none() { - let mut index_config = - crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( - layout.repo_prefix(), - layout.global_prefix(), - ); - if let Some(path) = config.metadb.chunk_index.db.path.as_ref() { - index_config.chunk_index_path.clone_from(path); - } - // The ref is already committed. Index failures only remove acceleration; - // reporting an uncommitted push here would invite a duplicate publication. - match crab_metadata::remote_index::write_capsule_chunk_candidates( - Arc::clone(layout.store().inner()), - &index_config, - layout.repo_prefix(), - &pointer_delta, - &xet_stats.origins, - ) - .await - { - Ok(()) => { - let cache_path = crate::cache::add_remote_candidate_cache_path( - &crate::cache::default_cache_root(), - &store.bucket_identity(), - layout.global_prefix(), - ); - if cache_path.is_file() { - let hashes = pointer_delta - .xorbs() - .values() - .flat_map(|entry| entry.chunks()) - .filter_map(|chunk| MerkleHash::from_hex(chunk.hash()).ok()) - .collect::>(); - let invalidation = tokio::task::spawn_blocking(move || { - crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( - &cache_path, - ) - .and_then(|cache| cache.forget_negatives(&hashes)) - }) - .await; - if !matches!(invalidation, Ok(Ok(()))) { - tracing::warn!( - "capsule committed; candidate cache invalidation failed" - ); - } - } - } - Err(error) => { - tracing::warn!(%error, "capsule committed; chunk candidate index needs repair") + if hint_indexed { + let cache_path = crate::cache::add_remote_candidate_cache_path( + &crate::cache::default_cache_root(), + &store.bucket_identity(), + layout.global_prefix(), + ); + if cache_path.is_file() { + let hashes = pointer_delta + .xorbs() + .values() + .flat_map(|entry| entry.chunks()) + .filter_map(|chunk| MerkleHash::from_hex(chunk.hash()).ok()) + .collect::>(); + let invalidation = tokio::task::spawn_blocking(move || { + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open(&cache_path) + .and_then(|cache| cache.forget_negatives(&hashes)) + }) + .await; + if !matches!(invalidation, Ok(Ok(()))) { + tracing::warn!("capsule committed; candidate cache invalidation failed"); } } } diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index 2a2381029..651b2fc2f 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -4265,10 +4265,15 @@ pub struct PushPipeline { cas_dependency_replans: std::sync::atomic::AtomicU64, } +enum AddChunkIndex { + Capsule(crab_metadata::capsule_protocol::CapsuleChunkIndex), + Manifest(crate::metadata::ChunkIndexStore), +} + /// Shared add-time classifier backed by the push pipeline's full proof path. pub(crate) struct AddRemoteChunkClassifier { pipeline: PushPipeline, - remote_chunk_index: tokio::sync::OnceCell>, + remote_chunk_index: tokio::sync::OnceCell>, candidate_cache: Option>, candidate_cache_hits: std::sync::atomic::AtomicU64, candidate_cache_misses: std::sync::atomic::AtomicU64, @@ -4464,12 +4469,52 @@ impl crab_staging::push_plan::ExistingChunkLookup for AddRemoteChunkClassifier { let chunk_index = self .remote_chunk_index .get_or_try_init(|| async { + let Some(store) = self.pipeline.store.as_ref() else { + return Ok(None); + }; + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + self.pipeline.router.repo_prefix().to_owned(), + self.pipeline.router.global_prefix().to_owned(), + ); + // The published v1.2.4 add contract remains attached to + // an explicit v1 layout. Capsule targets and fresh repos + // must not discover or initialize its receipt database. + let manifest = match store.head(&layout.capsule_root_path()).await { + Ok(_) => false, + Err(CrabError::NotFound { .. }) => { + match crab_metadata::layout_descriptor::read_canonical_layout( + layout.store(), + &layout, + ) + .await + { + Ok(_) => true, + Err(crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::NotFound { .. }, + }) => false, + Err(error) => return Err(error.into()), + } + } + Err(error) => return Err(error), + }; + if !manifest { + let index = + crab_metadata::capsule_protocol::CapsuleChunkIndex::new(&layout); + return Ok(index + .available( + layout.store(), + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await? + .then_some(AddChunkIndex::Capsule(index))); + } let guard = self.pipeline.metadb.lock().await; let Some(guard) = guard.as_ref() else { return Ok(None); }; match guard.chunk_index().await { - Ok(store) => Ok(Some(store)), + Ok(store) => Ok(Some(AddChunkIndex::Manifest(store))), Err(error) if error.is_metadb_read_only_uninitialized() => Ok(None), Err(error) => Err(error), } @@ -4481,7 +4526,7 @@ impl crab_staging::push_plan::ExistingChunkLookup for AddRemoteChunkClassifier { // the index later; transient open failures are still retried. if let Some(chunk_index) = chunk_index { self.pipeline - .lookup_proven_remote_chunks_for_add(&misses, chunk_index.clone()) + .lookup_proven_remote_chunks_for_add(&misses, chunk_index) .await } else { Ok(HashMap::new()) @@ -4490,9 +4535,7 @@ impl crab_staging::push_plan::ExistingChunkLookup for AddRemoteChunkClassifier { .await .map_err(|error| match error { CrabError::Cancelled => crab_staging::StagingError::Cancelled, - error => crab_staging::StagingError::Internal(format!( - "remote add classifier failed: {error}" - )), + error => crab_staging::StagingError::Io(std::io::Error::other(error)), })?; // Keep index absence in the session's OnceCell: persisting one miss // per chunk makes cache writes scale with the entire new file. @@ -10375,18 +10418,8 @@ impl PushPipeline { if planned.is_empty() { return Ok(VerifiedRemoteCandidates::default()); } - let chunk_store = { - let guard = self.metadb.lock().await; - let Some(guard) = guard.as_ref() else { - return Ok(VerifiedRemoteCandidates::default()); - }; - match guard.chunk_index().await { - Ok(store) => store, - Err(error) if error.is_metadb_read_only_uninitialized() => { - return Ok(VerifiedRemoteCandidates::default()); - } - Err(error) => return Err(error), - } + let Some(store) = self.store.as_ref() else { + return Ok(VerifiedRemoteCandidates::default()); }; let exact_candidates = planned .iter() @@ -10395,9 +10428,19 @@ impl PushPipeline { let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; let (capsule, capsule_indexed, capsule_matched) = tokio::time::timeout_at(deadline, async { - let mut candidates = chunk_store - .capsule_candidates_by_id(&exact_candidates) - .await?; + let candidates_layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + self.router.repo_prefix().to_owned(), + self.router.global_prefix().to_owned(), + ); + let mut candidates = + crab_metadata::capsule_protocol::CapsuleChunkIndex::new(&candidates_layout) + .candidates_by_id( + store.as_storage(), + &exact_candidates, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await?; let indexed = candidates.placements.len(); candidates.placements.retain(|hash, placement| { planned @@ -10431,6 +10474,36 @@ impl PushPipeline { .into_iter() .filter(|(hash, _)| !capsule.candidates.contains_key(hash)) .collect::>(); + if exact_candidates.is_empty() { + debug!( + planned = planned.len(), + indexed = capsule_indexed, + matched = capsule_matched, + generation_verified = 0, + capsule_source_verified = capsule.candidates.len(), + payload_verified = capsule.candidates.len(), + placement_mismatches = 0, + origin_proof_mismatches = 0, + "revalidated staged remote placement proofs" + ); + return Ok(capsule); + } + // Released v1.2.4 plans store opaque placement IDs. Only unresolved + // staged IDs need that migration proof; ordinary capsule hits must + // never initialize the old manifest-receipt database. + let chunk_store = { + let guard = self.metadb.lock().await; + let Some(guard) = guard.as_ref() else { + return Ok(capsule); + }; + match guard.chunk_index().await { + Ok(store) => store, + Err(error) if error.is_metadb_read_only_uninitialized() => { + return Ok(capsule); + } + Err(error) => return Err(error), + } + }; let remote_candidates = tokio::time::timeout_at( deadline, chunk_store.get_committed_candidates_by_id_batch(&exact_candidates), @@ -15071,9 +15144,18 @@ impl PushPipeline { async fn lookup_capsule_chunks_for_add( &self, hashes: &[MerkleHash], - chunk_store: &crate::metadata::ChunkIndexStore, + index: &crab_metadata::capsule_protocol::CapsuleChunkIndex, ) -> Result> { - let candidates = chunk_store.capsule_candidates(hashes).await?; + let Some(store) = self.store.as_ref() else { + return Ok(HashMap::new()); + }; + let candidates = index + .candidates( + store.as_storage(), + hashes, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await?; self.verify_capsule_chunk_candidates(candidates) .await .map(|verified| verified.candidates) @@ -15238,7 +15320,7 @@ impl PushPipeline { async fn lookup_proven_remote_chunks_for_add( &self, chunk_hashes: &[MerkleHash], - chunk_store: crate::metadata::ChunkIndexStore, + index: &AddChunkIndex, ) -> Result> { if chunk_hashes.is_empty() { return Ok(HashMap::new()); @@ -15254,23 +15336,20 @@ impl PushPipeline { )), }) }; - let mut capsule = tokio::time::timeout_at( - deadline, - self.lookup_capsule_chunks_for_add(chunk_hashes, &chunk_store), - ) - .await - .map_err(|_| timeout_error())??; - let remaining = chunk_hashes - .iter() - .copied() - .filter(|hash| !capsule.contains_key(hash)) - .collect::>(); - if remaining.is_empty() { - return Ok(capsule); - } + let chunk_store = match index { + AddChunkIndex::Capsule(index) => { + return tokio::time::timeout_at( + deadline, + self.lookup_capsule_chunks_for_add(chunk_hashes, index), + ) + .await + .map_err(|_| timeout_error())?; + } + AddChunkIndex::Manifest(store) => store, + }; let remote_candidates = tokio::time::timeout_at( deadline, - chunk_store.get_committed_candidates_batch(&remaining), + chunk_store.get_committed_candidates_batch(chunk_hashes), ) .await .map_err(|_| timeout_error())??; @@ -15302,18 +15381,20 @@ impl PushPipeline { let verified = self .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins, false) .await?; - capsule.extend(verified.into_iter().filter_map(|(chunk_hash, xorb_ref)| { - let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; - (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( - chunk_hash, - crab_staging::push_plan::ExistingChunkCandidate { - xorb_ref, - placement_id, - origin_proof_id, - }, - )) - })); - Ok(capsule) + Ok(verified + .into_iter() + .filter_map(|(chunk_hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; + (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( + chunk_hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + }) + .collect()) } async fn lookup_cache_service_chunk_refs( @@ -22364,7 +22445,7 @@ mod tests { mod capsule_chunk_tests; - fn add_classifier_with_private_cache( + async fn add_classifier_with_private_cache( store: &Store, router: &StoreLayout, cache_root: &std::path::Path, @@ -22372,6 +22453,9 @@ mod tests { AddRemoteChunkClassifier, Arc, ) { + crate::core::remote_layout::initialize(store, router) + .await + .expect("released v1 layout"); let cache_root = cache_root.join("cache"); crab_cache::ensure_private_cache_directory(&cache_root).expect("private cache root"); let candidate_cache = Arc::new( @@ -22417,7 +22501,7 @@ mod tests { let store = Store::new(recording_store); let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); let (classifier, candidate_cache) = - add_classifier_with_private_cache(&store, &router, cache_root.path()); + add_classifier_with_private_cache(&store, &router, cache_root.path()).await; let first = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( &classifier, &[(MerkleHash::from([1; 32]), 4)], @@ -22478,7 +22562,7 @@ mod tests { seed.chunk_index_system_keys().await.expect("seed manifest"); seed.close().await.expect("close seed writer"); let (classifier, candidate_cache) = - add_classifier_with_private_cache(&store, &router, cache_root.path()); + add_classifier_with_private_cache(&store, &router, cache_root.path()).await; let hash = MerkleHash::from([4; 32]); let lookup = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( &classifier, diff --git a/crab/src/git/push/tests/capsule_chunk_tests.rs b/crab/src/git/push/tests/capsule_chunk_tests.rs index 0f60b530e..45de25887 100644 --- a/crab/src/git/push/tests/capsule_chunk_tests.rs +++ b/crab/src/git/push/tests/capsule_chunk_tests.rs @@ -225,12 +225,11 @@ async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { origin.proven_at_unix_secs += 10; origins.insert(candidate.xorb_ref.xorb_hash, origin); } - crab_metadata::remote_index::write_capsule_chunk_candidates( - Arc::clone(layout.store().inner()), - &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), - "repos/source", + crab_metadata::capsule_protocol::write_capsule_chunk_candidates( + &layout, &catalog, &origins, + &|| false, ) .await .unwrap(); @@ -434,23 +433,20 @@ async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { ), ); } - crab_metadata::remote_index::write_capsule_chunk_candidates( - Arc::clone(layout.store().inner()), - &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), - "repos/source", + crab_metadata::capsule_protocol::write_capsule_chunk_candidates( + &layout, &catalog, &fresh_origins, + &|| false, ) .await .unwrap(); - let guard = build_push_metadb_guard(&store, &router, None, &metadata, true); - let chunk_store = guard.chunk_index().await.unwrap(); + let index = crab_metadata::capsule_protocol::CapsuleChunkIndex::new(&layout); let hashes = planned.keys().copied().collect::>(); let fresh = classifier .pipeline - .lookup_capsule_chunks_for_add(&hashes, &chunk_store) + .lookup_capsule_chunks_for_add(&hashes, &index) .await; - guard.close().await.unwrap(); let fresh = fresh.unwrap(); assert_eq!( fresh.len(), diff --git a/crab/src/metadata/metadb/stores/chunk_index.rs b/crab/src/metadata/metadb/stores/chunk_index.rs index 0b1d91563..341d53898 100644 --- a/crab/src/metadata/metadb/stores/chunk_index.rs +++ b/crab/src/metadata/metadb/stores/chunk_index.rs @@ -104,91 +104,6 @@ impl ChunkIndexStore { self } - /// Load capsule hints without interpreting them as released manifest receipts. - pub(crate) async fn capsule_candidates( - &self, - hashes: &[MerkleHash], - ) -> Result { - use crab_metadata::capsule_protocol::capsule_chunk_head_key; - let requests = hashes - .iter() - .map(|hash| (*hash, capsule_chunk_head_key(&(*hash).into()), None)) - .collect::>(); - self.capsule_candidate_records(&requests).await - } - - /// Read immutable capsule placements selected during add, independent of newer heads. - pub(crate) async fn capsule_candidates_by_id( - &self, - candidates: &[(MerkleHash, [u8; 32])], - ) -> Result { - let requests = candidates - .iter() - .map(|(hash, id)| { - ( - *hash, - crab_metadata::key_codec::capsule_chunk_placement_key(id), - Some(*id), - ) - }) - .collect::>(); - self.capsule_candidate_records(&requests).await - } - - async fn capsule_candidate_records( - &self, - requests: &[(MerkleHash, [u8; 33], Option<[u8; 32]>)], - ) -> Result { - use crab_metadata::capsule_protocol::{ - CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, - }; - let mut output = CapsuleChunkCandidates::default(); - // Stay below Db's dense range-scan threshold: unrelated chunk history - // must not turn an add-time point batch into a scan of the global index. - for requests in requests.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { - let keys = requests - .iter() - .map(|(_, key, _)| Bytes::copy_from_slice(key)) - .collect::>(); - let values = self.db.get_batch(&keys).await?; - let mut anchor_ids = HashSet::new(); - for ((hash, _, expected_id), value) in requests.iter().zip(values) { - let Some(value) = value else { - continue; - }; - let placement = CommittedChunkPlacement::decode(&value)?; - if placement.chunk_hash != <[u8; 32]>::from(*hash) - || expected_id.is_some_and(|id| placement.placement_id() != id) - { - return Err(CrabError::StagingCorrupt( - "capsule candidate hash differs from index key".to_owned(), - )); - } - anchor_ids.insert(placement.source_anchor_id); - output.placements.insert(placement.chunk_hash, placement); - } - let ids = anchor_ids - .into_iter() - .filter(|id| !output.anchors.contains_key(id)) - .collect::>(); - let keys = ids - .iter() - .map(|id| Bytes::copy_from_slice(&capsule_chunk_anchor_key(id))) - .collect::>(); - let values = self.db.get_batch(&keys).await?; - for (id, value) in ids.into_iter().zip(values) { - let value = value.ok_or_else(|| { - CrabError::StagingCorrupt( - "capsule candidate source anchor is absent".to_owned(), - ) - })?; - let anchor = CapsuleChunkAnchor::decode(&id, &value)?; - output.anchors.insert(id, anchor); - } - } - Ok(output) - } - /// Look up a single chunk hash through all three tiers. /// /// Cache-first: in-memory → persistent, when available → remote. diff --git a/crates/crab-auth-server/AGENTS.md b/crates/crab-auth-server/AGENTS.md index 6fec1d634..ad69faa28 100644 --- a/crates/crab-auth-server/AGENTS.md +++ b/crates/crab-auth-server/AGENTS.md @@ -34,6 +34,10 @@ operations in `crates/crab-auth-server/src/receive/git_workspace.rs`. Source: `crates/crab-auth-server/src/receive.rs`. - Read-path authorization and denied paths must remain part of view construction; do not reuse an unscoped view merely because its source generation matches. Source: `crates/crab-auth-server/src/view.rs`. +- Protected commit drains its operation before releasing GC fences. Propagate + the fence token into capsule namespace/preparation work, and check again before + visibility. Once committed, a fence-release error is repair work, not rejection. + Source: `crates/crab-auth-server/src/receive/workflow.rs` and `receive/capsule.rs`. - Operation-owned Git workspace cleanup and worker draining are part of error/cancellation handling, not just the success path. Inspect workspace creation and callers together. Source: `crates/crab-auth-server/src/receive/git_workspace.rs`. diff --git a/crates/crab-auth-server/src/error.rs b/crates/crab-auth-server/src/error.rs index c41f9faf1..4108061a7 100644 --- a/crates/crab-auth-server/src/error.rs +++ b/crates/crab-auth-server/src/error.rs @@ -149,7 +149,8 @@ impl From for AuthServerError { impl From for AuthServerError { fn from(error: crab_metadata::error::MetadataError) -> Self { match error { - error @ crab_metadata::error::MetadataError::RefJournalCancelled => { + error @ (crab_metadata::error::MetadataError::RefJournalCancelled + | crab_metadata::error::MetadataError::CapsuleHintCancelled) => { Self::Io(io::Error::other(error)) } error @ (crab_metadata::error::MetadataError::FileLookupAdmission { .. } diff --git a/crates/crab-auth-server/src/receive/capsule.rs b/crates/crab-auth-server/src/receive/capsule.rs index 1a6a613eb..80ef56da2 100644 --- a/crates/crab-auth-server/src/receive/capsule.rs +++ b/crates/crab-auth-server/src/receive/capsule.rs @@ -11,6 +11,7 @@ use super::capsule_dependencies::{ verify_pointer_dependencies, }; use super::git_workspace::{materialize_capsule_source_push, verify_capsule_git_candidate}; +use super::workflow::check_receive_cancelled; use super::{ ActiveActiveReceiveConfig, ProtectedCapsulePushPlan, PushPrepareRecord, ReceiveContext, active_active_coordinator_registration, conflict, invalid, promote_staged_writes, @@ -37,6 +38,7 @@ pub(super) async fn commit_capsule_candidate( replication_objects: &[String], cancel: &tokio_util::sync::CancellationToken, ) -> Result> { + check_receive_cancelled(cancel)?; let view = open_candidate_view(ctx).await?; if plan.ref_updates.iter().all(|update| { view.refs().get(&update.ref_name) == Some(&update.new_oid) @@ -58,7 +60,6 @@ pub(super) async fn commit_capsule_candidate( plan.run_size, ); return commit_active_active_candidate( - ctx.store(), ctx.router(), ctx.repo_prefix(), active_active, @@ -66,6 +67,7 @@ pub(super) async fn commit_capsule_candidate( &transaction, replication_objects, None, + cancel, ) .await .map(Some); @@ -90,7 +92,6 @@ pub(super) async fn commit_capsule_candidate( run.bytes().len() as u64, ); return commit_active_active_candidate( - ctx.store(), ctx.router(), ctx.repo_prefix(), active_active, @@ -98,6 +99,7 @@ pub(super) async fn commit_capsule_candidate( &transaction, replication_objects, None, + cancel, ) .await .map(Some); @@ -167,7 +169,6 @@ pub(super) async fn commit_capsule_candidate( ) .await?; publish_verified_candidate( - layout.store(), &layout, &repo_prefix, base, @@ -175,6 +176,7 @@ pub(super) async fn commit_capsule_candidate( &capsule, active_active.as_ref(), &replication_objects, + &scoped, ) .await }, @@ -182,7 +184,6 @@ pub(super) async fn commit_capsule_candidate( .await } else { publish_verified_candidate( - ctx.store(), ctx.router(), ctx.repo_prefix(), base, @@ -190,31 +191,25 @@ pub(super) async fn commit_capsule_candidate( &capsule, active_active, replication_objects, + cancel, ) .await }; - if result.is_ok() { - let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( - ctx.repo_prefix(), - ctx.router().global_prefix(), - ); - if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( - std::sync::Arc::clone(ctx.store().inner()), - &config, - ctx.repo_prefix(), + if result.is_ok() + && let Err(error) = crab_metadata::capsule_protocol::write_capsule_chunk_candidates( + ctx.router(), &delta, &origins, + &|| cancel.is_cancelled(), ) .await - { - tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); - } + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); } result } async fn publish_verified_candidate( - store: &crab_storage::Store, layout: &crab_storage::StoreLayout, repo_prefix: &str, base: crab_metadata::capsule_protocol::RootSnapshot, @@ -222,9 +217,17 @@ async fn publish_verified_candidate( capsule: &Capsule, active_active: Option<&ActiveActiveReceiveConfig>, replication_objects: &[String], + cancel: &tokio_util::sync::CancellationToken, ) -> Result> { + check_receive_cancelled(cancel)?; let Some(active_active) = active_active else { - crab_write::capsule_protocol::publish(layout, base, transaction, capsule).await?; + let prepared = + crab_write::capsule_protocol::prepare_publication(layout, base, transaction, capsule) + .await?; + // Immutable preparation may outlive fence/namespace renewal. Check + // again before visibility, then await its exact commit outcome. + check_receive_cancelled(cancel)?; + crab_write::capsule_protocol::publish_prepared(layout, prepared).await?; return Ok(None); }; let prepared = crab_write::capsule_protocol::prepare_coordinated_publication( @@ -236,7 +239,6 @@ async fn publish_verified_candidate( .await?; let descriptor = prepared.descriptor().clone(); commit_active_active_candidate( - store, layout, repo_prefix, active_active, @@ -244,13 +246,13 @@ async fn publish_verified_candidate( transaction, replication_objects, Some(prepared), + cancel, ) .await .map(Some) } async fn commit_active_active_candidate( - store: &crab_storage::Store, layout: &crab_storage::StoreLayout, repo_prefix: &str, active_active: &ActiveActiveReceiveConfig, @@ -258,6 +260,7 @@ async fn commit_active_active_candidate( transaction: &crab_metadata::capsule_protocol::CapsuleTransaction, replication_objects: &[String], prepared: Option, + cancel: &tokio_util::sync::CancellationToken, ) -> Result { let refs = transaction .edits() @@ -280,7 +283,7 @@ async fn commit_active_active_candidate( )?; let registration = active_active_coordinator_registration(&active_active.replication)?; crab_metadata::ref_registry::register_active_active_coordinator_for_repo( - store, + layout.store(), layout, registration, ) @@ -290,6 +293,7 @@ async fn commit_active_active_candidate( repo_prefix, ) .await?; + check_receive_cancelled(cancel)?; let mut outcome = crab_coordination::write_coordinator::commit_uploaded_push_refs( coordinator.as_ref(), plan.request.clone(), @@ -648,6 +652,7 @@ mod tests { }; use crab_types::pointer::Pointer; use crab_xet::hash::MerkleHash; + use std::sync::Arc; use super::*; use crate::git_pointer_scan::ReachablePointerScan; @@ -660,6 +665,65 @@ mod tests { std::iter::repeat_n(ch, 64).collect() } + struct CancelAfterWrite(tokio_util::sync::CancellationToken); + + impl crab_storage::StorageObserver for CancelAfterWrite { + fn started(&self, _: crab_storage::StorageOperation) {} + + fn finished(&self, observation: crab_storage::StorageObservation) { + if observation.operation == crab_storage::StorageOperation::Put + && observation.outcome == crab_storage::StorageOutcome::Success + { + self.0.cancel(); + } + } + } + + #[tokio::test] + async fn cancelled_immutable_preparation_cannot_publish_a_ref() -> Result<()> { + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())); + let layout = crab_storage::StoreLayout::new(store, "repo".to_owned()); + let base = crab_write::capsule_protocol::initialize(&layout, &hash('a'), "refs/heads/main") + .await?; + let transaction = CapsuleTransaction::new( + base.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(oid('1')), + None, + )], + )?; + let capsule = Capsule::build(&transaction, Vec::new(), Vec::new())?; + let cancel = tokio_util::sync::CancellationToken::new(); + let observed = layout + .store() + .clone() + .with_storage_observer(Arc::new(CancelAfterWrite(cancel.clone()))); + let observed_layout = crab_storage::StoreLayout::new(observed.clone(), "repo".to_owned()); + let result = publish_verified_candidate( + &observed_layout, + "repo", + base, + &transaction, + &capsule, + None, + &[], + &cancel, + ) + .await; + assert!( + matches!(result, Err(crate::error::AuthServerError::Write(error)) + if matches!(*error, crab_write::WriteError::Cancelled)) + ); + assert!( + crab_read::capsule_protocol::read_visible_refs(&layout) + .await? + .is_empty() + ); + Ok(()) + } + #[test] fn protected_plan_binds_exact_capsule_transaction() { let base = hash('a'); diff --git a/crates/crab-auth-server/src/receive/workflow.rs b/crates/crab-auth-server/src/receive/workflow.rs index e25927e46..5e9c87034 100644 --- a/crates/crab-auth-server/src/receive/workflow.rs +++ b/crates/crab-auth-server/src/receive/workflow.rs @@ -355,17 +355,34 @@ pub async fn commit_receive( ctx.validate_layout().await?; let operation_cancel = CancellationToken::new(); let fences = ReceiveWriterFences::acquire(ctx, &operation_cancel).await?; - let operation = tokio::select! { - result = commit_receive_inner(ctx, repo_url, plan_digest, active_active_json) => result, - _ = operation_cancel.cancelled() => { - Err(conflict("GC writer fence lease lost during protected receive")) - } - }; + // The operation owns ref leases and possibly in-flight publication. Fence + // loss requests cancellation, but cleanup and commit resolution must drain + // before either GC fence is released. + let operation = commit_receive_inner( + ctx, + repo_url, + plan_digest, + active_active_json, + &operation_cancel, + ) + .await; let release = fences.release().await; match (operation, release) { - (Ok(response), Ok(())) => Ok(response), - (Err(error), _) | (Ok(_), Err(error)) => Err(error), + (Ok(response), release) => { + if let Err(error) = release { + tracing::warn!(%error, "protected receive committed; fence cleanup requires repair"); + } + Ok(response) + } + (Err(error), _) => Err(error), + } +} + +pub(super) fn check_receive_cancelled(cancel: &CancellationToken) -> Result<()> { + if cancel.is_cancelled() { + return Err(crab_write::WriteError::Cancelled.into()); } + Ok(()) } async fn commit_receive_inner( @@ -373,14 +390,16 @@ async fn commit_receive_inner( repo_url: &str, plan_digest: &str, active_active_json: Option<&str>, + cancel: &CancellationToken, ) -> Result { + check_receive_cancelled(cancel)?; let active_active = parse_active_active_receive_config(active_active_json, repo_url)?; match ctx.read_plan_document().await? { ProtectedReceivePlan::Manifest(plan) => { - commit_manifest_receive(ctx, *plan, plan_digest, active_active.as_ref()).await + commit_manifest_receive(ctx, *plan, plan_digest, active_active.as_ref(), cancel).await } ProtectedReceivePlan::Capsule(plan) => { - commit_capsule_receive(ctx, plan, plan_digest, active_active.as_ref()).await + commit_capsule_receive(ctx, plan, plan_digest, active_active.as_ref(), cancel).await } } } @@ -390,18 +409,18 @@ async fn commit_capsule_receive( plan: super::ProtectedCapsulePushPlan, plan_digest: &str, active_active: Option<&super::ActiveActiveReceiveConfig>, + cancel: &CancellationToken, ) -> Result { super::validate_protected_capsule_plan_shape(&plan, ctx.repo_prefix(), ctx.push_id())?; let prepare = ctx.read_prepare_record().await?; let evidence = read_verified_capsule_receive_evidence(ctx, plan_digest, &plan, &prepare).await?; - let cancel = CancellationToken::new(); let outcome = commit_capsule_candidate( ctx, &plan, active_active, &evidence.replication_objects, - &cancel, + cancel, ) .await?; Ok(PushFinalizeResponse::updated_with_commit_outcome( @@ -415,6 +434,7 @@ async fn commit_manifest_receive( plan: super::ProtectedPushPlan, plan_digest: &str, active_active: Option<&super::ActiveActiveReceiveConfig>, + cancel: &CancellationToken, ) -> Result { validate_push_plan_shape(&plan, ctx.repo_prefix(), ctx.push_id())?; if plan.mirror_plan_id.is_some() && active_active.is_some() { @@ -498,6 +518,7 @@ async fn commit_manifest_receive( gc_registry_generation, ) .await?; + check_receive_cancelled(cancel)?; let active_active_commit = commit_receive_manifest( ctx.store(), ctx.router(), diff --git a/crates/crab-metadata/AGENTS.md b/crates/crab-metadata/AGENTS.md index 6466acad8..60a459b12 100644 --- a/crates/crab-metadata/AGENTS.md +++ b/crates/crab-metadata/AGENTS.md @@ -26,6 +26,7 @@ Trace one path: `crates/crab-write/src/journal.rs` → `commit_ref_transaction` | Manifest/journal change | `crates/crab-metadata/src/manifest_store.rs` | `crates/crab-write/src/journal.rs` | | Catalog coverage | `crates/crab-metadata/src/git_object_locator` | `crates/crab-remote-git/src/repository.rs` | | Local dedup index | `crates/crab-metadata/src/persistent_chunk_index.rs` | `crates/crab-staging/src/index.rs` | +| Capsule chunk hints | `crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs` | CLI/shared/protected capsule publishers and bucket GC | ## Invariants @@ -43,6 +44,13 @@ Trace one path: `crates/crab-write/src/journal.rs` → `commit_ref_transaction` - Keep explicit close ownership when adding index readers/writers; inspect error and cancellation paths as well as successful reads. Source: `crates/crab-metadata/src/git_object_locator`. +- Capsule chunk hints are advisory. `CapsuleChunkIndex` authenticates bounded + pages; callers still verify current source membership and fresh origin identity. + Writers retain global GC admission through verified pages and conditional root + publication. Read-only discovery can recapture a changed advisory root after a + missing page; unchanged-root absence/corruption remains an error. + Source: `crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs`. + ## Features and platform Empty default. `storage` adds object-store helpers; `file-index-reader` and `remote-index` also enable storage and SlateDB paths; `local-index` adds SQLite. Inspect locked SlateDB session/writer close contracts before changing lifecycle. Payload-only checks must remain a separate slice. diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 49439536c..736c39d59 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -37,7 +37,14 @@ the semantically validated Git state to a BLAKE3 digest; readers call `validate_manifest_payload` before trusting refs or index pointers. Capsule protocol v2 uses one compacted root plus independently mutable ref -heads and immutable capsule runs. Pointer catalogs keep shard/xorb payloads +heads and immutable capsule runs. +`capsule_protocol::CapsuleChunkIndex` reads paged global advisory chunk heads +and exact placement IDs without opening the released manifest receipt database. +`write_capsule_chunk_candidates` conditionally publishes verified immutable pages +after ref visibility, under caller-held GC writer admission. Readers need no +write grant; hints still require current source-catalog and canonical-origin +proof. `CapsuleChunkIndex::objects` authenticates the physical closure for GC; +it does not make hints authoritative payload roots. Pointer catalogs keep shard/xorb payloads external while authenticating their identities and reconstruction closure. Readers that already loaded and verified a root use `load_pointer_catalog_from_root`; this preserves the same catalog validation diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs index 7ba9ebd52..37d8c7389 100644 --- a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs @@ -1,5 +1,5 @@ //! Rebuildable chunk hints whose source authority is a capsule catalog. -#[cfg(any(test, feature = "remote-index"))] +#[cfg(any(test, feature = "storage"))] use std::collections::BTreeMap; use std::collections::HashMap; @@ -7,13 +7,18 @@ use serde::{Deserialize, Serialize}; use super::XorbCatalogEntry; use crate::error::{MetadataError, Result}; -#[cfg(any(test, feature = "remote-index"))] +#[cfg(any(test, feature = "storage"))] use crate::receipts::RECEIPT_SCHEMA_VERSION; use crate::receipts::{CommittedChunkPlacement, OriginReceipt}; use crab_xet::hash::MerkleHash; pub use crate::key_codec::{capsule_chunk_anchor_key, capsule_chunk_head_key}; +#[cfg(feature = "storage")] +mod store; +#[cfg(feature = "storage")] +pub use store::{CapsuleChunkIndex, write_capsule_chunk_candidates}; + /// Capsule source and verified physical origin for one xorb. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -114,7 +119,7 @@ pub struct CapsuleChunkCandidates { pub anchors: HashMap<[u8; 32], CapsuleChunkAnchor>, } -#[cfg(any(test, feature = "remote-index"))] +#[cfg(any(test, feature = "storage"))] /// Encode one bounded xorb's candidates after full verification and ref publication. pub(crate) fn capsule_chunk_candidate_rows( repo_prefix: &str, diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs new file mode 100644 index 000000000..2c1bc0b28 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs @@ -0,0 +1,418 @@ +//! Immutable paged capsule hints under one conditional global root. +use super::*; +use crate::pages::object::{Node, ObjectPages, Reference}; +use crate::pages::{Kind, PageWork, Rows}; +use bytes::Bytes; +use crab_storage::{StorageError, Store, StoreLayout}; +use object_store::path::Path; +use std::collections::{BTreeMap, BTreeSet}; + +const ROOT_BYTES: u64 = 4096; +const READ_ATTEMPTS: usize = 3; +const WRITE_ATTEMPTS: usize = crab_storage::cas::DEFAULT_MAX_ATTEMPTS as usize; +const KIND: Kind = Kind::CapsuleChunkHints; +type Request = (MerkleHash, [u8; 33], Option<[u8; 32]>); + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Root { + version: u32, + records: Reference, +} + +/// Bounded global capsule-hint access without legacy database initialization. +/// +/// Each lookup captures a current root. Hints do not authorize payload reuse; +/// callers still prove current source membership and exact physical origin. +#[derive(Debug)] +pub struct CapsuleChunkIndex { + root: Path, + pages: ObjectPages, +} + +impl CapsuleChunkIndex { + /// Bind page identities to one caller-selected global scope. + #[must_use] + pub fn new(layout: &StoreLayout) -> Self { + Self { + root: layout.capsule_chunk_index_path(), + pages: ObjectPages::new(layout.capsule_chunk_pages_prefix()), + } + } + + /// Report an advertised root without loading its page dependencies. + pub async fn available(&self, store: &Store, maximum_bytes: u64) -> Result { + Ok(self + .load(store, &mut PageWork::new(maximum_bytes, 0)) + .await? + .is_some()) + } + + /// Read current chunk heads and their immutable source anchors. + pub async fn candidates( + &self, + store: &Store, + hashes: &[MerkleHash], + maximum_bytes: u64, + ) -> Result { + admit_requests(hashes.len(), maximum_bytes)?; + let requests = hashes + .iter() + .map(|hash| (*hash, capsule_chunk_head_key(&(*hash).into()), None)) + .collect::>(); + self.read(store, &requests, maximum_bytes).await + } + + /// Read exact add-time placements independently of later head replacement. + pub async fn candidates_by_id( + &self, + store: &Store, + candidates: &[(MerkleHash, [u8; 32])], + maximum_bytes: u64, + ) -> Result { + admit_requests(candidates.len(), maximum_bytes)?; + let requests = candidates + .iter() + .map(|(hash, id)| { + ( + *hash, + crate::key_codec::capsule_chunk_placement_key(id), + Some(*id), + ) + }) + .collect::>(); + self.read(store, &requests, maximum_bytes).await + } + + async fn read( + &self, + store: &Store, + requests: &[Request], + maximum: u64, + ) -> Result { + if requests.is_empty() { + return Ok(CapsuleChunkCandidates::default()); + } + let mut work = PageWork::new(maximum, 0); + for attempt in 0..READ_ATTEMPTS { + let Some((root, _)) = self.load(store, &mut work).await? else { + return Ok(CapsuleChunkCandidates::default()); + }; + match self + .records(store, &root.records, requests, &mut work) + .await + { + Err( + error @ MetadataError::Storage { + source: StorageError::NotFound { .. }, + }, + ) => { + let current = self.load(store, &mut work).await?; + if current + .as_ref() + .is_none_or(|(current, _)| current.records == root.records) + { + return Err(error); + } + // These heads are floating advisory discovery, not a pinned + // repository view. A superseded root can lose old pages to GC; + // restart the entire query without requiring writes from readers. + if attempt + 1 == READ_ATTEMPTS { + return Err(StorageError::StateConflict { + path: self.root.to_string(), + } + .into()); + } + } + result => return result, + } + } + Err(StorageError::StateConflict { + path: self.root.to_string(), + } + .into()) + } + + async fn records( + &self, + store: &Store, + root: &Reference, + requests: &[Request], + work: &mut PageWork, + ) -> Result { + let keys = requests + .iter() + .map(|(_, key, _)| key.to_vec()) + .collect::>(); + let values = self + .pages + .lookup(store, Some(root), KIND, &keys, work) + .await?; + let mut output = CapsuleChunkCandidates::default(); + let mut anchors = BTreeSet::new(); + for ((hash, _, expected), value) in requests.iter().zip(values) { + let Some(value) = value else { + continue; + }; + let placement = CommittedChunkPlacement::decode(&value)?; + if placement.chunk_hash != <[u8; 32]>::from(*hash) + || expected.is_some_and(|id| placement.placement_id() != id) + { + return Err(corrupt( + "capsule placement differs from its requested identity", + )); + } + if let Some(previous) = output + .placements + .insert(placement.chunk_hash, placement.clone()) + && previous != placement + { + return Err(corrupt( + "one query requests conflicting placements for a chunk", + )); + } + anchors.insert(placement.source_anchor_id); + } + let keys = anchors + .iter() + .map(|id| capsule_chunk_anchor_key(id).to_vec()) + .collect::>(); + let values = self + .pages + .lookup(store, Some(root), KIND, &keys, work) + .await?; + for (id, value) in anchors.into_iter().zip(values) { + let value = + value.ok_or_else(|| corrupt("capsule placement source anchor is absent"))?; + output + .anchors + .insert(id, CapsuleChunkAnchor::decode(&id, &value)?); + } + Ok(output) + } + + async fn load( + &self, + store: &Store, + work: &mut PageWork, + ) -> Result> { + let body = match store + .get_with_etag_bounded(&self.root, ROOT_BYTES.min(work.maximum)) + .await + { + Ok(body) => body, + Err(StorageError::NotFound { .. }) => return Ok(None), + Err(error) => return Err(error.into()), + }; + work.admit_object(*blake3::hash(&body.0).as_bytes(), body.0.len() as u64)?; + let root: Root = serde_json::from_slice(&body.0) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if root.version != 1 { + return Err(corrupt("capsule hint root version is unsupported")); + } + root.records.validate(KIND)?; + Ok(Some((root, body.1))) + } + + /// Authenticate a current root and every immutable page before maintenance. + /// + /// The caller holds global sweep admission through marking and deletion. + /// The root digest participates in the caller's plan revalidation. + pub async fn objects( + &self, + store: &Store, + maximum_bytes: u64, + ) -> Result<(Option, Vec<(Path, u64)>)> { + let mut work = PageWork::new(maximum_bytes, 0); + let Some((root, _)) = self.load(store, &mut work).await? else { + return Ok((None, Vec::new())); + }; + let mut pending = vec![root.records.clone()]; + let mut objects = BTreeMap::new(); + while let Some((reference, node)) = self + .pages + .next_page(store, &mut pending, KIND, &mut work) + .await? + { + if let Node::Leaf(rows) = node.as_ref() { + for (key, value) in rows { + validate_row(key, value)?; + } + } + objects.insert(self.pages.path(&reference.hash), u64::from(reference.bytes)); + } + let body = serde_json::to_vec(&root) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + Ok(( + Some(blake3::hash(&body).to_hex().to_string()), + objects.into_iter().collect(), + )) + } + + async fn publish( + &self, + store: &Store, + rows: Rows, + maximum: u64, + cancelled: &(dyn Fn() -> bool + Sync), + ) -> Result<()> { + if rows.is_empty() { + return Ok(()); + } + let mut work = PageWork::new(maximum, 0); + let keys = rows.iter().map(|(key, _)| key.clone()).collect::>(); + for attempt in 0..WRITE_ATTEMPTS { + if cancelled() { + return Err(MetadataError::CapsuleHintCancelled); + } + let snapshot = self.load(store, &mut work).await?; + let root = snapshot.as_ref().map(|(root, _)| &root.records); + let previous = self + .pages + .lookup(store, root, KIND, &keys, &mut work) + .await?; + for ((key, incoming), previous) in rows.iter().zip(previous) { + if let Some(previous) = previous { + validate_row(key, &previous)?; + if key[0] != 0x06 && previous != *incoming { + return Err(corrupt( + "immutable capsule hint identity has conflicting bytes", + )); + } + } + } + let Some(records) = self + .pages + .edit(store, root, KIND, rows.clone(), &mut work) + .await? + else { + return Err(corrupt("nonempty capsule hints produced no root")); + }; + if root == Some(&records) { + return Ok(()); + } + let body = Bytes::from( + serde_json::to_vec(&Root { + version: 1, + records, + }) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?, + ); + if body.len() as u64 > ROOT_BYTES { + return Err(corrupt("capsule hint root exceeds its byte bound")); + } + // Every edited immutable page is read-back verified by ObjectPages + // before the conditional root transition can advertise it. + if cancelled() { + return Err(MetadataError::CapsuleHintCancelled); + } + let result = match snapshot { + Some((_, etag)) => store.update(&self.root, body, etag).await.map(|_| ()), + None => store.create_strict(&self.root, body).await, + }; + match result { + Ok(()) => return Ok(()), + Err(StorageError::StateConflict { .. }) if attempt + 1 < WRITE_ATTEMPTS => {} + Err(error) => return Err(error.into()), + } + } + Err(StorageError::StateConflict { + path: self.root.to_string(), + } + .into()) + } +} + +fn admit_requests(count: usize, maximum: u64) -> Result<()> { + if (count as u64) + .checked_mul(std::mem::size_of::() as u64) + .is_none_or(|bytes| bytes > maximum) + { + return Err(corrupt("capsule hint query exceeds its request byte bound")); + } + Ok(()) +} + +fn validate_row(key: &[u8], value: &[u8]) -> Result<()> { + let identity: [u8; 32] = key + .get(1..) + .and_then(|key| key.try_into().ok()) + .ok_or_else(|| corrupt("capsule hint key length differs"))?; + match key.first() { + Some(0x07) => { + CapsuleChunkAnchor::decode(&identity, value)?; + } + Some(0x06 | 0x08) => { + let placement = CommittedChunkPlacement::decode(value)?; + let expected = if key[0] == 0x06 { + placement.chunk_hash + } else { + placement.placement_id() + }; + if expected != identity { + return Err(corrupt("capsule hint placement key differs")); + } + } + _ => return Err(corrupt("capsule hint key class is unsupported")), + } + Ok(()) +} + +/// Publish rebuildable hints only after ref visibility, under global writer admission. +/// +/// Existing exact placement IDs remain discoverable after head replacement. +/// A failure cannot revoke a Git publication that the caller already committed. +pub async fn write_capsule_chunk_candidates( + layout: &StoreLayout, + catalog: &super::super::PointerCatalog, + origins: &HashMap, + cancelled: &(dyn Fn() -> bool + Sync), +) -> Result<()> { + if cancelled() { + return Err(MetadataError::CapsuleHintCancelled); + } + let maximum = super::super::MAX_CATALOG_BYTES; + super::super::pointer::admit_catalog_rows(std::slice::from_ref(catalog), maximum)?; + let mut source_shards = BTreeMap::new(); + for (shard_hash, shard) in catalog.shards() { + for xorb in shard.xorb_hashes() { + source_shards + .entry(xorb.as_str()) + .or_insert(shard_hash.as_str()); + } + } + let mut rows = BTreeMap::new(); + let mut materialized = 0_u64; + for (hash, entry) in catalog.xorbs().iter().rev() { + let parsed = + MerkleHash::from_hex(hash).map_err(|_| corrupt("invalid capsule xorb hash"))?; + let Some(origin) = origins.get(&parsed) else { + continue; + }; + let source = source_shards + .get(hash.as_str()) + .ok_or_else(|| corrupt("capsule xorb candidate has no source shard root"))?; + let incoming = + capsule_chunk_candidate_rows(layout.repo_prefix(), source, parsed, entry, origin)?; + for (key, value) in incoming { + materialized = materialized + .checked_add((key.len() + value.len() + 96) as u64) + .ok_or_else(|| corrupt("capsule hint publication byte count overflows"))?; + if materialized > maximum { + return Err(corrupt("capsule hint publication exceeds its byte bound")); + } + rows.insert(key, value); + } + } + CapsuleChunkIndex::new(layout) + .publish( + layout.store(), + rows.into_iter().collect(), + maximum, + cancelled, + ) + .await +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests.rs new file mode 100644 index 000000000..773a9b3cc --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests.rs @@ -0,0 +1,212 @@ +use super::*; +use crate::capsule_protocol::{PointerCatalog, ShardCatalogEntry, XorbChunkEntry}; +use crab_storage::{StorageObservation, StorageObserver, StorageOperation}; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct Observations(Mutex>); +impl StorageObserver for Observations { + fn started(&self, _: StorageOperation) {} + fn finished(&self, value: StorageObservation) { + self.0.lock().unwrap().push(value); + } +} + +fn fixture( + count: u32, + seed: u64, +) -> ( + PointerCatalog, + HashMap, + Vec, +) { + let xorb = MerkleHash::from([seed; 4]); + let hashes = (0..count) + .map(|index| MerkleHash::from([u64::from(index), seed, 9, 7])) + .collect::>(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 80, + "4".repeat(64), + hashes + .iter() + .map(|hash| XorbChunkEntry::new(hash.hex(), 7)) + .collect(), + ), + ) + .unwrap(); + catalog + .insert_shard("2".repeat(64), ShardCatalogEntry::new(10, vec![xorb.hex()])) + .unwrap(); + let origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/{}/{}", &xorb.hex()[..2], xorb.hex()), + xorb.into(), + [5; 32], + 80, + Some("etag".to_owned()), + None, + ); + (catalog, HashMap::from([(xorb, origin)]), hashes) +} + +fn layout() -> StoreLayout { + StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())), + "source".to_owned(), + ) +} + +#[tokio::test] +async fn exact_capsule_hints_survive_head_replacement_without_legacy_metadata() { + let layout = layout(); + let (catalog, origins, hashes) = fixture(3, 3); + write_capsule_chunk_candidates(&layout, &catalog, &origins, &|| false) + .await + .unwrap(); + let first = CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &hashes, 1024 * 1024) + .await + .unwrap(); + let newer = StoreLayout::new(layout.store().clone(), "newer-source".to_owned()); + write_capsule_chunk_candidates(&newer, &catalog, &origins, &|| false) + .await + .unwrap(); + let current = CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &hashes, 1024 * 1024) + .await + .unwrap(); + let exact = first + .placements + .values() + .map(|p| (MerkleHash::from(p.chunk_hash), p.placement_id())) + .collect::>(); + let restored = CapsuleChunkIndex::new(&layout) + .candidates_by_id(layout.store(), &exact, 1024 * 1024) + .await + .unwrap(); + assert!(first.placements != current.placements && restored.placements == first.placements); + assert!( + layout + .store() + .list_prefix(&Path::from(".crab/chunk_index_db")) + .await + .unwrap() + .is_empty() + ); +} + +#[tokio::test] +async fn unchanged_root_missing_or_corrupt_pages_remain_errors() { + let layout = layout(); + let (catalog, origins, hashes) = fixture(1, 3); + write_capsule_chunk_candidates(&layout, &catalog, &origins, &|| false) + .await + .unwrap(); + let index = CapsuleChunkIndex::new(&layout); + let mut work = PageWork::new(1024 * 1024, 0); + let (root, _) = index + .load(layout.store(), &mut work) + .await + .unwrap() + .unwrap(); + let path = index.pages.path(&root.records.hash); + let (body, _) = layout + .store() + .get_with_etag_bounded(&path, u64::from(root.records.bytes)) + .await + .unwrap(); + layout.store().delete(&path).await.unwrap(); + assert!(matches!( + CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &hashes, 1024 * 1024) + .await, + Err(MetadataError::Storage { + source: StorageError::NotFound { .. } + }) + )); + let mut damaged = body.to_vec(); + damaged[0] ^= 1; + layout + .store() + .put_overwrite(&path, Bytes::from(damaged)) + .await + .unwrap(); + assert!(matches!( + CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &hashes, 1024 * 1024) + .await, + Err(MetadataError::CorruptObject { .. }) + )); +} + +#[tokio::test] +async fn cancellation_after_page_writes_does_not_advertise_partial_hints() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let layout = layout(); + let (catalog, origins, _) = fixture(1, 3); + let calls = AtomicUsize::new(0); + let result = write_capsule_chunk_candidates(&layout, &catalog, &origins, &|| { + calls.fetch_add(1, Ordering::Relaxed) >= 2 + }) + .await; + assert!(matches!(result, Err(MetadataError::CapsuleHintCancelled))); + assert!( + !layout + .store() + .list_prefix(&layout.capsule_chunk_pages_prefix()) + .await + .unwrap() + .is_empty() + ); + assert!(matches!( + layout + .store() + .get_with_etag(&layout.capsule_chunk_index_path()) + .await, + Err(StorageError::NotFound { .. }) + )); +} + +#[tokio::test] +async fn hint_reads_and_selected_writes_do_not_follow_unrelated_history() { + for count in [100, 10_000, 100_000] { + let observations = Arc::new(Observations::default()); + let layout = StoreLayout::new( + Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(observations.clone()), + "source".to_owned(), + ); + let (catalog, origins, _) = fixture(count, 3); + write_capsule_chunk_candidates(&layout, &catalog, &origins, &|| false) + .await + .unwrap(); + let (delta, origins, hashes) = fixture(1, 5); + observations.0.lock().unwrap().clear(); + write_capsule_chunk_candidates(&layout, &delta, &origins, &|| false) + .await + .unwrap(); + let writes = observations.0.lock().unwrap().drain(..).collect::>(); + let read = writes.iter().map(|o| o.bytes_read).sum::(); + let written = writes.iter().map(|o| o.bytes_written).sum::(); + assert!( + read <= 512 * 1024 && written <= 512 * 1024, + "{count}: read {read}, written {written}" + ); + let result = CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &hashes, 256 * 1024) + .await + .unwrap(); + assert_eq!(result.placements.len(), 1); + let operations = observations.0.lock().unwrap(); + assert!(!operations.iter().any(|o| matches!( + o.operation, + StorageOperation::Put | StorageOperation::Delete | StorageOperation::MultipartStart + ))); + } +} + +mod races; diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests/races.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests/races.rs new file mode 100644 index 000000000..78c00b10a --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/store/tests/races.rs @@ -0,0 +1,257 @@ +use super::*; +use futures_util::{StreamExt, stream::BoxStream}; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct Replacement { + trigger: Path, + root: Path, + body: Bytes, + retired_page: Path, +} +#[derive(Default)] +struct Hooks { + replacement: Mutex>, + root_barrier: Option<(Path, tokio::sync::Barrier)>, + root_reads: AtomicUsize, + reads: AtomicUsize, + denied_writes: AtomicUsize, +} +struct ScriptedStore { + inner: Arc, + read_only: bool, + hooks: Arc, +} +impl std::fmt::Debug for ScriptedStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ScriptedStore") + } +} +impl std::fmt::Display for ScriptedStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("scripted hint store") + } +} +fn denied(path: &Path) -> object_store::Error { + object_store::Error::PermissionDenied { + path: path.to_string(), + source: Box::new(std::io::Error::from(std::io::ErrorKind::PermissionDenied)), + } +} +#[async_trait::async_trait] +impl ObjectStore for ScriptedStore { + async fn put_opts( + &self, + path: &Path, + body: PutPayload, + options: PutOptions, + ) -> object_store::Result { + if self.read_only { + self.hooks.denied_writes.fetch_add(1, Ordering::Relaxed); + return Err(denied(path)); + } + self.inner.put_opts(path, body, options).await + } + async fn put_multipart_opts( + &self, + path: &Path, + options: PutMultipartOptions, + ) -> object_store::Result> { + if self.read_only { + self.hooks.denied_writes.fetch_add(1, Ordering::Relaxed); + return Err(denied(path)); + } + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts(&self, path: &Path, options: GetOptions) -> object_store::Result { + self.hooks.reads.fetch_add(1, Ordering::Relaxed); + let replacement = { + let mut pending = self.hooks.replacement.lock().unwrap(); + if pending.as_ref().is_some_and(|p| p.trigger == *path) { + pending.take() + } else { + None + } + }; + if let Some(replacement) = replacement { + // Model an independent publisher and sweep, not writes by the reader. + self.inner + .put(&replacement.root, replacement.body.into()) + .await?; + self.inner.delete(&replacement.retired_page).await?; + } + let result = self.inner.get_opts(path, options).await; + if let Some((root, barrier)) = &self.hooks.root_barrier + && root == path + && self.hooks.root_reads.fetch_add(1, Ordering::Relaxed) < 2 + { + barrier.wait().await; + } + result + } + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + if !self.read_only { + return self.inner.delete_stream(paths); + } + let hooks = self.hooks.clone(); + paths + .map(move |path| { + hooks.denied_writes.fetch_add(1, Ordering::Relaxed); + path.and_then(|path| Err(denied(&path))) + }) + .boxed() + } + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> object_store::Result<()> { + if self.read_only { + self.hooks.denied_writes.fetch_add(1, Ordering::Relaxed); + return Err(denied(to)); + } + self.inner.copy_opts(from, to, options).await + } +} + +#[tokio::test] +async fn read_only_lookup_recaptures_after_retired_page_is_swept() { + let layout = layout(); + let (catalog, origins, hashes) = fixture(2, 3); + write_capsule_chunk_candidates(&layout, &catalog, &origins, &|| false) + .await + .unwrap(); + let old_index = CapsuleChunkIndex::new(&layout); + let (old_body, _) = layout + .store() + .get_with_etag(&layout.capsule_chunk_index_path()) + .await + .unwrap(); + let old_root: Root = serde_json::from_slice(&old_body).unwrap(); + let old_page = old_index.pages.path(&old_root.records.hash); + let old = old_index + .candidates(layout.store(), &hashes, 1024 * 1024) + .await + .unwrap(); + let newer = StoreLayout::new(layout.store().clone(), "newer-source".to_owned()); + write_capsule_chunk_candidates(&newer, &catalog, &origins, &|| false) + .await + .unwrap(); + let (new_body, _) = layout + .store() + .get_with_etag(&layout.capsule_chunk_index_path()) + .await + .unwrap(); + let (_, live) = CapsuleChunkIndex::new(&layout) + .objects(layout.store(), 1024 * 1024) + .await + .unwrap(); + assert!(!live.iter().any(|(path, _)| *path == old_page)); + layout + .store() + .put_overwrite(&layout.capsule_chunk_index_path(), old_body) + .await + .unwrap(); + let hooks = Arc::new(Hooks { + replacement: Mutex::new(Some(Replacement { + trigger: old_page.clone(), + root: layout.capsule_chunk_index_path(), + body: new_body, + retired_page: old_page, + })), + ..Hooks::default() + }); + let reader = Store::new(Arc::new(ScriptedStore { + inner: layout.store().inner().clone(), + read_only: true, + hooks: hooks.clone(), + })); + let reader_layout = StoreLayout::new(reader.clone(), layout.repo_prefix().to_owned()); + let current = CapsuleChunkIndex::new(&reader_layout) + .candidates(&reader, &hashes, 1024 * 1024) + .await + .unwrap(); + assert!( + current + .anchors + .values() + .all(|anchor| anchor.source_repo_prefix == "newer-source") + ); + let ids = old + .placements + .values() + .map(|p| (MerkleHash::from(p.chunk_hash), p.placement_id())) + .collect::>(); + let exact = CapsuleChunkIndex::new(&reader_layout) + .candidates_by_id(&reader, &ids, 1024 * 1024) + .await + .unwrap(); + assert_eq!(exact.placements, old.placements); + assert_eq!(hooks.denied_writes.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn concurrent_hint_publishers_merge_after_root_create_conflict() { + let memory: Arc = Arc::new(object_store::memory::InMemory::new()); + let plain = StoreLayout::new(Store::new(memory.clone()), "first".to_owned()); + let hooks = Arc::new(Hooks { + root_barrier: Some(( + plain.capsule_chunk_index_path(), + tokio::sync::Barrier::new(2), + )), + ..Hooks::default() + }); + let store = Store::new(Arc::new(ScriptedStore { + inner: memory, + read_only: false, + hooks, + })); + let first = StoreLayout::new(store.clone(), "first".to_owned()); + let second = StoreLayout::new(store.clone(), "second".to_owned()); + let (one, one_origins, mut hashes) = fixture(2, 3); + let (two, two_origins, other_hashes) = fixture(2, 5); + let (a, b) = tokio::join!( + write_capsule_chunk_candidates(&first, &one, &one_origins, &|| false), + write_capsule_chunk_candidates(&second, &two, &two_origins, &|| false) + ); + a.unwrap(); + b.unwrap(); + hashes.extend(other_hashes); + let result = CapsuleChunkIndex::new(&first) + .candidates(&store, &hashes, 1024 * 1024) + .await + .unwrap(); + assert_eq!(result.placements.len(), 4); +} + +#[tokio::test] +async fn request_budget_failure_precedes_storage_access() { + let plain = layout(); + let hooks = Arc::new(Hooks::default()); + let reader = Store::new(Arc::new(ScriptedStore { + inner: plain.store().inner().clone(), + read_only: true, + hooks: hooks.clone(), + })); + let reader_layout = StoreLayout::new(reader.clone(), plain.repo_prefix().to_owned()); + assert!( + CapsuleChunkIndex::new(&reader_layout) + .candidates(&reader, &[MerkleHash::from([1; 32])], 1) + .await + .is_err() + ); + assert_eq!(hooks.reads.load(Ordering::Relaxed), 0); +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 7d2081a4e..9a34135c9 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -129,5 +129,5 @@ pub use chunk_candidates::{ CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, capsule_chunk_head_key, }; -#[cfg(feature = "remote-index")] -pub(crate) use chunk_candidates::capsule_chunk_candidate_rows; +#[cfg(feature = "storage")] +pub use chunk_candidates::{CapsuleChunkIndex, write_capsule_chunk_candidates}; diff --git a/crates/crab-metadata/src/error.rs b/crates/crab-metadata/src/error.rs index 6fb201f7b..f9e6d52d0 100644 --- a/crates/crab-metadata/src/error.rs +++ b/crates/crab-metadata/src/error.rs @@ -102,6 +102,11 @@ pub enum MetadataError { #[error("ref journal publication cancelled before commit")] RefJournalCancelled, + /// Hint publication stopped before advertising its replacement global root. + #[cfg(feature = "storage")] + #[error("capsule hint publication cancelled before root visibility")] + CapsuleHintCancelled, + /// Durable attempt evidence prevents a new execution of this plan. #[cfg(feature = "storage")] #[error( diff --git a/crates/crab-metadata/src/pages.rs b/crates/crab-metadata/src/pages.rs index 1a0466e3f..a83d3a4a7 100644 --- a/crates/crab-metadata/src/pages.rs +++ b/crates/crab-metadata/src/pages.rs @@ -75,6 +75,7 @@ pub(crate) enum Kind { RunCatalogShards = 27, RunCatalogXorbs = 28, RunCatalogChunks = 29, + CapsuleChunkHints = 30, } impl Kind { @@ -119,6 +120,7 @@ impl Kind { Self::ObjectLocators => 20, #[cfg(feature = "remote-index")] Self::PackOffsets => 12, + Self::CapsuleChunkHints => 33, _ => 4, } } diff --git a/crates/crab-metadata/src/remote_index.rs b/crates/crab-metadata/src/remote_index.rs index 4c94dc6a8..f9c956b22 100644 --- a/crates/crab-metadata/src/remote_index.rs +++ b/crates/crab-metadata/src/remote_index.rs @@ -100,73 +100,6 @@ pub async fn write_index_entries( close_result } -/// Publish rebuildable capsule chunk hints using origin receipts captured during byte verification. -/// -/// Call after ref visibility. Every opened database is closed on success or error; -/// callers must await this future through completion. Readers still revalidate source -/// visibility and exact origin identity, so partial batches never confer authority. -pub async fn write_capsule_chunk_candidates( - store: Arc, - config: &RemoteIndexConfig, - repo_prefix: &str, - catalog: &crate::capsule_protocol::PointerCatalog, - origins: &HashMap, -) -> Result<()> { - let mut source_shards = HashMap::new(); - for (shard_hash, shard) in catalog.shards() { - for xorb in shard.xorb_hashes() { - source_shards - .entry(xorb.as_str()) - .or_insert(shard_hash.as_str()); - } - } - let mut selected = Vec::new(); - for (hash, entry) in catalog.xorbs().iter().rev() { - let parsed = MerkleHash::from_hex(hash).map_err(|_| MetadataError::CorruptObject { - path: CHUNK_INDEX_DB_LABEL.to_owned(), - reason: "invalid capsule xorb hash".to_owned(), - })?; - if let Some(origin) = origins.get(&parsed) { - selected.push((hash, parsed, entry, origin)); - } - } - if selected.is_empty() { - return Ok(()); - } - for (hash, _, _, _) in &selected { - if !source_shards.contains_key(hash.as_str()) { - return Err(MetadataError::CorruptObject { - path: CHUNK_INDEX_DB_LABEL.to_owned(), - reason: "capsule xorb candidate has no source shard root".to_owned(), - }); - } - } - let writer = RemoteIndexWriter::open(store, config, false, true).await?; - let result = async { - // Last-write heads choose the lowest canonical xorb and first occurrence. - for (hash, parsed, entry, origin) in selected { - let source_shard = source_shards.get(hash.as_str()).ok_or_else(|| { - MetadataError::Internal( - "admitted capsule candidate lost its source shard".to_owned(), - ) - })?; - let rows = crate::capsule_protocol::capsule_chunk_candidate_rows( - repo_prefix, - source_shard, - parsed, - entry, - origin, - )?; - writer.write_capsule_candidates(&rows).await?; - } - Ok::<_, MetadataError>(()) - } - .await; - let close = writer.close().await; - result?; - close -} - /// Long-lived writer for caller-bounded remote index batches. /// /// Writes buffer data without waiting for durability. Always await [`Self::close`] @@ -231,33 +164,6 @@ impl RemoteIndexWriter { .await } - /// Buffer a bounded set of capsule candidate rows; close remains the durability barrier. - async fn write_capsule_candidates(&self, rows: &[(Vec, Vec)]) -> Result<()> { - let db = self - .chunk_db - .as_ref() - .ok_or_else(|| MetadataError::Internal("chunk index was not opened".to_owned()))?; - for rows in rows.chunks(4096) { - let mut batch = slatedb::WriteBatch::new(); - for (key, value) in rows { - batch.put(key.as_slice(), value.as_slice()); - } - db.write_with_options( - batch, - &slatedb::config::WriteOptions { - await_durable: false, - ..Default::default() - }, - ) - .await - .map_err(|source| MetadataError::SlateDbWrite { - db: CHUNK_INDEX_DB_LABEL.to_owned(), - source, - })?; - } - Ok(()) - } - /// Flush and close every database opened by this writer. /// /// Both closes are attempted. If both fail, return the file-index error. @@ -968,4 +874,4 @@ mod tests { } #[cfg(test)] -mod capsule_tests; +mod format_tests; diff --git a/crates/crab-metadata/src/remote_index/capsule_tests.rs b/crates/crab-metadata/src/remote_index/capsule_tests.rs deleted file mode 100644 index 269567268..000000000 --- a/crates/crab-metadata/src/remote_index/capsule_tests.rs +++ /dev/null @@ -1,165 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; - -use crate::capsule_protocol::{PointerCatalog, XorbCatalogEntry, XorbChunkEntry}; -use crate::receipts::OriginReceipt; -use crab_xet::hash::MerkleHash; -use object_store::memory::InMemory; - -use super::{RemoteIndexConfig, write_capsule_chunk_candidates}; - -#[tokio::test] -async fn capsule_candidates_reject_xorbs_without_a_source_shard_root() { - let hash = MerkleHash::from([3; 32]); - let mut catalog = PointerCatalog::new(); - catalog - .insert_xorb( - hash.hex(), - XorbCatalogEntry::new( - 100, - "4".repeat(64), - vec![XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9)], - ), - ) - .unwrap(); - let origin = OriginReceipt::new( - "canonical-origin".to_owned(), - format!(".crab/xorbs/03/{}", hash.hex()), - hash.into(), - [5; 32], - 100, - Some("etag".to_owned()), - None, - ); - let result = write_capsule_chunk_candidates( - Arc::new(InMemory::new()), - &RemoteIndexConfig::for_repo("repo"), - "repo", - &catalog, - &HashMap::from([(hash, origin)]), - ) - .await; - assert!(result.is_err(), "an xorb descriptor alone is not a GC root"); -} - -#[tokio::test] -async fn capsule_candidates_writer_initializes_canonical_format() { - let store: Arc = Arc::new(InMemory::new()); - let config = RemoteIndexConfig::for_repo("repo"); - let writer = super::RemoteIndexWriter::open(Arc::clone(&store), &config, false, true) - .await - .unwrap(); - writer.close().await.unwrap(); - let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") - .await - .unwrap() - .unwrap(); - let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); - let value = reader.get(&key).await.unwrap(); - reader.close().await.unwrap(); - assert_eq!(value.as_deref(), Some(1u32.to_le_bytes().as_slice())); -} - -#[tokio::test] -async fn capsule_candidates_writer_rejects_noncanonical_existing_databases() { - for marker in [None, Some(2u32.to_le_bytes().to_vec()), Some(vec![1])] { - let store: Arc = Arc::new(InMemory::new()); - let config = RemoteIndexConfig::for_repo("repo"); - let seed = slatedb::Db::open(config.chunk_index_path.clone(), Arc::clone(&store)) - .await - .unwrap(); - if let Some(marker) = marker { - seed.put( - crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION), - marker, - ) - .await - .unwrap(); - } - seed.close().await.unwrap(); - let result = super::RemoteIndexWriter::open(store, &config, false, true).await; - if let Ok(writer) = result { - writer.close().await.unwrap(); - panic!("invalid existing format was accepted"); - } - } -} - -#[tokio::test] -async fn capsule_exact_placement_survives_head_replacement() { - use crate::capsule_protocol::ShardCatalogEntry; - use crate::receipts::CommittedChunkPlacement; - - let store: Arc = Arc::new(InMemory::new()); - let config = RemoteIndexConfig::for_repo("source"); - let xorb = MerkleHash::from([3; 32]); - let chunk = MerkleHash::from([1; 32]); - let mut catalog = PointerCatalog::new(); - catalog - .insert_xorb( - xorb.hex(), - XorbCatalogEntry::new( - 100, - "4".repeat(64), - vec![XorbChunkEntry::new(chunk.hex(), 9)], - ), - ) - .unwrap(); - catalog - .insert_shard("2".repeat(64), ShardCatalogEntry::new(20, vec![xorb.hex()])) - .unwrap(); - let mut origin = OriginReceipt::new( - "canonical-origin".to_owned(), - format!(".crab/xorbs/03/{}", xorb.hex()), - xorb.into(), - [5; 32], - 100, - Some("first".to_owned()), - None, - ); - let mut placements = Vec::new(); - for repo in ["source", "newer-source"] { - write_capsule_chunk_candidates( - Arc::clone(&store), - &config, - repo, - &catalog, - &HashMap::from([(xorb, origin.clone())]), - ) - .await - .unwrap(); - let reader = super::open_reader( - Arc::clone(&store), - &config.chunk_index_path, - "chunk_index_db", - ) - .await - .unwrap() - .unwrap(); - let head = reader - .get(crate::key_codec::capsule_chunk_head_key(&chunk.into())) - .await - .unwrap() - .unwrap(); - reader.close().await.unwrap(); - placements.push(CommittedChunkPlacement::decode(&head).unwrap()); - origin.etag = Some("second".to_owned()); - } - let first = &placements[0]; - let mut key = vec![0x08]; - key.extend_from_slice(&first.placement_id()); - let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") - .await - .unwrap() - .unwrap(); - let exact = reader.get(key).await.unwrap(); - reader.close().await.unwrap(); - assert_eq!( - ( - placements[0] != placements[1], - exact.map(|value| CommittedChunkPlacement::decode(&value).unwrap()) - ), - (true, Some(first.clone())), - "a changed shared head must not erase the proof selected by an earlier add", - ); -} diff --git a/crates/crab-metadata/src/remote_index/format_tests.rs b/crates/crab-metadata/src/remote_index/format_tests.rs new file mode 100644 index 000000000..7fb4a3371 --- /dev/null +++ b/crates/crab-metadata/src/remote_index/format_tests.rs @@ -0,0 +1,46 @@ +use super::RemoteIndexConfig; +use object_store::memory::InMemory; +use std::sync::Arc; + +#[tokio::test] +async fn writer_initializes_canonical_format() { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let writer = super::RemoteIndexWriter::open(Arc::clone(&store), &config, false, true) + .await + .unwrap(); + writer.close().await.unwrap(); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = reader.get(&key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!(value.as_deref(), Some(1u32.to_le_bytes().as_slice())); +} + +#[tokio::test] +async fn writer_rejects_noncanonical_existing_databases() { + for marker in [None, Some(2u32.to_le_bytes().to_vec()), Some(vec![1])] { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let seed = slatedb::Db::open(config.chunk_index_path.clone(), Arc::clone(&store)) + .await + .unwrap(); + if let Some(marker) = marker { + seed.put( + crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION), + marker, + ) + .await + .unwrap(); + } + seed.close().await.unwrap(); + let result = super::RemoteIndexWriter::open(store, &config, false, true).await; + if let Ok(writer) = result { + writer.close().await.unwrap(); + panic!("invalid existing format was accepted"); + } + } +} diff --git a/crates/crab-remote/src/prepare/capsule.rs b/crates/crab-remote/src/prepare/capsule.rs index dcf26b988..f44016f66 100644 --- a/crates/crab-remote/src/prepare/capsule.rs +++ b/crates/crab-remote/src/prepare/capsule.rs @@ -248,17 +248,11 @@ impl CapsuleArtifacts { Ok(_) => { // Ref visibility owns the payload before hints become discoverable. // Failed hint maintenance cannot reject an already committed Git push. - let config = - crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( - self.layout.repo_prefix(), - self.layout.global_prefix(), - ); - if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( - Arc::clone(self.layout.store().inner()), - &config, - self.layout.repo_prefix(), + if let Err(error) = crab_metadata::capsule_protocol::write_capsule_chunk_candidates( + &self.layout, &self.pointer_delta, &self.origins, + &|| cancel.is_cancelled(), ) .await { diff --git a/crates/crab-sdk/src/remote_error.rs b/crates/crab-sdk/src/remote_error.rs index 2b18206b5..53e0ee676 100644 --- a/crates/crab-sdk/src/remote_error.rs +++ b/crates/crab-sdk/src/remote_error.rs @@ -106,7 +106,7 @@ pub(crate) fn metadata_kind(error: &crab_metadata::error::MetadataError) -> Erro E::Io { source } if source.kind() == std::io::ErrorKind::InvalidData => { ErrorKind::Corruption } - E::RefJournalCancelled => ErrorKind::Cancelled, + E::RefJournalCancelled | E::CapsuleHintCancelled => ErrorKind::Cancelled, E::ManifestCasConflict { .. } => ErrorKind::Conflict, E::ObjectStore { source } => object_store_kind(source), _ => ErrorKind::Io, diff --git a/crates/crab-storage/AGENTS.md b/crates/crab-storage/AGENTS.md index 0af308c91..10f2cf222 100644 --- a/crates/crab-storage/AGENTS.md +++ b/crates/crab-storage/AGENTS.md @@ -27,6 +27,7 @@ dependency source before asserting provider behavior. | CAS behavior | `crates/crab-storage/src/cas.rs` | `crates/crab-metadata/src/manifest_store.rs` | | Provider or target identity | `crates/crab-storage/src/provider_store.rs` | `crates/crab-auth-store/src/lib.rs` | | Object key layout | `crates/crab-storage/src/layout.rs` | `crates/crab-cache/src/path_class.rs` | +| Global capsule hint paths | `StoreLayout::capsule_chunk_index_path / capsule_chunk_pages_prefix` in `layout.rs` | `crab-metadata::capsule_protocol::CapsuleChunkIndex` and bucket GC | | Private local spills | `crates/crab-storage/src/scratch.rs` | `crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs` | ## Invariants diff --git a/crates/crab-storage/src/layout.rs b/crates/crab-storage/src/layout.rs index 0f67b3ca6..5768ccff7 100644 --- a/crates/crab-storage/src/layout.rs +++ b/crates/crab-storage/src/layout.rs @@ -9,6 +9,8 @@ use object_store::path::Path as ObjectPath; pub const GLOBAL_PREFIX: &str = ".crab"; /// Number of leading hash characters used to partition global content. pub const GLOBAL_CONTENT_FANOUT_WIDTH: usize = 2; +/// Global immutable pages owned by the rebuildable capsule chunk-hint root. +pub const CAPSULE_CHUNK_PAGE_KIND: &str = "capsule-chunk-pages"; /// Build the directory containing one global content kind. #[must_use] @@ -214,6 +216,18 @@ impl StoreLayout { self.repo_path(&format!("v2/checkpoints/{partition}/{hash}")) } + /// Path to the conditional global root of rebuildable capsule chunk hints. + #[must_use] + pub fn capsule_chunk_index_path(&self) -> ObjectPath { + ObjectPath::from(format!("{}/capsule-chunk-index", self.global_prefix)) + } + + /// Prefix containing immutable pages owned by the global capsule hint root. + #[must_use] + pub fn capsule_chunk_pages_prefix(&self) -> ObjectPath { + global_content_prefix(&self.global_prefix, CAPSULE_CHUNK_PAGE_KIND) + } + /// Prefix containing immutable checkpoint catalog objects. #[must_use] pub fn capsule_catalog_prefix(&self) -> ObjectPath { diff --git a/crates/crab-write/AGENTS.md b/crates/crab-write/AGENTS.md index 43ac9f9b8..66e3f6653 100644 --- a/crates/crab-write/AGENTS.md +++ b/crates/crab-write/AGENTS.md @@ -32,6 +32,10 @@ Trace one path: `crab/src/git/push.rs` → `commit_edits` in - commit_edits requires a snapshot captured under every edited ref lease; callers retain authorization, dependency checks, uploads, and lease renewal. Source: `crates/crab-write/src/journal.rs`. +- Cancellation-sensitive capsule callers use `prepare_publication`, check their + lease token, then await `publish_prepared` through commit resolution. Do not + drop publication on renewal loss or reinterpret committed success as cleanup + failure. Source: `crates/crab-write/src/capsule_protocol.rs`. - A marker storage error can leave an uncertain commit. Await the commit future and resolve outcome rather than treating every error as rejection. Source: `crates/crab-write/src/journal.rs`. - Maintenance combines operation and cleanup errors through `finish_after_cleanup` diff --git a/crates/crab-write/README.md b/crates/crab-write/README.md index 0372db111..6c21c69fd 100644 --- a/crates/crab-write/README.md +++ b/crates/crab-write/README.md @@ -21,7 +21,11 @@ Some(manifest): ready None: capture fresh state and try another pass A commit error may be uncertain; resolve that outcome before proceeding or reporting rejection. Read readiness is a separate result from ref durability. -Protocol v2 uses `capsule_protocol::publish` instead of the v1 journal. A +Protocol v2 uses `capsule_protocol::publish` instead of the v1 journal. +Callers with cancellation-sensitive immutable preparation can use +`prepare_publication` then `publish_prepared`, checking their lease token between +them. Publication must be awaited through outcome resolution, even if renewal +fails after that boundary. A single-ref push commits by conditionally replacing only that ref head. A multi-ref push creates a unique preparing transaction record, conditionally prepares each edited head, wins a commit-vs-abort CAS on that record, and diff --git a/crates/crab-write/src/capsule_protocol.rs b/crates/crab-write/src/capsule_protocol.rs index 4eb49265a..5600edb8e 100644 --- a/crates/crab-write/src/capsule_protocol.rs +++ b/crates/crab-write/src/capsule_protocol.rs @@ -323,7 +323,11 @@ pub async fn prepare_publication_with_ref_head_bases( }) } -async fn publish_prepared( +/// Make a prepared capsule visible and resolve any uncertain commit outcome. +/// +/// Callers retain ref/GC leases and check cancellation before entering. Await +/// completion without dropping this future, including after lease renewal loss. +pub async fn publish_prepared( router: &StoreLayout, prepared: PreparedCapsulePublication, ) -> Result { From 8cb899e00e0c1fc9fa4b3c3be01448438c9ca4f1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 23:29:28 -0700 Subject: [PATCH 115/127] fix(push): preserve cancellation commit outcomes Check scoped cancellation after immutable preparation and before ref visibility. Drain visibility resolution and retain acknowledged commits across late cancellation and cleanup failure. --- .../docs/architecture/indexed-remote-reads.md | 21 +++ crab/src/git/capsule_push.rs | 143 ++++++++++++------ .../git/capsule_push/tests/cancellation.rs | 75 +++++++++ crab/src/import/publish.rs | 49 +++++- crates/crab-remote/README.md | 4 + crates/crab-remote/src/prepare/capsule.rs | 26 +++- .../crab-remote/tests/capsule_preparation.rs | 89 ++++++----- .../tests/capsule_preparation/cancellation.rs | 107 +++++++++++++ crates/crab-storage/README.md | 5 + crates/crab-storage/src/test_support.rs | 25 ++- 10 files changed, 445 insertions(+), 99 deletions(-) create mode 100644 crab/src/git/capsule_push/tests/cancellation.rs create mode 100644 crates/crab-remote/tests/capsule_preparation/cancellation.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e17006324..0f3c6bbdc 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2586,3 +2586,24 @@ The fixed edit must stay below 512 KiB for both bytes read and written; its look has a 256 KiB budget. These are encoded-request bounds, not latency claims. Full-source RustFS, cache-service, protected receive and final 10 GiB acceptance remain separate gates until their receipts pass. + +## Publication cancellation retains the authoritative outcome + +CLI capsule push, shared preparation and protected receive check their scoped +lease token after immutable preparation and before visibility. Namespace renewal +must cancel that same token; checking only the caller token before preparation +misses a lease lost during upload or frontier compaction. Once the ref CAS or +coordinator operation starts, the operation resolves its outcome before cleanup. + +CLI GC-admission cleanup may report late cancellation or a release failure after +a successful ref transition. That is repair work, not a rejected push: preserve +the committed outcome and finish local staging retirement. Uncertain publication +and ordinary rejection remain distinct. Import consumes the same terminal +publisher outcome; a second cancellation check after the return cannot revoke an +acknowledged ref or leave a completed import looking retryable. + +Deterministic tests cancel on successful immutable or ref-head PUTs. They cover +new and existing shared refs, direct/coordinated CLI preparation, exact staging +retirement after late cancellation, and imported payload reconstruction. The +shared cold-index fixture uses the capsule hint owner while preserving current +source-catalog, exact provider-identity and complete reconstruction assertions. diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index 8b3b8d27e..873b48ab7 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -479,9 +479,24 @@ async fn run_inner( .await?; return Ok(Some(outcome)); } + let payload_objects = if config.active_active_replication.is_some() { + pointer_delta + .xorbs() + .keys() + .map(|hash| layout.xorb_path(hash).to_string()) + .chain( + pointer_delta + .shards() + .keys() + .map(|hash| layout.shard_path(hash).to_string()), + ) + .chain(lfs_objects) + .collect::>() + } else { + Vec::new() + }; let result = if changes_namespace { let commit_layout = layout.clone(); - let commit_pointer_delta = &pointer_delta; let commit_push_ref_head_bases = push_ref_head_bases.clone(); crab_write::with_ref_namespaces_wait( layout.store(), @@ -506,9 +521,9 @@ async fn run_inner( base, &transaction, &capsule, - commit_pointer_delta, - &lfs_objects, + &payload_objects, &commit_push_ref_head_bases, + &scoped, ) .await }, @@ -521,9 +536,9 @@ async fn run_inner( base, &transaction, &capsule, - &pointer_delta, - &lfs_objects, + &payload_objects, &push_ref_head_bases, + cancel, ) .await }; @@ -561,13 +576,25 @@ async fn run_inner( None => Ok(()), }; let committed = match (publication, release) { - (Ok(committed), Ok(())) => committed, - (Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error), - (Err(error), Err(release_error)) => { - tracing::warn!( - error = %release_error, - "capsule-protocol GC admission release failed after push failure" - ); + (Ok(Some(committed)), release) => { + // Visibility already owns the uploaded payload. Late cancellation + // or fence cleanup failure cannot turn that commit into rejection. + if let Err(error) = release { + tracing::warn!(%error, "capsule committed; GC admission cleanup requires repair"); + } + Some(committed) + } + (Ok(None), release) => { + release?; + None + } + (Err(error), release) => { + if let Err(release_error) = release { + tracing::warn!( + error = %release_error, + "capsule-protocol GC admission release failed after push failure" + ); + } return Err(error); } }; @@ -671,20 +698,30 @@ async fn publish_capsule( base: crab_metadata::capsule_protocol::RootSnapshot, transaction: &crab_metadata::capsule_protocol::CapsuleTransaction, capsule: &crab_metadata::capsule_protocol::Capsule, - pointer_delta: &crab_metadata::capsule_protocol::PointerCatalog, - lfs_objects: &[String], + payload_objects: &[String], push_ref_head_bases: &BTreeMap>, + cancel: &CancellationToken, ) -> Result { + check_cancelled(cancel)?; let Some(replication) = config.active_active_replication.as_ref() else { - return match crab_write::capsule_protocol::publish_with_ref_head_bases( - layout, - base, - transaction, - capsule, - push_ref_head_bases, - ) - .await - { + let publish = async { + let prepared = crab_write::capsule_protocol::prepare_publication_with_ref_head_bases( + layout, + base, + transaction, + capsule, + push_ref_head_bases, + ) + .await?; + // Ref/GC renewal may fail while immutable preparation runs. Check + // its scoped token before visibility, then resolve the commit fully. + if cancel.is_cancelled() { + return Err(crab_write::WriteError::Cancelled); + } + crab_write::capsule_protocol::publish_prepared(layout, prepared).await + } + .await; + return match publish { Ok(_) => Ok(CapsulePublishAttempt::Committed(None)), Err(crab_write::WriteError::RefChanged { .. }) => Ok(CapsulePublishAttempt::RefChanged), Err(error) => Err(error.into()), @@ -724,19 +761,7 @@ async fn publish_capsule( .collect::>(); let mut uploaded_objects = BTreeSet::new(); uploaded_objects.insert(layout.capsule_path(&descriptor.run_hash).to_string()); - uploaded_objects.extend( - pointer_delta - .xorbs() - .keys() - .map(|hash| layout.xorb_path(hash).to_string()), - ); - uploaded_objects.extend(lfs_objects.iter().cloned()); - uploaded_objects.extend( - pointer_delta - .shards() - .keys() - .map(|hash| layout.shard_path(hash).to_string()), - ); + uploaded_objects.extend(payload_objects.iter().cloned()); if let Some(plan_id) = transaction.plan_id() { uploaded_objects.insert(layout.capsule_plan_intent_path(plan_id).to_string()); } @@ -747,6 +772,7 @@ async fn publish_capsule( refs, uploaded_objects.into_iter().collect(), )?; + check_cancelled(cancel)?; let mut outcome = match crab_coordination::write_coordinator::commit_uploaded_push_refs( coordinator.as_ref(), plan.request.clone(), @@ -1232,6 +1258,8 @@ mod tests { use super::*; + mod cancellation; + #[derive(Default)] struct RecordingObserver { observations: Mutex>, @@ -1391,9 +1419,9 @@ mod tests { base, &transaction, &capsule, - &crab_metadata::capsule_protocol::PointerCatalog::new(), &[], &BTreeMap::new(), + &CancellationToken::new(), ) .await .unwrap(); @@ -1457,9 +1485,9 @@ mod tests { base, &transaction, &capsule, - &crab_metadata::capsule_protocol::PointerCatalog::new(), &[], &BTreeMap::new(), + &CancellationToken::new(), ) .await .unwrap(); @@ -2286,12 +2314,13 @@ mod tests { use bstr::ByteSlice; use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; let _git_env = crate::test::git_repo::CleanGitEnvGuard::new(); - for (stale, native_intent, restored_index) in [ - (false, false, false), - (true, false, false), - (false, true, false), - (true, true, false), - (false, true, true), + for (stale, native_intent, restored_index, cancel_after_commit) in [ + (false, false, false, false), + (true, false, false, false), + (false, true, false, false), + (true, true, false, false), + (false, true, true, false), + (false, false, false, true), ] { let source = tempfile::tempdir().unwrap(); git(source.path(), &["init", "-b", "main"]); @@ -2355,7 +2384,10 @@ mod tests { } staging.close().await.unwrap(); let staging = Arc::new(StagingAreaReadOnly::open(staging_root).await.unwrap()); - let storage = crab_storage::Store::new(Arc::new(InMemory::new())); + let hooked = Arc::new(crab_storage::test_support::CountingObjectStore::new( + Arc::new(InMemory::new()), + )); + let storage = crab_storage::Store::new(hooked.clone()); let store = crate::storage::store::Store::from_storage(storage); let router = crate::storage::StoreLayout::new(store.clone(), "repos/test".to_owned()); let layout = @@ -2391,6 +2423,10 @@ mod tests { git_dir: Some(source.path().join(".git")), ..Default::default() }; + let cancel = CancellationToken::new(); + if cancel_after_commit { + hooked.cancel_after_put(layout.capsule_ref_heads_prefix(), cancel.clone()); + } let (result, _) = run( &config, &[PushSpec { @@ -2405,10 +2441,23 @@ mod tests { Some(&staging), None, None, - &CancellationToken::new(), + &cancel, ) .await .unwrap(); + assert_eq!(cancel.is_cancelled(), cancel_after_commit); + if cancel_after_commit { + for domain in [layout.global_prefix(), layout.repo_prefix()] { + let sweep = crab_coordination::GcFenceLease::acquire_sweep( + layout.store().inner(), + domain, + std::time::Duration::from_secs(60), + ) + .await + .unwrap(); + sweep.release().await.unwrap(); + } + } let health = staging.lifecycle_health().unwrap(); let view = crab_read::capsule_protocol::open_view(&layout, limits) .await @@ -2423,7 +2472,7 @@ mod tests { && health.committed_push_snapshots == 0 && health.unresolved_publications == 0 && staging.list_inflight().unwrap().is_empty(), - "stale={stale}, native_intent={native_intent}, restored_index={restored_index}, health={health:?}" + "stale={stale}, native_intent={native_intent}, restored_index={restored_index}, cancel_after_commit={cancel_after_commit}, health={health:?}" ); } } diff --git a/crab/src/git/capsule_push/tests/cancellation.rs b/crab/src/git/capsule_push/tests/cancellation.rs new file mode 100644 index 000000000..5158634d1 --- /dev/null +++ b/crab/src/git/capsule_push/tests/cancellation.rs @@ -0,0 +1,75 @@ +use super::*; + +#[tokio::test] +async fn cancellation_during_preparation_does_not_publish_direct_or_coordinated_refs() { + for coordinated in [false, true] { + let hooked = Arc::new(crab_storage::test_support::CountingObjectStore::new( + Arc::new(InMemory::new()), + )); + let layout = crab_storage::StoreLayout::new( + crab_storage::Store::new(hooked.clone()), + "repos/cancel".to_owned(), + ); + let base = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let transaction = crab_metadata::capsule_protocol::CapsuleTransaction::new( + base.record().digest(), + vec![crab_metadata::capsule_protocol::CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = + crab_metadata::capsule_protocol::Capsule::build(&transaction, Vec::new(), Vec::new()) + .unwrap(); + let coordinator = Arc::new(crab_coordination::InMemoryWriteCoordinator::new()); + let config = if coordinated { + PushConfig { + active_active_replication: Some(active_active_replication()), + active_active_writer: Some("east".to_owned()), + active_active_coordinator: Some( + crate::git::push::ActiveActiveWriteCoordinator::new(coordinator.clone()), + ), + ..PushConfig::default() + } + } else { + PushConfig::default() + }; + let cancel = CancellationToken::new(); + hooked.cancel_after_put(layout.repo_path("v2/capsules"), cancel.clone()); + let result = publish_capsule( + &config, + &layout, + base, + &transaction, + &capsule, + &[], + &BTreeMap::new(), + &cancel, + ) + .await; + assert!( + matches!(result, Err(CrabError::Cancelled)), + "coordinated={coordinated}" + ); + assert!( + crab_read::capsule_protocol::read_visible_refs(&layout) + .await + .unwrap() + .is_empty() + ); + assert!( + coordinator + .repair_snapshot() + .await + .unwrap() + .materialization_gaps + .is_empty() + ); + } +} diff --git a/crab/src/import/publish.rs b/crab/src/import/publish.rs index 05dda8019..78630e553 100644 --- a/crab/src/import/publish.rs +++ b/crab/src/import/publish.rs @@ -81,8 +81,8 @@ pub struct PublishInputs { /// before/after snapshot so [`PublishStats::bytes_uploaded`] /// reflects this publish only. pub metrics: Option>, - /// Cancellation token. Honored before the push kicks off and - /// again after it returns. + /// Cancellation token, honored until publication begins; a known commit + /// remains successful if cancellation arrives during outcome resolution. pub cancel: CancellationToken, } @@ -123,8 +123,7 @@ pub struct PublishStats { /// /// # Errors /// -/// - [`CrabError::Cancelled`] if the cancellation token trips -/// before or after the push. +/// - [`CrabError::Cancelled`] if cancellation stops the push before commitment. /// - [`CrabError::Internal`] carrying the combined per-ref error /// messages when any ref outcome is not `Ok`. The stage treats /// this as a hard failure — the import command should not proceed @@ -206,8 +205,8 @@ pub async fn run_publish(inputs: PublishInputs) -> Result { ) .await?; - check_cancelled(&cancel)?; - + // The publisher resolved the ref outcome and drained its leases. Preserve + // that result even if cancellation arrived after visibility. let (refs_pushed, refs_failed, failure_messages) = summarize_outcomes(&result); // Translate metrics deltas and the push transfer summary into @@ -704,6 +703,44 @@ mod tests { // ── Unit: outcome summarization ────────────────────────────── + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn late_cancellation_preserves_published_import_and_reconstruction() { + let _git_dir_guard = GitDirOverride::locked_without_env(); + let tmp = TempDir::new().unwrap(); + let source_inner: Arc = Arc::new(InMemory::new()); + let objects = vec![("model.bin", vec![0x11; 8192])]; + seed_object(&source_inner, "", objects[0].0, &objects[0].1).await; + let e2e = run_ingest_and_assemble(resolved(source_inner, ""), &objects, &tmp).await; + let hooked = Arc::new(crab_storage::test_support::CountingObjectStore::new( + Arc::new(InMemory::new()), + )); + let target_inner: Arc = hooked.clone(); + let target = resolved(target_inner.clone(), "repos/late-cancel"); + let layout = crab_storage::StoreLayout::new( + target.store.as_storage().clone(), + "repos/late-cancel".to_owned(), + ); + let cancel = CancellationToken::new(); + hooked.cancel_after_put(layout.capsule_ref_heads_prefix(), cancel.clone()); + let stats = run_publish(PublishInputs { + target, + repo_prefix: "repos/late-cancel".to_owned(), + staging: Arc::new(StagingAreaReadOnly::open(e2e.staging_root).await.unwrap()), + branch: "main".to_owned(), + head_commit_oid: e2e.head_oid.clone(), + git_dir: e2e.repo_root.join(".git"), + metrics: None, + cancel: cancel.clone(), + }) + .await + .unwrap(); + assert!(cancel.is_cancelled() && stats.refs_pushed == 1 && stats.refs_failed == 0); + let (_, view) = target_view(&target_inner, "repos/late-cancel").await; + assert_eq!(view.refs().get("refs/heads/main"), Some(&e2e.head_oid)); + verify_reconstructed_files(&target_inner, "repos/late-cancel", &e2e.repo_root, &objects) + .await; + } + #[test] fn summarize_outcomes_counts_ok_and_errors() { use std::collections::HashMap; diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 0364edf60..44f8c6dc7 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -70,6 +70,10 @@ verified create/read identities with their source catalog, publishes rebuildable chunk hints only after proven ref visibility, and preserves a committed outcome if hint-index maintenance fails. Uncertain publication does not publish hints. Caller-owned ref leases and GC fences cover this complete operation. +Preparation rechecks caller cancellation and namespace renewal immediately before +visibility. Cancellation during immutable preparation leaves only unadvertised +objects; cancellation after visibility retains the known committed result while +all publication and lease cleanup drains. New refs and explicit cross-ref visibility bases publish complete replacement closures from the captured proof. Only same-ref updates retain additive evidence; a different ref's later deletion must not invalidate an acknowledged branch. diff --git a/crates/crab-remote/src/prepare/capsule.rs b/crates/crab-remote/src/prepare/capsule.rs index f44016f66..fd57be9e3 100644 --- a/crates/crab-remote/src/prepare/capsule.rs +++ b/crates/crab-remote/src/prepare/capsule.rs @@ -230,19 +230,37 @@ impl CapsuleArtifacts { transaction.edits(), ) .await?; - crab_write::capsule_protocol::publish( + let prepared = crab_write::capsule_protocol::prepare_publication( &commit_layout, base, &transaction, &capsule, ) - .await + .await?; + // Namespace renewal joins caller cancellation. Recheck that + // scoped token after preparation, then drain visibility. + if scoped.is_cancelled() { + return Err(crab_write::WriteError::Cancelled); + } + crab_write::capsule_protocol::publish_prepared(&commit_layout, prepared).await }, ) .await } else { - crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) - .await + let prepared = crab_write::capsule_protocol::prepare_publication( + &self.layout, + self.base, + &transaction, + &capsule, + ) + .await; + match prepared { + Ok(_) if cancel.is_cancelled() => Err(crab_write::WriteError::Cancelled), + Ok(prepared) => { + crab_write::capsule_protocol::publish_prepared(&self.layout, prepared).await + } + Err(error) => Err(error), + } }; match publish { Ok(_) => { diff --git a/crates/crab-remote/tests/capsule_preparation.rs b/crates/crab-remote/tests/capsule_preparation.rs index 76539482e..c00f33ef4 100644 --- a/crates/crab-remote/tests/capsule_preparation.rs +++ b/crates/crab-remote/tests/capsule_preparation.rs @@ -384,37 +384,13 @@ async fn verify_indexed_publication( result.unwrap(), crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } )); - let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( - layout.repo_prefix(), - layout.global_prefix(), - ); - let reader = - slatedb::DbReader::builder(config.chunk_index_path, Arc::clone(layout.store().inner())) - .build() - .await - .expect("shared publication must populate cold native-add candidates"); let hash = MerkleHash::from_hex(&chunk).unwrap(); - let candidate = reader - .get(crab_metadata::key_codec::capsule_chunk_head_key( - &hash.into(), - )) - .await - .unwrap() - .expect("published payload candidate"); - let placement = crab_metadata::receipts::CommittedChunkPlacement::decode(&candidate).unwrap(); - let anchor = reader - .get(crab_metadata::key_codec::capsule_chunk_anchor_key( - &placement.source_anchor_id, - )) + let candidates = crab_metadata::capsule_protocol::CapsuleChunkIndex::new(&layout) + .candidates(layout.store(), &[hash], LIMIT) .await - .unwrap() - .unwrap(); - let anchor = crab_metadata::capsule_protocol::CapsuleChunkAnchor::decode( - &placement.source_anchor_id, - &anchor, - ) - .unwrap(); - reader.close().await.unwrap(); + .expect("shared publication must populate cold native-add candidates"); + let placement = candidates.placements[&<[u8; 32]>::from(hash)].clone(); + let anchor = candidates.anchors[&placement.source_anchor_id].clone(); let path = layout.xorb_path(&MerkleHash::from(placement.xorb_hash)); let (body, identity) = layout.store().get_with_etag(&path).await.unwrap(); let root = crab_write::capsule_protocol::open_root(&layout) @@ -546,21 +522,18 @@ async fn git_only_capsule_upload_does_not_read_historical_catalog() { async fn committed_capsule_survives_candidate_index_failure() { let (directory, content) = content_fixture().await; source_commit(directory.path(), &content); + let file = content.files()[0].clone(); let (layout, _) = layout().await; - let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( - layout.repo_prefix(), - layout.global_prefix(), - ); - let db = slatedb::Db::open(config.chunk_index_path, Arc::clone(layout.store().inner())) + // Inject the failure in the capsule owner. A retired manifest database + // no longer participates in this publication or proves a repair boundary. + layout + .store() + .put_exact( + &layout.capsule_chunk_index_path(), + Bytes::from_static(b"invalid capsule hint root"), + ) .await .unwrap(); - db.put( - crab_metadata::key_codec::encode_system_key(crab_metadata::key_codec::SYS_FORMAT_VERSION), - 2u32.to_le_bytes(), - ) - .await - .unwrap(); - db.close().await.unwrap(); let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; let artifacts = prepared .upload_capsule(&view, proof_limits(), &CancellationToken::new()) @@ -582,6 +555,37 @@ async fn committed_capsule_survives_candidate_index_failure() { ), "repairable hint-index failure must not reject committed refs" ); + assert!( + crab_metadata::capsule_protocol::CapsuleChunkIndex::new(&layout) + .available(layout.store(), LIMIT) + .await + .is_err() + ); + let tip = String::from_utf8(git(directory.path(), &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .to_owned(); + assert_eq!( + crab_read::capsule_protocol::read_visible_refs(&layout) + .await + .unwrap() + .get("refs/heads/main"), + Some(&tip) + ); + let crab_git::pointer_detect::PointerKind::Crab(pointer) = crab_git::pointer_detect::classify( + &std::fs::read(directory.path().join("model.bin")).unwrap(), + ) else { + panic!("fixture pointer"); + }; + crab_read::pointer_proof::verify_crab_pointer( + &layout, + &pointer, + MerkleHash::from(*file.shard_hash()), + proof_limits().content, + &CancellationToken::new(), + ) + .await + .unwrap(); } #[tokio::test(flavor = "multi_thread")] @@ -642,3 +646,6 @@ async fn historical_pointer_lookup_preserves_declared_size_checks() { #[path = "capsule_preparation/ref_visibility.rs"] mod ref_visibility; + +#[path = "capsule_preparation/cancellation.rs"] +mod cancellation; diff --git a/crates/crab-remote/tests/capsule_preparation/cancellation.rs b/crates/crab-remote/tests/capsule_preparation/cancellation.rs new file mode 100644 index 000000000..f7fc89fb3 --- /dev/null +++ b/crates/crab-remote/tests/capsule_preparation/cancellation.rs @@ -0,0 +1,107 @@ +use super::*; + +#[tokio::test(flavor = "multi_thread")] +async fn preparation_cancellation_and_known_commit_keep_distinct_outcomes() { + use crab_remote::prepare::{CapsuleCommitOutcome, Error}; + for existing in [false, true] { + for late in [false, true] { + let hooked = Arc::new(crab_storage::test_support::CountingObjectStore::new( + Arc::new(object_store::memory::InMemory::new()), + )); + let layout = crab_storage::StoreLayout::new( + crab_storage::Store::new(hooked.clone()), + "repositories/cancel".to_owned(), + ); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let initial_tip = String::from_utf8(git(directory.path(), &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .to_owned(); + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let (artifacts, runtime, leases, previous) = if existing { + let initial = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!(initial, CapsuleCommitOutcome::Committed { .. })); + leases.release_after_commit().await; + runtime.shutdown().await; + std::fs::write(directory.path().join("next.txt"), b"next version").unwrap(); + git(directory.path(), &["add", "next.txt"]); + git(directory.path(), &["commit", "-m", "next"]); + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), None).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + (artifacts, runtime, leases, Some(initial_tip)) + } else { + (artifacts, runtime, leases, None) + }; + let tip = String::from_utf8(git(directory.path(), &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .to_owned(); + let cancel = CancellationToken::new(); + let trigger = if late { + layout.capsule_ref_heads_prefix() + } else { + layout.repo_path("v2/capsules") + }; + hooked.cancel_after_put(trigger, cancel.clone()); + let result = artifacts + .commit(None, std::time::Duration::from_secs(60), &cancel) + .await; + if result.is_ok() { + leases.release_after_commit().await; + } else { + leases.release().await; + } + runtime.shutdown().await; + assert!(cancel.is_cancelled(), "existing={existing}, late={late}"); + let refs = crab_read::capsule_protocol::read_visible_refs(&layout) + .await + .unwrap(); + if late { + assert!( + matches!(result, Ok(CapsuleCommitOutcome::Committed { .. })), + "existing={existing}: {result:?}" + ); + assert_eq!(refs.get("refs/heads/main"), Some(&tip)); + } else { + assert!( + matches!(result, Err(Error::Write(crab_write::WriteError::Cancelled))), + "existing={existing}: {result:?}" + ); + assert_eq!(refs.get("refs/heads/main"), previous.as_ref()); + } + let new_cancel = CancellationToken::new(); + let successor = crab_remote::publication::acquire_leases( + layout.store(), + &layout, + ["refs/heads/main".to_owned()], + crab_remote::publication::LeaseOptions::renewing(std::time::Duration::from_secs( + 60, + )), + &new_cancel, + ) + .await + .unwrap(); + successor.release().await; + } + } +} diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index 0e0932d29..aa20227ed 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -208,3 +208,8 @@ awaiting future is dropped. `close` drains those workers before removing the roo callers retain primary validation errors while reporting cleanup failures. Drop performs best-effort off-thread cleanup after the last owner exits. No SlateDB, SQLite, new dependency or additional configuration is introduced by this transport. + +The `test-support` `CountingObjectStore::cancel_after_put` hook arms one +cancellation after a successful PUT beneath a selected component prefix. It +leaves failed writes and unrelated keys untouched, so publication tests can +separate immutable-preparation cancellation from cancellation after visibility. diff --git a/crates/crab-storage/src/test_support.rs b/crates/crab-storage/src/test_support.rs index 5135f38d2..96629295c 100644 --- a/crates/crab-storage/src/test_support.rs +++ b/crates/crab-storage/src/test_support.rs @@ -68,6 +68,7 @@ pub struct CountingObjectStore { body_reads_enabled: AtomicBool, blocked_body_paths: std::sync::Mutex>, requests: std::sync::Mutex>, + put_cancel: std::sync::Mutex>, } impl CountingObjectStore { @@ -84,9 +85,19 @@ impl CountingObjectStore { body_reads_enabled: AtomicBool::new(true), blocked_body_paths: std::sync::Mutex::new(HashSet::new()), requests: std::sync::Mutex::new(Vec::new()), + put_cancel: std::sync::Mutex::new(None), } } + /// Cancel once after a successful PUT at or beneath the selected prefix. + pub fn cancel_after_put(&self, prefix: Path, cancel: tokio_util::sync::CancellationToken) { + let mut hook = self + .put_cancel + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + *hook = Some((prefix, cancel)); + } + /// Enable or disable range and full-body reads while continuing to allow HEAD. pub fn set_body_reads_enabled(&self, enabled: bool) { self.body_reads_enabled.store(enabled, Ordering::Release); @@ -207,7 +218,19 @@ impl ObjectStore for CountingObjectStore { options: PutOptions, ) -> object_store::Result { self.puts.fetch_add(1, Ordering::AcqRel); - self.inner.put_opts(location, payload, options).await + let result = self.inner.put_opts(location, payload, options).await?; + let mut hook = self + .put_cancel + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if hook + .as_ref() + .is_some_and(|(prefix, _)| location.prefix_matches(prefix)) + && let Some((_, cancel)) = hook.take() + { + cancel.cancel(); + } + Ok(result) } async fn put_multipart_opts( From 0bb921f6d3a204220a1b0821f9bc660bd747fdd8 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 23:44:54 -0700 Subject: [PATCH 116/127] fix(push): remove unused advisory state writes Capsule pack selection uses captured remote refs. Rewriting unchanged push-state could report failure after a committed ref; keep that advisory contract in its released journal helper only. Extend the RustFS ingestion smoke with an unwritable advisory path and verify the canonical SQLite payload cache instead of requiring initialization of the retired receipt warm index. --- .../docs/architecture/indexed-remote-reads.md | 12 ++++++++++ .../e2e/run_add_commit_push_rustfs_smoke.py | 24 ++++++++++++++++--- crab/src/cmd/push.rs | 6 ----- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 0f3c6bbdc..b0713245b 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2607,3 +2607,15 @@ new and existing shared refs, direct/coordinated CLI preparation, exact staging retirement after late cancellation, and imported payload reconstruction. The shared cold-index fixture uses the capsule hint owner while preserving current source-catalog, exact provider-identity and complete reconstruction assertions. + +The native capsule command does not read or update the released advisory +`.crab/push-state` hint. Incremental pack selection uses the captured remote ref +view. Removing its redundant load/save prevents a local hint write from rejecting +an already visible push. The existing RustFS ingestion smoke holds an unwritable +hint path across native publication and independent cold reconstruction. The +released journal helper retains its own advisory-state contract. + +The ingestion smoke checks populated shard/xorb entries in the current SQLite +payload catalog and absence of the released receipt warm index. The capsule +classifier uses the authenticated hint tree, so requiring that old index file +would incorrectly demand reinitialization of the retired metadata path. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index c2e018145..919015faf 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -1681,7 +1681,11 @@ def run_case(self, case_name: str, use_crab_add: bool) -> None: self.run_git(repo, ["commit", "-m", f"{case_name}: add staged binaries"]) if use_crab_add: - self.run_crab( + # A committed capsule must not fail while rewriting an unused + # advisory pack-walk hint. Leave the fault through cold reconstruction. + advisory_state = repo / ".crab" / "push-state" + advisory_state.mkdir() + pushed = self.run_crab( repo, [ "push", @@ -1694,6 +1698,12 @@ def run_case(self, case_name: str, use_crab_add: bool) -> None: name=f"{case_name} crab push", timeout=self.args.push_timeout, ) + summary = json.loads(self.read_stdout(pushed))["data"] + self.check( + f"{case_name}-advisory-state-fault-preserves-success", + summary["refs_pushed"] == 1 and advisory_state.is_dir(), + summary, + ) else: self.run_git( repo, @@ -1822,10 +1832,18 @@ def run_case(self, case_name: str, use_crab_add: bool) -> None: ) sqlite_files = self.sqlite_cache_files() + # Capsule lookup does not initialize the released receipt warm index. + # Prove the canonical payload cache holds actual shards and xorbs. + catalog = self.cache_dir / ".catalog.sqlite" + with sqlite3.connect(f"file:{catalog}?mode=ro", uri=True) as connection: + cached = dict(connection.execute( + "SELECT family, SUM(size) FROM cache_entries GROUP BY family" + )) self.check( f"{case_name}-sqlite-cache-used", - "chunk-index.sqlite" in sqlite_files, - {"sqlite_files": sqlite_files}, + cached.get("shard", 0) > 0 and cached.get("xorb", 0) > 0 + and "chunk-index.sqlite" not in sqlite_files, + {"sqlite_files": sqlite_files, "cached_bytes": cached}, ) redb_files = self.redb_cache_files() self.check( diff --git a/crab/src/cmd/push.rs b/crab/src/cmd/push.rs index 6bbd1ab4c..884521ec7 100644 --- a/crab/src/cmd/push.rs +++ b/crab/src/cmd/push.rs @@ -24,7 +24,6 @@ use crate::git::push::{ PushConfig, PushFailureStage, PushRejectReason, PushResult, RefPushOutcome, configure_active_active_push_coordinator, record_push_audit_event, }; -use crate::git::push_state::PushState; use crate::git::remote_helper::{AGENT_REBASE_FETCH_REF_FILTERING_ENV, PushSpec}; use crate::git::url::CrabUrl; use crate::replication::StoreResolver; @@ -520,9 +519,6 @@ async fn run_push_once( "starting push" ); - // Load push state for incremental walk. - let push_state = PushState::load(&repo_root); - // Dry-run: print what would be pushed and return. if args.dry_run && !args.follow_tags { print_dry_run(&remote_name, &remote_url, &specs); @@ -738,8 +734,6 @@ async fn run_push_once( ) { warn!(%err, "failed to append push audit event"); } - push_state.save(&repo_root)?; - let elapsed = start.elapsed(); let integration = push_integration_summary(args, integration_retries, integration_retry_stages); From ae1c2630072ffad24c14c2535a697be4a4cb33d2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 00:07:24 -0700 Subject: [PATCH 117/127] fix(metadata): keep spill setup storage-owned Move platform temporary-root selection to storage, retain the spill byte ceiling and cleanup ownership, and keep metadata tests free of output policy. Use the supported fixed-array slice API without changing extent validation. --- .../pointer/object/reader/inspection.rs | 4 +++- .../pointer/object/reader/inspection/join.rs | 22 +++++++++++++------ .../capsule_protocol/pointer/object/tests.rs | 1 - crates/crab-storage/README.md | 6 ++--- crates/crab-storage/src/scratch.rs | 7 ++++++ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs index d4b6f8a25..8c46edfd8 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs @@ -16,7 +16,9 @@ impl CatalogPages { ) -> Result> { let mut work = PageWork::new(maximum, 0); self.open(store, &mut work).await?; - let mut join = Join::create(&std::env::temp_dir(), maximum).await?; + let space = + crab_storage::scratch::ScratchSpace::temporary(maximum.saturating_mul(4)).await?; + let mut join = Join::new(space); let checked = self.inspect(store, &mut work, &mut join).await; join.finish(checked).await?; Ok(work diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs index b5986d6a8..b574ff38b 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs @@ -148,7 +148,9 @@ fn decode(bytes: &[u8]) -> Result> { return Err(corrupt("inspection spill block count differs")); } bytes[12..] - .chunks_exact(ROW_BYTES) + .as_chunks::() + .0 + .iter() .map(|row| { let key: [u8; 33] = row[..33] .try_into() @@ -198,12 +200,12 @@ pub(super) struct Join { } impl Join { - pub(super) async fn create(parent: &std::path::Path, maximum: u64) -> Result { - Ok(Self { - space: ScratchSpace::create(parent, maximum.saturating_mul(4)).await?, + pub(super) fn new(space: ScratchSpace) -> Self { + Self { + space, buffer: Vec::with_capacity(BLOCK_ROWS), levels: Vec::new(), - }) + } } pub(super) async fn add(&mut self, row: Record) -> Result<()> { @@ -337,7 +339,10 @@ mod tests { async fn spilled_join_uses_the_lowest_owner_across_separate_runs() { for correct in [false, true] { let parent = tempfile::tempdir().unwrap(); - let mut join = Join::create(parent.path(), 8 * 1024 * 1024).await.unwrap(); + let space = ScratchSpace::create(parent.path(), 32 * 1024 * 1024) + .await + .unwrap(); + let mut join = Join::new(space); let mut high = [9; 40]; let mut low = high; low[..32].fill(1); @@ -367,7 +372,10 @@ mod tests { #[tokio::test] async fn prior_scan_failure_drains_and_removes_written_runs() { let parent = tempfile::tempdir().unwrap(); - let mut join = Join::create(parent.path(), 8 * 1024 * 1024).await.unwrap(); + let space = ScratchSpace::create(parent.path(), 32 * 1024 * 1024) + .await + .unwrap(); + let mut join = Join::new(space); for index in 0..5000u32 { let mut key = [0; 32]; key[28..].copy_from_slice(&index.to_be_bytes()); diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs index 1b3044520..9d472642c 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -569,7 +569,6 @@ async fn streamed_inspection_bounds_io_for_distinct_recipes_beyond_the_page_cach .iter() .all(|op| op.bytes_written == 0) ); - eprintln!("catalog_maintenance_io full_bytes={full} streamed_bytes={streamed}"); assert!( streamed <= 4 * full, "streaming must not trade catalog-sized memory for repeated cold joins: {streamed} versus {full}" diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index aa20227ed..bcab8aaa5 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -197,9 +197,9 @@ caller policies; all share the provider construction implementation. ## Private maintenance spills -`scratch::ScratchSpace` owns bounded immutable local spill files below an existing -private temporary parent. It admits live bytes before creation and verifies exact -length, EOF and Blake3 on reads. References cannot cross workspace identities. +`scratch::ScratchSpace` owns bounded immutable local spill files. `temporary` +selects platform temporary storage; `create` accepts an existing private parent. +It admits live bytes before creation and verifies exact length, EOF and Blake3 on reads. References cannot cross workspace identities. Catalog maintenance uses it to replace repeated remote joins with sorted local reductions. This does not change foreground metadata reads or provider routing. diff --git a/crates/crab-storage/src/scratch.rs b/crates/crab-storage/src/scratch.rs index d25bd4407..f978aeabc 100644 --- a/crates/crab-storage/src/scratch.rs +++ b/crates/crab-storage/src/scratch.rs @@ -43,6 +43,13 @@ fn worker(source: impl std::error::Error + Send + Sync + 'static) -> StorageErro } impl ScratchSpace { + /// Create private spill storage in the platform temporary directory. + /// + /// The live-byte ceiling and worker/cleanup ownership match `create`. + pub async fn temporary(maximum: u64) -> Result { + Self::create(&std::env::temp_dir(), maximum).await + } + /// Create a private child of an existing caller-owned directory with a live-byte ceiling. /// /// The parent must remain private to the caller. Await `close` on normal and From 4934b5e42c213b188ef97490551a7b79559754e1 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 00:38:14 -0700 Subject: [PATCH 118/127] fix(gc): drain admitted capsule deletes --- .../docs/architecture/indexed-remote-reads.md | 11 ++ crab/src/cmd/gc/captured_index_tests.rs | 31 +++- .../gc/captured_index_tests/cancellation.rs | 152 ++++++++++++++++++ crab/src/cmd/gc/mod.rs | 61 +++---- 4 files changed, 220 insertions(+), 35 deletions(-) create mode 100644 crab/src/cmd/gc/captured_index_tests/cancellation.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index b0713245b..369b10f4c 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2619,3 +2619,14 @@ The ingestion smoke checks populated shard/xorb entries in the current SQLite payload catalog and absence of the released receipt warm index. The capsule classifier uses the authenticated hint tree, so requiring that old index file would incorrectly demand reinitialization of the retired metadata path. + +## Capsule GC drains admitted provider deletes + +Repository capsule GC uses the shared bounded deletion executor. Cancellation +stops admission of subsequent batches; every admitted provider response drains +before the root fence and repository sweep lease release. Delete-time HEAD checks +still protect identity, size and grace, and retained candidates do not count as +reclaimed bytes. The regression holds an accepted delete independently of its +client future, cancels on a sibling deletion and requires admission to remain +fenced until the held response completes. Successor writer admission must then +be available after the cancelled sweep returns. diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index 2703dbfb7..cedcdc4e3 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -13,7 +13,7 @@ use crab_read::capsule_protocol::{CapsuleReadLimits, CapsuleRepositoryView}; use crab_remote_git::{RemoteGitRuntime, RepositoryIdentity, RepositoryOptions}; use crab_storage::StoreLayout; use crab_xet::hash::MerkleHash; -use futures_util::{TryStreamExt, stream::BoxStream}; +use futures_util::{StreamExt, TryStreamExt, stream::BoxStream}; use object_store::{ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, memory::InMemory, path::Path, @@ -23,13 +23,16 @@ use tokio_util::sync::CancellationToken; use super::{GcArgs, sweep_capsule_objects}; use crate::storage::store::Store; +mod cancellation; + const LIMIT: u64 = 8 * 1024 * 1024; // Immutable bodies can predate their first reader by years. Alter only their // provider timestamps; native pack validation and conditional publication stay real. #[derive(Debug)] struct AgedObjects { - inner: InMemory, + inner: Arc, + deletes: Option>, paths: Mutex>, modified: SystemTime, list_calls: Mutex>, @@ -72,6 +75,15 @@ impl ObjectStore for AgedObjects { &self, paths: BoxStream<'static, object_store::Result>, ) -> BoxStream<'static, object_store::Result> { + if let Some(gate) = &self.deletes { + let gate = gate.clone(); + let inner = self.inner.clone(); + return Box::pin(paths.then(move |path| { + let gate = gate.clone(); + let inner = inner.clone(); + async move { gate.delete(inner, path?).await } + })); + } self.inner.delete_stream(paths) } @@ -194,7 +206,8 @@ async fn view(layout: &StoreLayout) -> CapsuleRepositoryVie async fn cold_capture_after_gc(replace_readiness: bool) -> crab_metadata::error::Result { let backend = Arc::new(AgedObjects { - inner: InMemory::new(), + inner: Arc::new(InMemory::new()), + deletes: None, paths: Mutex::new(HashSet::new()), modified: SystemTime::now() - Duration::from_secs(7200), list_calls: Mutex::new(Vec::new()), @@ -318,7 +331,8 @@ async fn gc_preserves_cold_catalog_sessions_without_browse_indexes() { XorbCatalogEntry, XorbChunkEntry, }; let backend = Arc::new(AgedObjects { - inner: InMemory::new(), + inner: Arc::new(InMemory::new()), + deletes: None, paths: Mutex::new(HashSet::new()), modified: SystemTime::now() - Duration::from_secs(7200), list_calls: Mutex::new(Vec::new()), @@ -449,7 +463,8 @@ async fn gc_preserves_cold_catalog_sessions_without_browse_indexes() { #[tokio::test] async fn capsule_gc_lists_each_candidate_namespace_once() { let backend = Arc::new(AgedObjects { - inner: InMemory::new(), + inner: Arc::new(InMemory::new()), + deletes: None, paths: Mutex::new(HashSet::new()), modified: SystemTime::now(), list_calls: Mutex::new(Vec::new()), @@ -502,7 +517,8 @@ async fn capsule_gc_lists_each_candidate_namespace_once() { #[tokio::test] async fn capsule_gc_preserves_metadata_outside_owned_index_directories() { let backend = Arc::new(AgedObjects { - inner: InMemory::new(), + inner: Arc::new(InMemory::new()), + deletes: None, paths: Mutex::new(HashSet::new()), modified: SystemTime::now() - Duration::from_secs(7200), list_calls: Mutex::new(Vec::new()), @@ -556,7 +572,8 @@ async fn capsule_gc_preserves_metadata_outside_owned_index_directories() { async fn checkpoint_catalog_gc_preserves_roots_and_grace_period() { use crab_metadata::capsule_protocol::{FileCatalogEntry, PointerCatalog, ShardCatalogEntry}; let backend = Arc::new(AgedObjects { - inner: InMemory::new(), + inner: Arc::new(InMemory::new()), + deletes: None, paths: Mutex::new(HashSet::new()), modified: SystemTime::now() - Duration::from_secs(7200), list_calls: Mutex::new(Vec::new()), diff --git a/crab/src/cmd/gc/captured_index_tests/cancellation.rs b/crab/src/cmd/gc/captured_index_tests/cancellation.rs new file mode 100644 index 000000000..4dcae588b --- /dev/null +++ b/crab/src/cmd/gc/captured_index_tests/cancellation.rs @@ -0,0 +1,152 @@ +use super::*; +use object_store::ObjectStoreExt; + +#[derive(Debug)] +pub(super) struct DeleteGate { + blocked: Path, + trigger: Path, + started: tokio::sync::Notify, + release: tokio::sync::Notify, + finished: tokio::sync::Notify, + cancel: CancellationToken, +} + +impl DeleteGate { + pub(super) async fn delete( + self: Arc, + inner: Arc, + path: Path, + ) -> object_store::Result { + if path == self.blocked { + let (reply, complete) = tokio::sync::oneshot::channel(); + let gate = self.clone(); + // A sent provider request can commit after its client future drops. + // Keep that side effect independent from the response waiter. + tokio::spawn(async move { + gate.release.notified().await; + let result = inner.delete(&path).await.map(|()| path); + let _ = reply.send(result); + gate.finished.notify_one(); + }); + self.started.notify_one(); + return complete.await.unwrap(); + } + if path == self.trigger { + self.started.notified().await; + self.cancel.cancel(); + } + inner.delete(&path).await?; + Ok(path) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn capsule_gc_drains_admitted_deletes_before_releasing_its_fence() { + assert_gc_drains(Arc::new(InMemory::new()), "org/cancel-gc").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] +async fn live_capsule_gc_drains_admitted_deletes_before_releasing_its_fence() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").unwrap(); + let store = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + let prefix = format!("gc-drain/{}", uuid::Uuid::now_v7()); + assert_gc_drains(store.inner().clone(), &prefix).await; +} + +async fn assert_gc_drains(inner: Arc, prefix: &str) { + let blocked = Path::from(format!("{prefix}/v2/capsules/{}", "1".repeat(64))); + let trigger = Path::from(format!("{prefix}/v2/capsules/{}", "2".repeat(64))); + let cancel = CancellationToken::new(); + let gate = Arc::new(DeleteGate { + blocked: blocked.clone(), + trigger: trigger.clone(), + started: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + finished: tokio::sync::Notify::new(), + cancel: cancel.clone(), + }); + let backend = Arc::new(AgedObjects { + inner, + deletes: Some(gate.clone()), + paths: Mutex::new(HashSet::from([blocked.clone(), trigger.clone()])), + modified: SystemTime::now() - Duration::from_secs(7200), + list_calls: Mutex::new(Vec::new()), + }); + let store = Store::new(backend); + let router = crate::storage::StoreLayout::new(store.clone(), prefix.to_owned()); + let layout = StoreLayout::new(store.as_storage().clone(), prefix.to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + for path in [blocked, trigger] { + store + .put(&path, Bytes::from_static(b"orphan")) + .await + .unwrap(); + } + let args = GcArgs { + delete_concurrency: 2, + ..GcArgs::default() + }; + let gc_store = store.clone(); + let gc_cancel = cancel.clone(); + let protected = HashSet::new(); + let grace = Duration::from_secs(3600); + let mut sweep = std::pin::pin!(super::super::run_repo_remote_gc( + &args, &gc_store, &router, &protected, &gc_cancel, grace, None, + )); + let mut completed = None; + tokio::select! { + cancelled = tokio::time::timeout(Duration::from_secs(5), cancel.cancelled()) => { + cancelled.unwrap(); + } + result = &mut sweep => completed = Some(result), + } + assert!( + cancel.is_cancelled(), + "GC never reached the delete gate: {completed:?}" + ); + if completed.is_none() + && let Ok(result) = tokio::time::timeout(Duration::from_millis(100), &mut sweep).await + { + completed = Some(result); + } + let returned_early = completed.is_some(); + let writer = crab_coordination::GcFenceLease::acquire_writer( + store.inner(), + prefix, + Duration::from_secs(60), + ) + .await; + let fenced = writer.is_err(); + if let Ok(writer) = writer { + writer.release().await.unwrap(); + } + gate.release.notify_one(); + let result = match completed { + Some(result) => result, + None => sweep.await, + }; + tokio::time::timeout(Duration::from_secs(5), gate.finished.notified()) + .await + .unwrap(); + assert!( + !returned_early && fenced, + "GC released admission while an accepted provider delete was unresolved" + ); + assert!(matches!( + result, + Err(crate::core::error::CrabError::Cancelled) + )); + let writer = crab_coordination::GcFenceLease::acquire_writer( + store.inner(), + prefix, + Duration::from_secs(60), + ) + .await + .unwrap(); + writer.release().await.unwrap(); +} diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index 7f18d9a2e..40c7fa939 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3365,40 +3365,45 @@ async fn sweep_capsule_objects( force: false, }; let concurrency = args.delete_concurrency.max(1); - let mut deletes = futures_util::stream::iter(candidates.iter().map(|object| { - let derived_index = derived_index_keys.contains(object.location.as_ref()); - let meta = ObjectMeta { + let objects = candidates + .into_iter() + .map(|object| ObjectMeta { key: object.location.to_string(), size: object.size, last_modified: SystemTime::from(object.last_modified), - e_tag: object.e_tag.clone(), - version: object.version.clone(), + e_tag: object.e_tag, + version: object.version, storage_class: None, transitioned_at: None, - }; - let deleter = &deleter; - async move { - ( - meta.size, - derived_index, - deleter.delete_candidate(&meta, policy).await, - ) - } - })) - .buffer_unordered(concurrency); - while let Some((size, derived_index, result)) = deletes.next().await { - check_cancelled(cancel)?; - // HEAD may retain a candidate whose identity or freshness changed - // after LIST. Planned bytes are not reclaimed in that case. - if result? == CandidateDelete::Deleted { - if derived_index { - accounting.derived_index_objects_deleted += 1; - } else { - accounting.packs_deleted += 1; - } - accounting.bytes_reclaimed = accounting.bytes_reclaimed.saturating_add(size); - } + }) + .collect::>(); + let mut deleted = GcOutcome::default(); + // A sent provider delete can finish after its request future is dropped. + // Keep admission until the shared executor drains every admitted result, + // including cancellation and returned errors. + let result = execute_deletes( + &objects, + cancel, + concurrency, + &deleter, + &mut deleted, + None, + Some(policy), + ) + .await; + check_cancelled(cancel)?; + if let Some(error) = result.first_error { + return Err(error); } + accounting.derived_index_objects_deleted = result + .deleted_keys + .iter() + .filter(|key| derived_index_keys.contains(key.as_str())) + .count() as u64; + accounting.packs_deleted = result.deleted_count - accounting.derived_index_objects_deleted; + // HEAD can retain objects whose identity or freshness changed after LIST. + // Their planned bytes never enter the executor's reclaimed-byte count. + accounting.bytes_reclaimed = deleted.bytes_reclaimed; } Ok(GcOutcome { list_requests: 8, From 90bc98d8f8e88c7aa7ec6b385e598663a06b028f Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 01:17:20 -0700 Subject: [PATCH 119/127] fix(gc): sweep owned view-local hint pages --- .../docs/architecture/indexed-remote-reads.md | 17 + crab/src/cmd/gc/captured_index_tests.rs | 7 + .../gc/captured_index_tests/cancellation.rs | 70 ++-- .../gc/captured_index_tests/scoped_hints.rs | 312 ++++++++++++++++++ crab/src/cmd/gc/mod.rs | 73 +++- 5 files changed, 443 insertions(+), 36 deletions(-) create mode 100644 crab/src/cmd/gc/captured_index_tests/scoped_hints.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 369b10f4c..5f160f2c8 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2630,3 +2630,20 @@ reclaimed bytes. The regression holds an accepted delete independently of its client future, cancels on a sibling deletion and requires admission to remain fenced until the held response completes. Successor writer admission must then be available after the cancelled sweep returns. + + +## Auth-issued view-local hint maintenance + +Repository GC sweeps capsule hint pages only when its store carries the matching +validated storage scope: both repository and global prefixes must match the +layout, and global content must be private to that repository. Explicit layout +strings alone do not establish ownership. The auth credential-response validator +and managed-grant builder issue that prefix pair. + +Destructive sweeps acquire the view-local global lease before the repository +lease, matching all hint publishers. They authenticate the current hint root and +every page before selecting garbage, preserve live pages and exact placement IDs, +and retain immutable-object grace even with force. Missing or corrupt current +pages abort before deletion. Admitted delete responses drain before either +lease releases; failed repository admission still releases the global lease. +Default shared-global hints remain owned by bucket maintenance. diff --git a/crab/src/cmd/gc/captured_index_tests.rs b/crab/src/cmd/gc/captured_index_tests.rs index cedcdc4e3..ee85e53f2 100644 --- a/crab/src/cmd/gc/captured_index_tests.rs +++ b/crab/src/cmd/gc/captured_index_tests.rs @@ -1,3 +1,9 @@ +#![expect( + clippy::unwrap_used, + clippy::panic, + reason = "test setup and assertions" +)] + use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime}; @@ -24,6 +30,7 @@ use super::{GcArgs, sweep_capsule_objects}; use crate::storage::store::Store; mod cancellation; +mod scoped_hints; const LIMIT: u64 = 8 * 1024 * 1024; diff --git a/crab/src/cmd/gc/captured_index_tests/cancellation.rs b/crab/src/cmd/gc/captured_index_tests/cancellation.rs index 4dcae588b..8aadf3521 100644 --- a/crab/src/cmd/gc/captured_index_tests/cancellation.rs +++ b/crab/src/cmd/gc/captured_index_tests/cancellation.rs @@ -42,7 +42,12 @@ impl DeleteGate { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn capsule_gc_drains_admitted_deletes_before_releasing_its_fence() { - assert_gc_drains(Arc::new(InMemory::new()), "org/cancel-gc").await; + assert_gc_drains(Arc::new(InMemory::new()), "org/cancel-gc", false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn scoped_capsule_gc_drains_deletes_before_releasing_either_domain() { + assert_gc_drains(Arc::new(InMemory::new()), "org/scoped-cancel-gc", true).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -52,11 +57,13 @@ async fn live_capsule_gc_drains_admitted_deletes_before_releasing_its_fence() { let store = crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) .unwrap(); - let prefix = format!("gc-drain/{}", uuid::Uuid::now_v7()); - assert_gc_drains(store.inner().clone(), &prefix).await; + for scoped in [false, true] { + let prefix = format!("gc-drain/{}", uuid::Uuid::now_v7()); + assert_gc_drains(store.inner().clone(), &prefix, scoped).await; + } } -async fn assert_gc_drains(inner: Arc, prefix: &str) { +async fn assert_gc_drains(inner: Arc, prefix: &str, scoped: bool) { let blocked = Path::from(format!("{prefix}/v2/capsules/{}", "1".repeat(64))); let trigger = Path::from(format!("{prefix}/v2/capsules/{}", "2".repeat(64))); let cancel = CancellationToken::new(); @@ -75,7 +82,18 @@ async fn assert_gc_drains(inner: Arc, prefix: &str) { modified: SystemTime::now() - Duration::from_secs(7200), list_calls: Mutex::new(Vec::new()), }); - let store = Store::new(backend); + let global = format!("{prefix}/.crab"); + let mut domains = vec![prefix]; + let mut store = Store::new(backend); + if scoped { + store = store.with_storage_scope(crab_types::storage::StorageScope { + repo_prefix: prefix.to_owned(), + global_prefix: global.clone(), + source_repo: prefix.to_owned(), + scope_hash: "1".repeat(64), + }); + domains.push(&global); + } let router = crate::storage::StoreLayout::new(store.clone(), prefix.to_owned()); let layout = StoreLayout::new(store.as_storage().clone(), prefix.to_owned()); crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") @@ -115,15 +133,21 @@ async fn assert_gc_drains(inner: Arc, prefix: &str) { completed = Some(result); } let returned_early = completed.is_some(); - let writer = crab_coordination::GcFenceLease::acquire_writer( - store.inner(), - prefix, - Duration::from_secs(60), - ) - .await; - let fenced = writer.is_err(); - if let Ok(writer) = writer { - writer.release().await.unwrap(); + let mut fenced = true; + for domain in &domains { + let writer = crab_coordination::GcFenceLease::acquire_writer( + store.inner(), + domain, + Duration::from_secs(60), + ) + .await; + fenced &= matches!( + &writer, + Err(crab_coordination::error::CoordinationError::GcFenceHeld { sweep: true, .. }) + ); + if let Ok(writer) = writer { + writer.release().await.unwrap(); + } } gate.release.notify_one(); let result = match completed { @@ -141,12 +165,14 @@ async fn assert_gc_drains(inner: Arc, prefix: &str) { result, Err(crate::core::error::CrabError::Cancelled) )); - let writer = crab_coordination::GcFenceLease::acquire_writer( - store.inner(), - prefix, - Duration::from_secs(60), - ) - .await - .unwrap(); - writer.release().await.unwrap(); + for domain in domains { + let writer = crab_coordination::GcFenceLease::acquire_writer( + store.inner(), + domain, + Duration::from_secs(60), + ) + .await + .unwrap(); + writer.release().await.unwrap(); + } } diff --git a/crab/src/cmd/gc/captured_index_tests/scoped_hints.rs b/crab/src/cmd/gc/captured_index_tests/scoped_hints.rs new file mode 100644 index 000000000..ca22002c4 --- /dev/null +++ b/crab/src/cmd/gc/captured_index_tests/scoped_hints.rs @@ -0,0 +1,312 @@ +use super::*; +use crab_metadata::capsule_protocol::{ + CapsuleChunkIndex, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + write_capsule_chunk_candidates, +}; +use crab_metadata::receipts::OriginReceipt; + +#[derive(Clone, Copy, Debug)] +enum Scope { + Issued, + Explicit, + Shared, +} + +async fn fixture( + scope: Scope, + inner: Arc, + repo: &str, +) -> (Arc, Store, StoreLayout) { + let backend = Arc::new(AgedObjects { + inner, + deletes: None, + paths: Mutex::new(HashSet::new()), + modified: SystemTime::now() - Duration::from_secs(7200), + list_calls: Mutex::new(Vec::new()), + }); + let mut store = Store::new(backend.clone()); + let global = if matches!(scope, Scope::Shared) { + ".crab".to_owned() + } else { + format!("{repo}/.crab") + }; + if matches!(scope, Scope::Issued) { + store = store.with_storage_scope(crab_types::storage::StorageScope { + repo_prefix: repo.to_owned(), + global_prefix: global.clone(), + source_repo: "org/source".to_owned(), + scope_hash: "1".repeat(64), + }); + } + let layout = + StoreLayout::with_global_prefix(store.as_storage().clone(), repo.to_owned(), global); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + (backend, store, layout) +} + +async fn publish_hints( + layout: &StoreLayout, + seed: u64, +) -> (MerkleHash, [u8; 32]) { + let xorb = MerkleHash::from([seed; 4]); + let chunk = MerkleHash::from([1; 4]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 80, + "4".repeat(64), + vec![XorbChunkEntry::new(chunk.hex(), 7)], + ), + ) + .unwrap(); + catalog + .insert_shard("2".repeat(64), ShardCatalogEntry::new(10, vec![xorb.hex()])) + .unwrap(); + let origin = OriginReceipt::new( + "canonical-origin".to_owned(), + layout.xorb_path(&xorb).to_string(), + xorb.into(), + [5; 32], + 80, + Some("etag".to_owned()), + None, + ); + let cancel = CancellationToken::new(); + let leases = crate::maintenance::GcWriterLeases::acquire( + &Store::from_storage(layout.store().clone()), + layout.global_prefix(), + layout.repo_prefix(), + &cancel, + ) + .await + .unwrap(); + let result = + write_capsule_chunk_candidates(layout, &catalog, &HashMap::from([(xorb, origin)]), &|| { + false + }) + .await; + leases.release().await.unwrap(); + result.unwrap(); + let candidates = CapsuleChunkIndex::new(layout) + .candidates(layout.store(), &[chunk], LIMIT) + .await + .unwrap(); + ( + chunk, + candidates.placements[&<[u8; 32]>::from(chunk)].placement_id(), + ) +} + +async fn sweep( + store: &Store, + layout: &StoreLayout, + force: bool, +) -> crate::core::error::Result { + let router = crate::storage::StoreLayout::with_global_prefix( + store.clone(), + layout.repo_prefix().to_owned(), + layout.global_prefix().to_owned(), + ); + super::super::run_repo_remote_gc( + &GcArgs { + force, + yes: true, + ..GcArgs::default() + }, + store, + &router, + &HashSet::new(), + &CancellationToken::new(), + Duration::from_secs(3600), + None, + ) + .await +} + +#[tokio::test] +async fn scoped_hint_gc_requires_issued_ownership_and_preserves_live_exact_ids_and_grace() { + for scope in [Scope::Issued, Scope::Explicit, Scope::Shared] { + for force in [false, true] { + let state = fixture(scope, Arc::new(InMemory::new()), "org/scoped-hints").await; + assert_hint_gc(scope, force, state).await; + } + } +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] +async fn live_scoped_hint_gc_preserves_live_exact_ids_and_grace() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").unwrap(); + let store = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + for force in [false, true] { + let prefix = format!("scoped-hint-gc/{}", uuid::Uuid::now_v7()); + let state = fixture(Scope::Issued, store.inner().clone(), &prefix).await; + assert_hint_gc(Scope::Issued, force, state).await; + } +} + +async fn assert_hint_gc( + scope: Scope, + force: bool, + state: (Arc, Store, StoreLayout), +) { + let (backend, store, layout) = state; + let (chunk, old_id) = publish_hints(&layout, 3).await; + publish_hints(&layout, 4).await; + let (_, live) = CapsuleChunkIndex::new(&layout) + .objects(layout.store(), LIMIT) + .await + .unwrap(); + let live = live + .into_iter() + .map(|(path, _)| path) + .collect::>(); + let before = store + .list_prefix(&layout.capsule_chunk_pages_prefix()) + .await + .unwrap(); + let before = before + .into_iter() + .map(|meta| meta.location) + .collect::>(); + assert!( + before.len() > live.len(), + "fixture must have superseded pages" + ); + backend.paths.lock().unwrap().extend(before.iter().cloned()); + let fresh = Path::from(format!( + "{}/aa/{}", + layout.capsule_chunk_pages_prefix(), + "a".repeat(64) + )); + let unrelated = Path::from(format!( + "other/.crab/capsule-chunk-pages/bb/{}", + "b".repeat(64) + )); + for path in [&fresh, &unrelated] { + store + .put(path, Bytes::from_static(b"unpublished page")) + .await + .unwrap(); + } + backend.paths.lock().unwrap().insert(unrelated.clone()); + let outcome = sweep(&store, &layout, force).await.unwrap(); + let after = store + .list_prefix(&layout.capsule_chunk_pages_prefix()) + .await + .unwrap(); + let after = after + .into_iter() + .map(|meta| meta.location) + .collect::>(); + let mut expected = if matches!(scope, Scope::Issued) { + live + } else { + before.clone() + }; + expected.insert(fresh); + assert_eq!(after, expected, "{scope:?}, force={force}"); + assert_eq!( + outcome.derived_index_objects_deleted, + (before.len() + 1 - after.len()) as u64 + ); + assert!(store.head(&unrelated).await.is_ok()); + let candidates = CapsuleChunkIndex::new(&layout) + .candidates_by_id(layout.store(), &[(chunk, old_id)], LIMIT) + .await + .unwrap(); + assert_eq!( + candidates.placements[&<[u8; 32]>::from(chunk)].placement_id(), + old_id + ); +} + +#[tokio::test] +async fn broken_scoped_hint_closure_aborts_before_any_sweep_and_releases_admission() { + for missing in [true, false] { + let (backend, store, layout) = + fixture(Scope::Issued, Arc::new(InMemory::new()), "org/scoped-hints").await; + publish_hints(&layout, 3).await; + let (_, pages) = CapsuleChunkIndex::new(&layout) + .objects(layout.store(), LIMIT) + .await + .unwrap(); + let page = &pages[0].0; + if missing { + store.delete(page).await.unwrap(); + } else { + // The product store rejects immutable-key replacement. Inject a + // provider fault below that guard so GC must detect the damage. + backend + .inner + .put_opts( + page, + Bytes::from_static(b"damaged page").into(), + PutOptions::default(), + ) + .await + .unwrap(); + } + let orphan = layout.capsule_path(&"f".repeat(64)); + store + .put(&orphan, Bytes::from_static(b"unreferenced capsule")) + .await + .unwrap(); + backend.paths.lock().unwrap().insert(orphan.clone()); + assert!( + sweep(&store, &layout, false).await.is_err(), + "missing={missing}" + ); + assert!(store.head(&orphan).await.is_ok()); + for domain in [layout.global_prefix(), layout.repo_prefix()] { + let writer = crab_coordination::GcFenceLease::acquire_writer( + store.inner(), + domain, + Duration::from_secs(60), + ) + .await + .unwrap(); + writer.release().await.unwrap(); + } + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + assert!(root.record().root().gc_fence().is_none()); + } +} + +#[tokio::test] +async fn scoped_hint_gc_respects_both_writer_domains_and_releases_failed_admission() { + for global in [true, false] { + let (_, store, layout) = + fixture(Scope::Issued, Arc::new(InMemory::new()), "org/scoped-hints").await; + let domain = if global { + layout.global_prefix() + } else { + layout.repo_prefix() + }; + let writer = crab_coordination::GcFenceLease::acquire_writer( + store.inner(), + domain, + Duration::from_secs(60), + ) + .await + .unwrap(); + assert!(matches!( + sweep(&store, &layout, false).await, + Err(crate::core::error::CrabError::PushLockHeld { .. }) + )); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + assert!(root.record().root().gc_fence().is_none()); + writer.release().await.unwrap(); + sweep(&store, &layout, false).await.unwrap(); + } +} diff --git a/crab/src/cmd/gc/mod.rs b/crab/src/cmd/gc/mod.rs index 40c7fa939..84004ed5e 100644 --- a/crab/src/cmd/gc/mod.rs +++ b/crab/src/cmd/gc/mod.rs @@ -3056,7 +3056,29 @@ pub async fn run_repo_remote_gc( grace_period: Duration, _jsonl_stream: Option<&std::sync::Mutex>>, ) -> Result { - run_capsule_gc( + if args.resume_run_id.is_some() { + return Err(CrabError::Configuration { + key: "gc.resume".to_owned(), + origin: "protocol-v2 GC completes under one root fence and has no journal resume mode" + .to_owned(), + }); + } + if args.force && !confirm_force(args)? { + return Ok(GcOutcome::default()); + } + check_cancelled(cancel)?; + // Auth-issued view-local content has one repository owner. Its hint pages + // need global admission before the repository sweep, matching publisher order. + // An arbitrary explicit layout does not establish that ownership. + let global = if !args.dry_run && owns_scoped_hint_pages(store, router) { + Some( + crate::maintenance::GcSweepLease::acquire(store, router.global_prefix(), cancel) + .await?, + ) + } else { + None + }; + let operation = run_capsule_gc( args, store, router, @@ -3064,7 +3086,24 @@ pub async fn run_repo_remote_gc( cancel, grace_period, ) - .await + .await; + let release = match global { + Some(lease) => lease.release().await, + None => Ok(()), + }; + match (operation, release) { + (Ok(outcome), Ok(())) => Ok(outcome), + (Err(error), _) | (Ok(_), Err(error)) => Err(error), + } +} + +fn owns_scoped_hint_pages(store: &Store, layout: &crab_storage::StoreLayout) -> bool { + store.storage_scope().is_some_and(|scope| { + !scope.repo_prefix.is_empty() + && scope.repo_prefix == layout.repo_prefix() + && scope.global_prefix == layout.global_prefix() + && scope.global_prefix == format!("{}/.crab", scope.repo_prefix) + }) } async fn run_capsule_gc( @@ -3077,17 +3116,6 @@ async fn run_capsule_gc( ) -> Result { const FENCE_TTL: Duration = Duration::from_secs(60 * 60); - if args.resume_run_id.is_some() { - return Err(CrabError::Configuration { - key: "gc.resume".to_owned(), - origin: "protocol-v2 GC completes under one root fence and has no journal resume mode" - .to_owned(), - }); - } - if args.force && !confirm_force(args)? { - return Ok(GcOutcome::default()); - } - check_cancelled(cancel)?; let started = Instant::now(); let snapshot_at = SystemTime::now(); let layout = crab_storage::StoreLayout::with_global_prefix( @@ -3251,6 +3279,22 @@ async fn sweep_capsule_objects( ) .await?; + let scoped_hints = owns_scoped_hint_pages(store, layout); + let hint_pages = if scoped_hints { + // Authenticate the entire current page closure before selecting garbage. + // Destructive sweeps hold global admission against hint-root publication. + let (_, live) = crab_metadata::capsule_protocol::CapsuleChunkIndex::new(layout) + .objects(layout.store(), u64::MAX) + .await?; + reachable.extend(live.into_iter().map(|(path, _)| path.to_string())); + store + .list_prefix(&layout.capsule_chunk_pages_prefix()) + .await? + } else { + Vec::new() + }; + check_cancelled(cancel)?; + let capsule_prefix = layout.repo_path("v2/capsules/"); let checkpoint_prefix = layout.repo_path("v2/checkpoints/"); let catalog_prefix = layout.capsule_catalog_prefix(); @@ -3305,6 +3349,7 @@ async fn sweep_capsule_objects( .any(|prefix| object.location.as_ref().starts_with(prefix)) }) .chain(browse_root_objects) + .chain(hint_pages) .chain(manifests.into_iter().filter(|object| { let key = object.location.as_ref(); key.starts_with(path_state_descriptor_prefix.as_ref()) @@ -3406,7 +3451,7 @@ async fn sweep_capsule_objects( accounting.bytes_reclaimed = deleted.bytes_reclaimed; } Ok(GcOutcome { - list_requests: 8, + list_requests: 8 + u64::from(scoped_hints), list_parallelism: 8, list_wall_seconds: started.elapsed().as_secs_f64(), dry_run: args.dry_run, From 0771e4c47ba6583dd5ee212c9d57fbfcc9f5cb95 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 02:41:18 -0700 Subject: [PATCH 120/127] perf(metadata): spill captured catalog roots --- .../docs/architecture/indexed-remote-reads.md | 27 +- crab/src/cmd/gc/bucket.rs | 27 +- .../src/capsule_protocol/layered/catalog.rs | 2 +- .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/pointer.rs | 43 ++- .../capsule_protocol/pointer/inspection.rs | 335 ++++++++++++++++++ .../pointer/inspection/tests.rs | 257 ++++++++++++++ .../pointer/object/reader/inspection.rs | 62 +++- .../capsule_protocol/pointer/object/tests.rs | 34 ++ .../inspection/join.rs => relations.rs} | 181 ++-------- .../src/capsule_protocol/pointer/spill.rs | 185 ++++++++++ crates/crab-read/src/capsule_protocol.rs | 42 +++ 12 files changed, 1027 insertions(+), 172 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs rename crates/crab-metadata/src/capsule_protocol/pointer/{object/reader/inspection/join.rs => relations.rs} (62%) create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/spill.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 5f160f2c8..5b226b25c 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2521,10 +2521,29 @@ spill growth; cleanup drains blocking workers. The existing temporary-directory setting controls placement, with no new config/env surface or dependency. Historical fsck uses the same streaming integrity inspection before checking pack sources, including catalogs replaced by an empty current catalog. It does -not retain complete historical recipe maps. Current fsck payload verification, -restore accounting, GC shard discovery and catalog-rewriting tools still use -full logical catalogs and remain separate conversion work. Materialization -admission continues to protect those callers retaining recipe copies. +not retain complete historical recipe maps. GC shard discovery now streams a +captured checkpoint and folds selected publication deltas through the same +private spill reducer. Capture rank selects the final file binding; all file +size and immutable descriptor conflicts still fail. A deferred relation join +checks final file-to-shard and shard-to-xorb closure before returning any shard +roots, including dependencies supplied by later refs. The checkpoint's physical +integrity inspection runs first, including reverse placement and alias checks. +No origin writes or snapshot recapture occur during this operation. + +The checkpoint reader retains page identities and bounded row windows, plus +one decoded recipe at a time. Complete physical admission precedes logical +reduction; each live recipe has separate materialization admission because its +copies are released before the next recipe. Normal materializers keep their +aggregate row budgets. Sorted runs use 512 KiB/4,096-row blocks (one +larger recipe may occupy its own block), binary merge levels and a live spill +ceiling of sixteen times the admitted catalog byte limit. The returned shard +list remains output-sized. This is bounded external reduction, not constant +space: physical page identities, spill handles, the largest recipe and output +roots still consume memory. Each selected packed delta still uses its existing +format-bounded decoder individually; streaming those deltas remains follow-up +work. Current fsck payload verification, restore accounting and catalog-rewriting +tools also retain full logical catalogs. Materialization admission continues to +protect those callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before this change; none contains the capsule codec. Unshipped checkpoint formats diff --git a/crab/src/cmd/gc/bucket.rs b/crab/src/cmd/gc/bucket.rs index cb8cb0965..c27f99ba4 100644 --- a/crab/src/cmd/gc/bucket.rs +++ b/crab/src/cmd/gc/bucket.rs @@ -1137,10 +1137,10 @@ async fn capsule_repository_roots( ) .await?; let root_digest = view.state_digest(); - let catalog = view.pointer_catalog().await?; + let shard_hashes = view.pointer_catalog_shard_hashes().await?; Ok(Some(CapsuleRepositoryRoots { root_digest, - shard_hashes: catalog.shards().keys().cloned().collect(), + shard_hashes, })) } @@ -3233,6 +3233,29 @@ mod tests { assert!(registry.complete_repos.contains("org/b")); } + #[tokio::test(flavor = "multi_thread")] + #[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] + async fn live_capsule_shard_inspection_reads_published_catalog_without_gc() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").unwrap(); + let storage = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + let store = Store::from_storage(storage); + let repo = format!("catalog-shard-inspection/{}", uuid::Uuid::now_v7()); + let (expected, _) = publish_capsule_pointer_catalog(&store, &repo).await; + let roots = capsule_repository_roots(&store, &repo) + .await + .unwrap() + .unwrap(); + assert_eq!(roots.shard_hashes, vec![expected]); + // Root capture only: this fixture must never invoke a bucket sweep. + let again = capsule_repository_roots(&store, &repo) + .await + .unwrap() + .unwrap(); + assert_eq!(again.root_digest, roots.root_digest); + } + #[tokio::test] async fn registry_repair_and_bucket_marks_preserve_capsule_catalog_shards() { use crab_metadata::capsule_protocol::{Capsule, CapsuleRefEdit, CapsuleTransaction}; diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs index 07ed6fef6..99663da35 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs @@ -50,7 +50,7 @@ impl CheckpointCatalog { } #[cfg(feature = "storage")] - fn reader( + pub(in crate::capsule_protocol) fn reader( &self, layout: &crab_storage::StoreLayout, ) -> Result { diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 9a34135c9..d8d35eee3 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -67,12 +67,12 @@ pub use plan::{ publish_capsule_plan_receipt, publish_capsule_plan_repair_receipt, read_capsule_plan_intent, resolve_capsule_plan_receipt, }; -#[cfg(feature = "storage")] -pub use pointer::PointerCatalogLookup; pub use pointer::{ FileCatalogEntry, MAX_CATALOG_BYTES, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, }; +#[cfg(feature = "storage")] +pub use pointer::{PointerCatalogLookup, inspect_pointer_catalog_shard_hashes}; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, MAX_CAPSULE_REF_FRONTIER, MAX_CAPSULE_REF_HEADS, capsule_ref_name_from_key, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index d8f1f31b0..723ce0a01 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -9,8 +9,16 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; +#[cfg(feature = "storage")] +pub(super) mod inspection; pub(in crate::capsule_protocol) mod object; pub(super) mod paged; +#[cfg(feature = "storage")] +mod relations; +#[cfg(feature = "storage")] +pub(super) mod spill; +#[cfg(feature = "storage")] +pub use inspection::inspect_pointer_catalog_shard_hashes; pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] pub use paged::PointerCatalogLookup; @@ -304,16 +312,10 @@ impl PointerCatalog { self.insert_shard(hash.clone(), entry.clone())?; } for (hash, entry) in &delta.files { - match self.files.get(hash) { - Some(existing) if existing.size != entry.size => { - return Err(contract_error(format!( - "file {hash} has conflicting declared sizes" - ))); - } - _ => { - self.files.insert(hash.clone(), entry.clone()); - } + if let Some(existing) = self.files.get(hash) { + validate_binding_size(hash, existing.size, entry.size)?; } + self.files.insert(hash.clone(), entry.clone()); } Ok(()) } @@ -434,12 +436,31 @@ fn insert_consistent( entry: T, kind: &str, ) -> Result<()> { - if map.get(&hash).is_some_and(|existing| existing != &entry) { + validate_consistent(map.get(&hash), &entry, &hash, kind)?; + map.insert(hash, entry); + Ok(()) +} + +fn validate_binding_size(hash: &str, previous: u64, incoming: u64) -> Result<()> { + if previous != incoming { + return Err(contract_error(format!( + "file {hash} has conflicting declared sizes" + ))); + } + Ok(()) +} + +fn validate_consistent( + previous: Option<&T>, + incoming: &T, + hash: &str, + kind: &str, +) -> Result<()> { + if previous.is_some_and(|previous| previous != incoming) { return Err(contract_error(format!( "{kind} {hash} has conflicting descriptors" ))); } - map.insert(hash, entry); Ok(()) } diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs new file mode 100644 index 000000000..e4e782f3c --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs @@ -0,0 +1,335 @@ +//! Captured logical catalog reduction with file precedence and deferred closure checks. +use super::relations::Record as Relation; +use super::spill::{Cursor, Reducer, Row}; +use super::*; +use crate::capsule_protocol::{Capsule, CapsuleControl, CheckpointCatalog}; +use crab_storage::{Store, StoreLayout, scratch::ScratchSpace}; + +const MAGIC: &[u8; 8] = b"CRBCLG01"; +const HEADER: usize = 45; + +pub(super) struct Entry { + key: [u8; 33], + rank: u64, + value: Bytes, +} + +impl Entry { + fn new(kind: u8, identity: &str, rank: u64, value: Vec) -> Result { + let mut key = [0; 33]; + key[0] = kind; + key[1..].copy_from_slice(&hash(identity)?); + let row = Self { + key, + rank, + value: value.into(), + }; + row.validate()?; + Ok(row) + } + + fn validate(&self) -> Result<()> { + let value = &self.value; + let valid = match self.key[0] { + 0 => value.len() == 40, + 1 => { + value.len() >= 8 + && (value.len() - 8).is_multiple_of(32) + && value[..8] != [0; 8] + && value[8..] + .as_chunks::<32>() + .0 + .windows(2) + .all(|pair| pair[0] < pair[1]) + } + 2 => { + value.len() >= 76 + && (value.len() - 40).is_multiple_of(36) + && value[..8] != [0; 8] + && value[40..] + .as_chunks::<36>() + .0 + .iter() + .all(|chunk| chunk[32..] != [0; 4]) + } + _ => false, + }; + if !valid || self.bytes() as u64 > Self::MAX_BYTES - 12 { + return Err(corrupt("logical catalog spill value has an invalid shape")); + } + Ok(()) + } +} + +impl Row for Entry { + const TARGET_BYTES: usize = 512 * 1024; + const MAX_BYTES: u64 = MAX_CATALOG_BYTES + 128; + fn key(&self) -> &[u8; 33] { + &self.key + } + fn bytes(&self) -> usize { + HEADER.saturating_add(self.value.len()) + } + fn merge(&mut self, other: Self) -> Result<()> { + let identity = hex(&self.key[1..])?; + if self.key[0] == 0 { + let size = |bytes: &[u8]| -> Result { + Ok(u64::from_be_bytes( + bytes[..8] + .try_into() + .map_err(|_| corrupt("file size is truncated"))?, + )) + }; + validate_binding_size(&identity, size(&self.value)?, size(&other.value)?)?; + if other.rank > self.rank { + *self = other; + } + } else { + let kind = if self.key[0] == 1 { "shard" } else { "xorb" }; + validate_consistent(Some(&self.value), &other.value, &identity, kind)?; + } + Ok(()) + } + fn encode(rows: &[Self]) -> Result { + let mut bytes = MAGIC.to_vec(); + bytes.extend_from_slice(&(rows.len() as u32).to_be_bytes()); + for row in rows { + row.validate()?; + bytes.extend_from_slice(&row.key); + bytes.extend_from_slice(&row.rank.to_be_bytes()); + let size = u32::try_from(row.value.len()) + .map_err(|_| corrupt("logical spill value exceeds its length field"))?; + bytes.extend_from_slice(&size.to_be_bytes()); + bytes.extend_from_slice(&row.value); + } + if rows.is_empty() || rows.len() > 4096 || bytes.len() as u64 > Self::MAX_BYTES { + return Err(corrupt("logical catalog spill block exceeds its bound")); + } + Ok(bytes.into()) + } + fn decode(bytes: Bytes) -> Result> { + if bytes.len() < 12 || bytes.get(..8) != Some(MAGIC.as_slice()) { + return Err(corrupt("logical catalog spill framing differs")); + } + let count = u32::from_be_bytes( + bytes[8..12] + .try_into() + .map_err(|_| corrupt("logical spill count is truncated"))?, + ) as usize; + if count == 0 || count > 4096 { + return Err(corrupt("logical spill count exceeds its bound")); + } + let mut rows = Vec::with_capacity(count); + let mut offset = 12; + for _ in 0..count { + let header = bytes + .get(offset..offset + HEADER) + .ok_or_else(|| corrupt("logical spill header is truncated"))?; + let key = header[..33] + .try_into() + .map_err(|_| corrupt("logical spill key is truncated"))?; + let rank = u64::from_be_bytes( + header[33..41] + .try_into() + .map_err(|_| corrupt("logical spill rank is truncated"))?, + ); + let length = u32::from_be_bytes( + header[41..] + .try_into() + .map_err(|_| corrupt("logical spill size is truncated"))?, + ) as usize; + offset += HEADER; + let end = offset + .checked_add(length) + .filter(|end| *end <= bytes.len()) + .ok_or_else(|| corrupt("logical spill value escapes its block"))?; + let row = Self { + key, + rank, + value: bytes.slice(offset..end), + }; + row.validate()?; + rows.push(row); + offset = end; + } + if offset != bytes.len() { + return Err(corrupt("logical spill block has trailing bytes")); + } + Ok(rows) + } +} + +pub(in crate::capsule_protocol) struct Inspector { + space: Option, + rows: Reducer, + maximum: u64, +} + +impl Inspector { + pub(super) fn new(maximum: u64) -> Self { + Self { + space: None, + rows: Reducer::new(), + maximum, + } + } + async fn add(&mut self, row: Entry) -> Result<()> { + if self.space.is_none() { + self.space = Some(ScratchSpace::temporary(self.maximum.saturating_mul(16)).await?); + } + let space = self + .space + .as_ref() + .ok_or_else(|| corrupt("logical spill has no workspace"))?; + self.rows.add(space, row).await + } + pub(super) async fn file( + &mut self, + identity: &str, + entry: FileCatalogEntry, + rank: u64, + ) -> Result<()> { + let mut value = entry.size.to_be_bytes().to_vec(); + value.extend_from_slice(&hash(&entry.shard_hash)?); + self.add(Entry::new(0, identity, rank, value)?).await + } + pub(super) async fn shard( + &mut self, + identity: &str, + entry: ShardCatalogEntry, + rank: u64, + ) -> Result<()> { + let mut value = entry.encoded_size.to_be_bytes().to_vec(); + for xorb in entry.xorb_hashes { + value.extend_from_slice(&hash(&xorb)?); + } + self.add(Entry::new(1, identity, rank, value)?).await + } + pub(super) async fn xorb( + &mut self, + identity: &str, + entry: XorbCatalogEntry, + rank: u64, + ) -> Result<()> { + let mut value = entry.encoded_size.to_be_bytes().to_vec(); + value.extend_from_slice(&hash(&entry.body_digest)?); + for chunk in entry.chunks { + value.extend_from_slice(&hash(&chunk.hash)?); + value.extend_from_slice(&chunk.uncompressed_size.to_be_bytes()); + } + self.add(Entry::new(2, identity, rank, value)?).await + } + async fn delta(&mut self, catalog: PointerCatalog, rank: u64) -> Result<()> { + catalog.validate(false, false)?; + for (hash, entry) in catalog.files { + self.file(&hash, entry, rank).await?; + } + for (hash, entry) in catalog.shards { + self.shard(&hash, entry, rank).await?; + } + for (hash, entry) in catalog.xorbs { + self.xorb(&hash, entry, rank).await?; + } + Ok(()) + } + pub(super) async fn finish(self, prior: Result<()>) -> Result> { + let Some(space) = self.space else { + prior?; + return Ok(Vec::new()); + }; + let result: Result> = async { + prior?; + let mut cursor = Cursor::::new(self.rows.finish(&space).await?); + let mut dependencies = Reducer::::new(); + let mut shards = Vec::new(); + while let Some(row) = cursor.next(&space).await? { + let identity = &row.key[1..]; + match row.key[0] { + 0 => { + dependencies + .add(&space, Relation::reference(0, &row.value[8..], false)?) + .await? + } + 1 => { + shards.push(hex(identity)?); + dependencies + .add(&space, Relation::reference(0, identity, true)?) + .await?; + for xorb in row.value[8..].as_chunks::<32>().0 { + dependencies + .add(&space, Relation::reference(1, xorb, false)?) + .await?; + } + } + 2 => { + dependencies + .add(&space, Relation::reference(1, identity, true)?) + .await? + } + _ => return Err(corrupt("logical catalog row kind differs")), + } + } + let mut cursor = Cursor::::new(dependencies.finish(&space).await?); + while let Some(row) = cursor.next(&space).await? { + row.validate()?; + } + Ok(shards) + } + .await; + let cleanup = space.close().await.map_err(MetadataError::from); + if let (Err(_), Err(source)) = (&result, &cleanup) { + tracing::warn!(%source, "logical catalog spill cleanup failed after validation error"); + } + let result = result?; + cleanup?; + Ok(result) + } +} + +/// Inspect one ordered captured catalog and return its complete shard identities. +/// +/// The caller retains these sources against GC. Checkpoint rows stream through +/// bounded spill; each packed delta retains its existing format-bounded decoder. +/// Immutable conflicts, file-size conflicts and incomplete final dependencies fail +/// before any result is returned. No origin object is written. +pub async fn inspect_pointer_catalog_shard_hashes( + layout: &StoreLayout, + checkpoint: Option<&CheckpointCatalog>, + capsules: &[Capsule], + controls: &[CapsuleControl], + maximum: u64, +) -> Result> { + let maximum = maximum.min(MAX_CATALOG_BYTES); + let mut inspector = Inspector::new(maximum); + let checked = async { + if let Some(checkpoint) = checkpoint { + checkpoint + .reader(layout)? + .inspect_into(layout.store(), maximum, &mut inspector) + .await?; + } + let mut rank = 1u64; + for capsule in capsules { + if let Some(delta) = capsule.pointer_catalog_delta()? { + inspector.delta(delta, rank).await?; + } + rank = rank + .checked_add(1) + .ok_or_else(|| corrupt("catalog capture rank overflows"))?; + } + for control in controls { + if let Some(delta) = control.materialize_catalog(layout.store()).await? { + inspector.delta(delta, rank).await?; + } + rank = rank + .checked_add(1) + .ok_or_else(|| corrupt("catalog capture rank overflows"))?; + } + Ok(()) + } + .await; + inspector.finish(checked).await +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs new file mode 100644 index 000000000..beed0c139 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -0,0 +1,257 @@ +use super::*; +use crate::capsule_protocol::pointer::object::{CatalogPages, Descriptor}; +use crab_storage::{StorageObservation, StorageObserver}; +use std::sync::{Arc, Mutex}; + +fn binding(file: u32, shard: u32, size: u64) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + catalog + .insert_file( + format!("{file:064x}"), + FileCatalogEntry::new(size, format!("{shard:064x}")), + ) + .unwrap(); + catalog +} + +fn shard(identity: u32, xorbs: Vec) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + catalog + .insert_shard(format!("{identity:064x}"), ShardCatalogEntry::new(7, xorbs)) + .unwrap(); + catalog +} + +async fn inspect(deltas: Vec) -> Result> { + let mut inspector = Inspector::new(16 * 1024 * 1024); + let checked = async { + for (rank, delta) in deltas.into_iter().enumerate() { + inspector.delta(delta, rank as u64).await?; + } + Ok(()) + } + .await; + inspector.finish(checked).await +} + +#[tokio::test] +async fn spilled_capture_matches_assembly_with_late_and_shadowed_dependencies() { + let mut initial = PointerCatalog::new(); + // Keep the missing old binding in the first run; a later rank must shadow it. + initial.merge_delta(&binding(0, 99, 3)).unwrap(); + for index in 1..13_000 { + initial.merge_delta(&binding(index, 1, 3)).unwrap(); + } + let mut xorb = PointerCatalog::new(); + xorb.insert_xorb( + "a".repeat(64), + XorbCatalogEntry::new( + 8, + "b".repeat(64), + vec![XorbChunkEntry::new("c".repeat(64), 3)], + ), + ) + .unwrap(); + let deltas = vec![ + initial, + binding(0, 2, 3), + shard(1, vec!["a".repeat(64)]), + shard(2, Vec::new()), + xorb, + ]; + let mut expected = PointerCatalogAssembly::new(PointerCatalog::new()); + for delta in &deltas { + expected.apply(delta).unwrap(); + } + let expected = expected + .finish() + .unwrap() + .shards + .into_keys() + .collect::>(); + assert_eq!(inspect(deltas).await.unwrap(), expected); +} + +#[tokio::test] +async fn captured_merge_rejects_conflicts_and_incomplete_final_closures() { + for scenario in [ + "file-size", + "shard-descriptor", + "xorb-descriptor", + "missing-shard", + "missing-xorb", + ] { + let mut deltas = vec![binding(0, 1, 3), shard(1, Vec::new())]; + match scenario { + "file-size" => deltas.push(binding(0, 99, 4)), + "shard-descriptor" => { + let mut conflicting = shard(1, Vec::new()); + conflicting + .shards + .get_mut(&format!("{:064x}", 1)) + .unwrap() + .encoded_size = 9; + deltas.push(conflicting); + } + "xorb-descriptor" => { + for size in [8, 9] { + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + "a".repeat(64), + XorbCatalogEntry::new( + size, + "b".repeat(64), + vec![XorbChunkEntry::new("c".repeat(64), 3)], + ), + ) + .unwrap(); + deltas.push(catalog); + } + } + "missing-shard" => deltas.push(binding(0, 99, 3)), + "missing-xorb" => deltas.push(shard(2, vec!["a".repeat(64)])), + _ => unreachable!(), + } + let mut oracle = PointerCatalogAssembly::new(PointerCatalog::new()); + let expected = deltas + .iter() + .try_for_each(|delta| oracle.apply(delta)) + .and_then(|()| oracle.finish()); + assert!(expected.is_err(), "oracle: {scenario}"); + assert!(inspect(deltas).await.is_err(), "spill: {scenario}"); + } +} + +#[tokio::test] +async fn failed_or_abandoned_capture_retires_private_spill() { + for abandon in [false, true] { + let parent = tempfile::tempdir().unwrap(); + let space = ScratchSpace::create(parent.path(), 8 * 1024 * 1024) + .await + .unwrap(); + let mut inspector = Inspector { + space: Some(space), + rows: Reducer::new(), + maximum: 1024 * 1024, + }; + for index in 0..5000 { + inspector.delta(binding(index, 1, 3), 0).await.unwrap(); + } + assert_ne!(std::fs::read_dir(parent.path()).unwrap().count(), 0); + if abandon { + drop(inspector); + } else { + assert!( + inspector + .finish(Err(corrupt("source disappeared"))) + .await + .is_err() + ); + } + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while std::fs::read_dir(parent.path()).unwrap().count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } +} + +#[test] +fn logical_spill_rejects_malformed_framing_values_and_lengths() { + let mut value = 3u64.to_be_bytes().to_vec(); + value.extend_from_slice(&[1; 32]); + let bytes = Entry::encode(&[Entry::new(0, &"a".repeat(64), 7, value).unwrap()]).unwrap(); + for scenario in ["count", "truncated", "trailing", "kind", "length"] { + let mut invalid = bytes.to_vec(); + match scenario { + "count" => invalid[8..12].copy_from_slice(&u32::MAX.to_be_bytes()), + "truncated" => { + invalid.pop(); + } + "trailing" => invalid.push(0), + "kind" => invalid[12] = 9, + "length" => invalid[53..57].copy_from_slice(&u32::MAX.to_be_bytes()), + _ => unreachable!(), + } + assert!(Entry::decode(invalid.into()).is_err(), "{scenario}"); + } +} + +#[derive(Default)] +struct Observations(Mutex>); +impl StorageObserver for Observations { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, operation: StorageObservation) { + self.0.lock().unwrap().push(operation); + } +} + +#[tokio::test] +async fn checkpoint_rows_stream_without_per_file_dependency_joins() { + let mut catalog = shard(1, vec!["a".repeat(64)]); + catalog + .insert_xorb( + "a".repeat(64), + XorbCatalogEntry::new( + 8, + "b".repeat(64), + vec![XorbChunkEntry::new("c".repeat(64), 3)], + ), + ) + .unwrap(); + for index in 0..100_000 { + catalog.merge_delta(&binding(index, 1, 3)).unwrap(); + } + let observations = Arc::new(Observations::default()); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(observations.clone()); + let prefix = object_store::path::Path::from("catalog"); + let (descriptor, objects) = Descriptor::build(&catalog).unwrap(); + let body = descriptor.encode().unwrap(); + let identity = *blake3::hash(&body).as_bytes(); + let pages = CatalogPages::new(prefix, descriptor, identity, body.len() as u64).unwrap(); + for (hash, body) in objects { + store + .put_if_absent_verified(&pages.pages.path(&hash), body) + .await + .unwrap(); + } + observations.0.lock().unwrap().clear(); + let maximum = 64 * 1024 * 1024; + assert_eq!( + pages + .full(&store, &mut crate::pages::PageWork::new(maximum, 0)) + .await + .unwrap(), + catalog + ); + let full = observations + .0 + .lock() + .unwrap() + .iter() + .map(|op| op.bytes_read) + .sum::(); + observations.0.lock().unwrap().clear(); + let cold = CatalogPages::new( + object_store::path::Path::from("catalog"), + pages.descriptor.clone(), + identity, + pages.descriptor.encode().unwrap().len() as u64, + ) + .unwrap(); + let mut inspector = Inspector::new(maximum); + let checked = cold.inspect_into(&store, maximum, &mut inspector).await; + assert_eq!( + inspector.finish(checked).await.unwrap(), + catalog.shards.into_keys().collect::>() + ); + let io = observations.0.lock().unwrap(); + let streamed = io.iter().map(|op| op.bytes_read).sum::(); + eprintln!("logical maintenance: {streamed} origin bytes; full materializer: {full}"); + assert!(streamed <= 4 * full, "{streamed} versus {full}"); + assert!(io.iter().all(|op| op.bytes_written == 0)); +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs index 8c46edfd8..aba3747e2 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs @@ -3,8 +3,7 @@ use super::*; use crate::pages::object::Node; use std::sync::Arc; -mod join; -use join::{Join, Record}; +use crate::capsule_protocol::pointer::relations::{Join, Record}; type Seen = BTreeMap<[u8; 32], (Kind, Reference)>; @@ -27,6 +26,65 @@ impl CatalogPages { .collect()) } + pub(in crate::capsule_protocol) async fn inspect_into( + &self, + store: &Store, + maximum: u64, + output: &mut crate::capsule_protocol::pointer::inspection::Inspector, + ) -> Result<()> { + // The complete physical proof includes reverse placement and alias checks. + // Streaming logical rows must not hide corruption in an unselected column. + self.inspected_objects(store, maximum).await?; + let mut work = PageWork::new(maximum, 0); + for kind in [Kind::CatalogFiles, Kind::CatalogShards, Kind::CatalogXorbs] { + let mut pending = self.root(kind)?.cloned().into_iter().collect(); + while let Some((_, node)) = self + .pages + .next_page(store, &mut pending, kind, &mut work) + .await? + { + if let Node::Leaf(rows) = node.as_ref() { + for (key, value) in rows { + let identity = hex(key)?; + // The physical pass admits the whole closure. Only this + // typed recipe survives until spill; cumulative row + // admission would reject copies already released. + let mut recipe_work = PageWork::new(maximum, 0); + match kind { + Kind::CatalogFiles => { + output.file(&identity, file_from_bytes(value)?, 0).await? + } + Kind::CatalogShards => { + output + .shard( + &identity, + self.shard(store, value, &mut recipe_work).await?, + 0, + ) + .await? + } + Kind::CatalogXorbs => { + output + .xorb( + &identity, + self.xorb(store, value, &mut recipe_work).await?, + 0, + ) + .await? + } + _ => { + return Err(corrupt( + "logical inspection selected an invalid column", + )); + } + } + } + } + } + } + Ok(()) + } + async fn inspect(&self, store: &Store, work: &mut PageWork, join: &mut Join) -> Result<()> { let mut seen = Seen::new(); for kind in [ diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs index 9d472642c..a97f31b49 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -323,6 +323,16 @@ async fn maintenance_streams_shared_recipes_without_materializing_the_catalog() .map(|(hash, body)| (pages.pages.path(&hash), body.len() as u64)) .collect::>(); assert_eq!(actual.into_iter().collect::>(), expected); + let cold = CatalogPages::new( + object_store::path::Path::from("catalog"), + pages.descriptor.clone(), + *blake3::hash(&body).as_bytes(), + body.len() as u64, + ) + .unwrap(); + let mut logical = crate::capsule_protocol::pointer::inspection::Inspector::new(maximum); + let checked = cold.inspect_into(&store, maximum, &mut logical).await; + assert!(logical.finish(checked).await.unwrap().is_empty()); } #[tokio::test] @@ -507,6 +517,12 @@ async fn streamed_inspection_rejects_resigned_broken_dependency_edges() { forged.inspected_objects(&store, 1024 * 1024).await.is_err(), "stream: {scenario}" ); + let mut logical = crate::capsule_protocol::pointer::inspection::Inspector::new(1024 * 1024); + let checked = forged.inspect_into(&store, 1024 * 1024, &mut logical).await; + assert!( + logical.finish(checked).await.is_err(), + "logical: {scenario}" + ); } } @@ -573,4 +589,22 @@ async fn streamed_inspection_bounds_io_for_distinct_recipes_beyond_the_page_cach streamed <= 4 * full, "streaming must not trade catalog-sized memory for repeated cold joins: {streamed} versus {full}" ); + observations.0.lock().unwrap().clear(); + let cold = CatalogPages::new( + object_store::path::Path::from("catalog"), + pages.descriptor.clone(), + *blake3::hash(&body).as_bytes(), + body.len() as u64, + ) + .unwrap(); + let mut logical = crate::capsule_protocol::pointer::inspection::Inspector::new(maximum); + let checked = cold.inspect_into(&store, maximum, &mut logical).await; + assert!(logical.finish(checked).await.unwrap().is_empty()); + let io = observations.0.lock().unwrap(); + let logical = io.iter().map(|op| op.bytes_read).sum::(); + eprintln!( + "distinct recipe logical maintenance: {logical} origin bytes; full materializer: {full}" + ); + assert!(logical <= 4 * full, "{logical} versus {full}"); + assert!(io.iter().all(|op| op.bytes_written == 0)); } diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs b/crates/crab-metadata/src/capsule_protocol/pointer/relations.rs similarity index 62% rename from crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs rename to crates/crab-metadata/src/capsule_protocol/pointer/relations.rs index b574ff38b..7e6f418d7 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection/join.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/relations.rs @@ -1,6 +1,6 @@ use super::*; -use crab_storage::scratch::{ScratchFile, ScratchSpace}; -use std::collections::VecDeque; +use crate::capsule_protocol::pointer::spill::{Cursor, Reducer, Row}; +use crab_storage::scratch::ScratchSpace; const MAGIC: &[u8; 8] = b"CRBCSJ01"; const BLOCK_ROWS: usize = 4096; @@ -62,7 +62,7 @@ impl Record { Ok(()) } - fn validate(&self) -> Result<()> { + pub(super) fn validate(&self) -> Result<()> { match self.key[0] { 0 | 1 if !self.required || self.defined => Ok(()), 2 if self.expected.is_some() && self.actual == self.expected => Ok(()), @@ -85,43 +85,6 @@ impl Record { } } -struct Run { - files: Vec, -} - -struct Cursor { - files: VecDeque, - rows: std::vec::IntoIter, - previous: Option<[u8; 33]>, -} - -impl Cursor { - fn new(run: Run) -> Self { - Self { - files: run.files.into(), - rows: Vec::new().into_iter(), - previous: None, - } - } - - async fn next(&mut self, space: &ScratchSpace) -> Result> { - loop { - if let Some(row) = self.rows.next() { - if self.previous.is_some_and(|previous| previous >= row.key) { - return Err(corrupt("inspection spill rows are not strictly ordered")); - } - self.previous = Some(row.key); - return Ok(Some(row)); - } - let Some(file) = self.files.pop_front() else { - return Ok(None); - }; - let body = space.read(&file, MAX_BLOCK_BYTES).await?; - self.rows = decode(&body)?.into_iter(); - } - } -} - fn encode(rows: &[Record]) -> Result { if rows.is_empty() || rows.len() > BLOCK_ROWS { return Err(corrupt("inspection spill block count exceeds its bound")); @@ -193,131 +156,49 @@ fn decode(bytes: &[u8]) -> Result> { .collect() } -pub(super) struct Join { +impl Row for Record { + const TARGET_BYTES: usize = BLOCK_ROWS * ROW_BYTES; + const MAX_BYTES: u64 = MAX_BLOCK_BYTES; + fn key(&self) -> &[u8; 33] { + &self.key + } + fn bytes(&self) -> usize { + ROW_BYTES + } + fn merge(&mut self, other: Self) -> Result<()> { + Record::merge(self, other) + } + fn encode(rows: &[Self]) -> Result { + encode(rows) + } + fn decode(bytes: Bytes) -> Result> { + decode(&bytes) + } +} + +pub(in crate::capsule_protocol) struct Join { space: ScratchSpace, - buffer: Vec, - levels: Vec>, + rows: Reducer, } impl Join { pub(super) fn new(space: ScratchSpace) -> Self { Self { space, - buffer: Vec::with_capacity(BLOCK_ROWS), - levels: Vec::new(), + rows: Reducer::new(), } } pub(super) async fn add(&mut self, row: Record) -> Result<()> { - self.buffer.push(row); - if self.buffer.len() == BLOCK_ROWS { - self.flush().await?; - } - Ok(()) + self.rows.add(&self.space, row).await } - async fn flush(&mut self) -> Result<()> { - if self.buffer.is_empty() { - return Ok(()); - } - let mut rows = std::mem::replace(&mut self.buffer, Vec::with_capacity(BLOCK_ROWS)); - rows.sort_unstable_by_key(|row| row.key); - let mut grouped: Vec = Vec::with_capacity(rows.len()); - for row in rows { - match grouped.last_mut().filter(|prior| prior.key == row.key) { - Some(prior) => prior.merge(row)?, - None => grouped.push(row), - } - } - let mut run = Run { - files: vec![self.space.write(encode(&grouped)?).await?], - }; - let mut level = 0; - loop { - if level == self.levels.len() { - self.levels.push(None); - } - match self.levels[level].take() { - Some(prior) => run = self.merge(prior, run).await?, - None => { - self.levels[level] = Some(run); - return Ok(()); - } - } - level += 1; - } - } - - async fn merge(&self, left: Run, right: Run) -> Result { - let retired = left - .files - .iter() - .chain(&right.files) - .cloned() - .collect::>(); - let mut left = Cursor::new(left); - let mut right = Cursor::new(right); - let mut a = left.next(&self.space).await?; - let mut b = right.next(&self.space).await?; - let mut rows = Vec::with_capacity(BLOCK_ROWS); - let mut output = Run { files: Vec::new() }; - while a.is_some() || b.is_some() { - let take_left = match (&a, &b) { - (Some(a), Some(b)) => a.key <= b.key, - (Some(_), None) => true, - _ => false, - }; - let row = if take_left { - let mut row = a - .take() - .ok_or_else(|| corrupt("spill merge lost its left row"))?; - a = left.next(&self.space).await?; - if b.as_ref().is_some_and(|other| other.key == row.key) { - row.merge( - b.take() - .ok_or_else(|| corrupt("spill merge lost its right row"))?, - )?; - b = right.next(&self.space).await?; - } - row - } else { - let row = b - .take() - .ok_or_else(|| corrupt("spill merge lost its right row"))?; - b = right.next(&self.space).await?; - row - }; - rows.push(row); - if rows.len() == BLOCK_ROWS { - output.files.push(self.space.write(encode(&rows)?).await?); - rows.clear(); - } - } - if !rows.is_empty() { - output.files.push(self.space.write(encode(&rows)?).await?); - } - for file in retired { - self.space.remove(file).await?; - } - Ok(output) - } - - pub(super) async fn finish(mut self, prior: Result<()>) -> Result<()> { + pub(super) async fn finish(self, prior: Result<()>) -> Result<()> { let result: Result<()> = async { prior?; - self.flush().await?; - let mut combined = None; - for run in std::mem::take(&mut self.levels).into_iter().flatten() { - combined = Some(match combined.take() { - Some(prior) => self.merge(prior, run).await?, - None => run, - }); - } - if let Some(run) = combined { - let mut cursor = Cursor::new(run); - while let Some(row) = cursor.next(&self.space).await? { - row.validate()?; - } + let mut cursor = Cursor::::new(self.rows.finish(&self.space).await?); + while let Some(row) = cursor.next(&self.space).await? { + row.validate()?; } Ok(()) } diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs b/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs new file mode 100644 index 000000000..bc22edd05 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs @@ -0,0 +1,185 @@ +//! Bounded sorted reductions over authenticated private spill blocks. +use super::*; +use crab_storage::scratch::{ScratchFile, ScratchSpace}; +use std::collections::VecDeque; + +pub(super) trait Row: Sized { + const TARGET_BYTES: usize; + const MAX_BYTES: u64; + fn key(&self) -> &[u8; 33]; + fn bytes(&self) -> usize; + fn merge(&mut self, other: Self) -> Result<()>; + fn encode(rows: &[Self]) -> Result; + fn decode(bytes: Bytes) -> Result>; +} + +pub(super) struct Run { + files: Vec, +} + +pub(super) struct Cursor { + files: VecDeque, + rows: std::vec::IntoIter, + previous: Option<[u8; 33]>, +} + +impl Cursor { + pub(super) fn new(run: Run) -> Self { + Self { + files: run.files.into(), + rows: Vec::new().into_iter(), + previous: None, + } + } + + pub(super) async fn next(&mut self, space: &ScratchSpace) -> Result> { + loop { + if let Some(row) = self.rows.next() { + if self.previous.is_some_and(|previous| previous >= *row.key()) { + return Err(corrupt("inspection spill rows are not strictly ordered")); + } + self.previous = Some(*row.key()); + return Ok(Some(row)); + } + let Some(file) = self.files.pop_front() else { + return Ok(None); + }; + self.rows = R::decode(space.read(&file, R::MAX_BYTES).await?)?.into_iter(); + } + } +} + +pub(super) struct Reducer { + buffer: Vec, + bytes: usize, + levels: Vec>, +} + +impl Reducer { + pub(super) fn new() -> Self { + Self { + buffer: Vec::new(), + bytes: 0, + levels: Vec::new(), + } + } + + pub(super) async fn add(&mut self, space: &ScratchSpace, row: R) -> Result<()> { + if !self.buffer.is_empty() && self.bytes.saturating_add(row.bytes()) > R::TARGET_BYTES { + self.flush(space).await?; + } + self.bytes = self.bytes.saturating_add(row.bytes()); + self.buffer.push(row); + if self.buffer.len() == 4096 || self.bytes >= R::TARGET_BYTES { + self.flush(space).await?; + } + Ok(()) + } + + async fn flush(&mut self, space: &ScratchSpace) -> Result<()> { + if self.buffer.is_empty() { + return Ok(()); + } + let mut rows = std::mem::take(&mut self.buffer); + self.bytes = 0; + rows.sort_unstable_by(|a, b| a.key().cmp(b.key())); + let mut grouped: Vec = Vec::with_capacity(rows.len()); + for row in rows { + match grouped.last_mut().filter(|prior| prior.key() == row.key()) { + Some(prior) => prior.merge(row)?, + None => grouped.push(row), + } + } + let mut run = Run { + files: vec![space.write(R::encode(&grouped)?).await?], + }; + let mut level = 0; + loop { + if level == self.levels.len() { + self.levels.push(None); + } + match self.levels[level].take() { + Some(prior) => run = Self::merge(space, prior, run).await?, + None => { + self.levels[level] = Some(run); + return Ok(()); + } + } + level += 1; + } + } + + async fn merge(space: &ScratchSpace, left: Run, right: Run) -> Result { + let retired = left + .files + .iter() + .chain(&right.files) + .cloned() + .collect::>(); + let mut left = Cursor::::new(left); + let mut right = Cursor::::new(right); + let mut a = left.next(space).await?; + let mut b = right.next(space).await?; + let mut rows = Vec::new(); + let mut bytes = 0usize; + let mut output = Run { files: Vec::new() }; + while a.is_some() || b.is_some() { + let take_left = match (&a, &b) { + (Some(a), Some(b)) => a.key() <= b.key(), + (Some(_), None) => true, + _ => false, + }; + let row = if take_left { + let mut row = a + .take() + .ok_or_else(|| corrupt("spill merge lost its left row"))?; + a = left.next(space).await?; + if b.as_ref().is_some_and(|other| other.key() == row.key()) { + row.merge( + b.take() + .ok_or_else(|| corrupt("spill merge lost its right row"))?, + )?; + b = right.next(space).await?; + } + row + } else { + let row = b + .take() + .ok_or_else(|| corrupt("spill merge lost its right row"))?; + b = right.next(space).await?; + row + }; + if !rows.is_empty() && bytes.saturating_add(row.bytes()) > R::TARGET_BYTES { + output.files.push(space.write(R::encode(&rows)?).await?); + rows.clear(); + bytes = 0; + } + bytes = bytes.saturating_add(row.bytes()); + rows.push(row); + if rows.len() == 4096 || bytes >= R::TARGET_BYTES { + output.files.push(space.write(R::encode(&rows)?).await?); + rows.clear(); + bytes = 0; + } + } + if !rows.is_empty() { + output.files.push(space.write(R::encode(&rows)?).await?); + } + for file in retired { + space.remove(file).await?; + } + Ok(output) + } + + pub(super) async fn finish(mut self, space: &ScratchSpace) -> Result { + self.flush(space).await?; + let mut combined = None; + for run in self.levels.into_iter().flatten() { + combined = Some(match combined.take() { + Some(prior) => Self::merge(space, prior, run).await?, + None => run, + }); + } + Ok(combined.unwrap_or(Run { files: Vec::new() })) + } +} diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 556399644..330f45ab9 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -847,6 +847,33 @@ impl CapsuleRepositoryView { Ok(catalog.finish()?) } + /// Inspect complete captured catalog dependencies without retaining the merged catalog. + /// + /// File bindings and immutable conflicts retain capture order. Fetch-only + /// captures remain incomplete; origin state is never recaptured or written. + /// The caller retains these captured sources against GC through completion. + pub async fn pointer_catalog_shard_hashes(&self) -> Result> { + self.catalog_sources + .as_ref() + .ok_or(ReadError::IncompleteCatalogView)?; + let layout = self + .metadata_layout + .as_ref() + .ok_or_else(|| ReadError::internal("catalog inspection has no captured origin"))?; + Ok( + crab_metadata::capsule_protocol::inspect_pointer_catalog_shard_hashes( + layout, + self.checkpoint + .as_ref() + .and_then(LayeredCheckpoint::catalog), + &self.capsules, + &self.capsule_controls, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await?, + ) + } + /// Open the authenticated embedded Git packs as a filesystem-free repository. /// /// The returned handle is pinned to this exact root and uses a private @@ -6096,10 +6123,21 @@ mod tests { fetch.pointer_catalog().await, Err(ReadError::IncompleteCatalogView) )); + assert!(matches!( + fetch.pointer_catalog_shard_hashes().await, + Err(ReadError::IncompleteCatalogView) + )); assert!(observer.observations.lock().unwrap().is_empty()); let complete = open_git_view_from_root(&router, root, TEST_LIMITS) .await .unwrap(); + assert!( + complete + .pointer_catalog_shard_hashes() + .await + .unwrap() + .is_empty() + ); let lookup = complete .pointer_catalog_lookup(&router, 1024 * 1024) .unwrap(); @@ -6312,6 +6350,10 @@ mod tests { observer.observations.lock().unwrap().clear(); lookup.files(router.store(), &[file]).await.unwrap(); assert!(observer.observations.lock().unwrap().is_empty()); + assert_eq!( + view.pointer_catalog_shard_hashes().await.unwrap(), + catalog.shards().keys().cloned().collect::>() + ); assert_eq!(view.pointer_catalog().await.unwrap(), catalog); } } From 18e41437cb11427049898f3dcdd71fbd6b3b0236 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 02:59:25 -0700 Subject: [PATCH 121/127] test(metadata): keep metrics in tracing --- .../src/capsule_protocol/pointer/inspection/tests.rs | 6 +++++- .../src/capsule_protocol/pointer/object/tests.rs | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs index beed0c139..d6d446a7c 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -251,7 +251,11 @@ async fn checkpoint_rows_stream_without_per_file_dependency_joins() { ); let io = observations.0.lock().unwrap(); let streamed = io.iter().map(|op| op.bytes_read).sum::(); - eprintln!("logical maintenance: {streamed} origin bytes; full materializer: {full}"); + tracing::debug!( + origin_bytes = streamed, + full_materializer_bytes = full, + "logical catalog maintenance" + ); assert!(streamed <= 4 * full, "{streamed} versus {full}"); assert!(io.iter().all(|op| op.bytes_written == 0)); } diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs index a97f31b49..45d12bd98 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -602,8 +602,10 @@ async fn streamed_inspection_bounds_io_for_distinct_recipes_beyond_the_page_cach assert!(logical.finish(checked).await.unwrap().is_empty()); let io = observations.0.lock().unwrap(); let logical = io.iter().map(|op| op.bytes_read).sum::(); - eprintln!( - "distinct recipe logical maintenance: {logical} origin bytes; full materializer: {full}" + tracing::debug!( + origin_bytes = logical, + full_materializer_bytes = full, + "distinct recipe catalog maintenance" ); assert!(logical <= 4 * full, "{logical} versus {full}"); assert!(io.iter().all(|op| op.bytes_written == 0)); From b2c0f1ce1bd9963e5b6ebe493ee01341b83718c8 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 04:04:17 -0700 Subject: [PATCH 122/127] perf(read): stream verified replica catalogs Retain the normalized inspection run through replica body probes instead of rebuilding a joined catalog. GC shares the same explicit spill owner. Advance cursor blocks only after verified reads finish: cancelling and resuming a pending read previously lost the unread block. Preserve cleanup ownership and primary errors through every exit. --- .../docs/architecture/indexed-remote-reads.md | 9 +- crates/crab-metadata/AGENTS.md | 6 + crates/crab-metadata/README.md | 12 + .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/pointer.rs | 2 +- .../capsule_protocol/pointer/inspection.rs | 179 ++++++++++++-- .../pointer/inspection/tests.rs | 168 ++++++++++++- .../capsule_protocol/pointer/object/tests.rs | 22 +- .../src/capsule_protocol/pointer/spill.rs | 28 ++- crates/crab-read/AGENTS.md | 1 + crates/crab-read/README.md | 22 +- crates/crab-read/src/capsule_protocol.rs | 40 ++-- crates/crab-read/src/selection.rs | 222 ++++++++++++++---- 13 files changed, 610 insertions(+), 105 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 5b226b25c..80ec8ecd3 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2541,8 +2541,13 @@ list remains output-sized. This is bounded external reduction, not constant space: physical page identities, spill handles, the largest recipe and output roots still consume memory. Each selected packed delta still uses its existing format-bounded decoder individually; streaming those deltas remains follow-up -work. Current fsck payload verification, restore accounting and catalog-rewriting -tools also retain full logical catalogs. Materialization admission continues to +work. Replica readiness now retains that verified local run and streams full +shard/xorb descriptors through the shared physical body verifiers. No body probe +starts before the final metadata proof; all returned paths close the cursor, +including provider errors and sample limits. Pending reads advance only after +verified I/O and decoding, so cancellation followed by resume cannot skip a block. +Current fsck payload verification, restore accounting and catalog-rewriting tools +also retain full logical catalogs. Materialization admission continues to protect those callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before diff --git a/crates/crab-metadata/AGENTS.md b/crates/crab-metadata/AGENTS.md index 60a459b12..fd2844570 100644 --- a/crates/crab-metadata/AGENTS.md +++ b/crates/crab-metadata/AGENTS.md @@ -26,6 +26,7 @@ Trace one path: `crates/crab-write/src/journal.rs` → `commit_ref_transaction` | Manifest/journal change | `crates/crab-metadata/src/manifest_store.rs` | `crates/crab-write/src/journal.rs` | | Catalog coverage | `crates/crab-metadata/src/git_object_locator` | `crates/crab-remote-git/src/repository.rs` | | Local dedup index | `crates/crab-metadata/src/persistent_chunk_index.rs` | `crates/crab-staging/src/index.rs` | +| Captured catalog inspection | `crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs` | GC shard discovery and replica readiness | | Capsule chunk hints | `crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs` | CLI/shared/protected capsule publishers and bucket GC | ## Invariants @@ -51,6 +52,11 @@ Trace one path: `crates/crab-write/src/journal.rs` → `commit_ref_transaction` missing page; unchanged-root absence/corruption remains an error. Source: `crates/crab-metadata/src/capsule_protocol/chunk_candidates/store.rs`. +- `PointerCatalogInspection` exposes external descriptors only after complete + precedence/conflict/dependency proof. Await `finish` on all returned paths; + cancelled reads must resume without losing a block or object. + Source: `crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs`. + ## Features and platform Empty default. `storage` adds object-store helpers; `file-index-reader` and `remote-index` also enable storage and SlateDB paths; `local-index` adds SQLite. Inspect locked SlateDB session/writer close contracts before changing lifecycle. Payload-only checks must remain a separate slice. diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 736c39d59..70c95d713 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -93,6 +93,18 @@ retain file ownership through cancellation and are drained before normal/error cleanup. No authoritative or cache object is written by inspection. Explicit full-catalog callers still materialize it; converting remaining maintenance consumers is separate work. + +`inspect_pointer_catalog` reduces a complete captured checkpoint and selected +deltas through private spill, resolves file-binding precedence and validates the +final dependency closure before returning `PointerCatalogInspection`. GC collects +shard identities from this owner; replica readiness streams full shard/xorb +descriptors without a joined catalog in memory. The cursor retains the normalized +run until `finish` drains workers and removes it, preserving the operation's primary +error if cleanup also fails. A cancelled pending read can resume at the same +object. Physical page identities, spill handles and the largest recipe still +consume memory; each packed delta retains its format-bounded decoder. Logical +reduction has a live disk ceiling of sixteen times the admitted catalog limit; +normal lookup and materialization budgets remain unchanged. GC traverses catalog pages from current, historical and retained browse roots. Protected keys and authenticated closure traversal are tracked separately. Retired unshipped checkpoint formats through 8 are rejected without fallback. diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index d8d35eee3..23e972442 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -72,7 +72,9 @@ pub use pointer::{ XorbCatalogEntry, XorbChunkEntry, }; #[cfg(feature = "storage")] -pub use pointer::{PointerCatalogLookup, inspect_pointer_catalog_shard_hashes}; +pub use pointer::{ + PointerCatalogInspection, PointerCatalogLookup, PointerCatalogObject, inspect_pointer_catalog, +}; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, MAX_CAPSULE_REF_FRONTIER, MAX_CAPSULE_REF_HEADS, capsule_ref_name_from_key, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 723ce0a01..d97c88495 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -18,7 +18,7 @@ mod relations; #[cfg(feature = "storage")] pub(super) mod spill; #[cfg(feature = "storage")] -pub use inspection::inspect_pointer_catalog_shard_hashes; +pub use inspection::{PointerCatalogInspection, PointerCatalogObject, inspect_pointer_catalog}; pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] pub use paged::PointerCatalogLookup; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs index e4e782f3c..8d53a6f43 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs @@ -1,6 +1,6 @@ //! Captured logical catalog reduction with file precedence and deferred closure checks. use super::relations::Record as Relation; -use super::spill::{Cursor, Reducer, Row}; +use super::spill::{Cursor, Reducer, Row, Run}; use super::*; use crate::capsule_protocol::{Capsule, CapsuleControl, CheckpointCatalog}; use crab_storage::{Store, StoreLayout, scratch::ScratchSpace}; @@ -232,16 +232,19 @@ impl Inspector { } Ok(()) } - pub(super) async fn finish(self, prior: Result<()>) -> Result> { + pub(super) async fn finish(self, prior: Result<()>) -> Result { let Some(space) = self.space else { prior?; - return Ok(Vec::new()); + return Ok(PointerCatalogInspection { + space: None, + rows: Cursor::new(Run::default()), + }); }; - let result: Result> = async { + let result: Result = async { prior?; - let mut cursor = Cursor::::new(self.rows.finish(&space).await?); + let run = self.rows.finish(&space).await?; + let mut cursor = Cursor::::new(run.clone()); let mut dependencies = Reducer::::new(); - let mut shards = Vec::new(); while let Some(row) = cursor.next(&space).await? { let identity = &row.key[1..]; match row.key[0] { @@ -251,7 +254,6 @@ impl Inspector { .await? } 1 => { - shards.push(hex(identity)?); dependencies .add(&space, Relation::reference(0, identity, true)?) .await?; @@ -269,36 +271,169 @@ impl Inspector { _ => return Err(corrupt("logical catalog row kind differs")), } } - let mut cursor = Cursor::::new(dependencies.finish(&space).await?); + let dependencies = dependencies.finish(&space).await?; + let mut cursor = Cursor::::new(dependencies.clone()); while let Some(row) = cursor.next(&space).await? { row.validate()?; } - Ok(shards) + dependencies.remove(&space).await?; + Ok(run) } .await; - let cleanup = space.close().await.map_err(MetadataError::from); - if let (Err(_), Err(source)) = (&result, &cleanup) { - tracing::warn!(%source, "logical catalog spill cleanup failed after validation error"); + match result { + Ok(run) => Ok(PointerCatalogInspection { + space: Some(space), + rows: Cursor::new(run), + }), + Err(error) => finish(Some(space), Err(error)).await, } - let result = result?; - cleanup?; - Ok(result) } } -/// Inspect one ordered captured catalog and return its complete shard identities. +/// One immutable external object authenticated by a complete captured catalog. +#[derive(Debug, PartialEq, Eq)] +pub enum PointerCatalogObject { + Shard { + hash: String, + entry: ShardCatalogEntry, + }, + Xorb { + hash: String, + entry: XorbCatalogEntry, + }, +} + +/// A complete catalog proof with bounded iteration over its external object descriptors. +/// +/// The private spill is owned until `finish` drains and removes it. Dropping the +/// cursor retains cleanup behind blocking workers. No origin writes or recapture occur. +#[must_use] +pub struct PointerCatalogInspection { + space: Option, + rows: Cursor, +} + +impl PointerCatalogInspection { + async fn next_entry(&mut self) -> Result> { + match &self.space { + Some(space) => self.rows.next(space).await, + None => Ok(None), + } + } + + /// Return the next shard or xorb in canonical hash order within each kind. + /// + /// Complete metadata validation precedes construction. One typed descriptor + /// is returned at a time; its recipe may reach the admitted catalog bound. + /// Cancelling a pending call leaves the next object available on resume. + pub async fn next_object(&mut self) -> Result> { + while let Some(row) = self.next_entry().await? { + if row.key[0] == 0 { + continue; + } + let hash = hex(&row.key[1..])?; + let size = u64::from_be_bytes( + row.value[..8] + .try_into() + .map_err(|_| corrupt("object size is truncated"))?, + ); + let object = match row.key[0] { + 1 => PointerCatalogObject::Shard { + hash, + entry: ShardCatalogEntry::new( + size, + row.value[8..] + .as_chunks::<32>() + .0 + .iter() + .map(|value| hex(value)) + .collect::>()?, + ), + }, + 2 => PointerCatalogObject::Xorb { + hash, + entry: XorbCatalogEntry::new( + size, + hex(&row.value[8..40])?, + row.value[40..] + .as_chunks::<36>() + .0 + .iter() + .map(|chunk| { + Ok(XorbChunkEntry::new( + hex(&chunk[..32])?, + u32::from_be_bytes( + chunk[32..] + .try_into() + .map_err(|_| corrupt("chunk size is truncated"))?, + ), + )) + }) + .collect::>()?, + ), + }, + _ => return Err(corrupt("logical catalog object kind differs")), + }; + return Ok(Some(object)); + } + Ok(None) + } + + /// Collect the remaining shard identities without decoding object recipes, then close. + pub async fn into_shard_hashes(mut self) -> Result> { + let result = async { + let mut hashes = Vec::new(); + while let Some(row) = self.next_entry().await? { + if row.key[0] == 1 { + hashes.push(hex(&row.key[1..])?); + } + } + Ok(hashes) + } + .await; + self.finish(result).await + } + + /// Close the private spill after an operation, preserving its primary error. + /// + /// Await completion on success and returned-error paths. A cleanup failure + /// becomes the result only when the operation itself succeeded. + pub async fn finish>( + self, + result: std::result::Result, + ) -> std::result::Result { + finish(self.space, result).await + } +} + +async fn finish>( + space: Option, + result: std::result::Result, +) -> std::result::Result { + let cleanup = match space { + Some(space) => space.close().await.map_err(MetadataError::from), + None => Ok(()), + }; + if let (Err(_), Err(source)) = (&result, &cleanup) { + tracing::warn!(%source, "logical catalog spill cleanup failed after operation error"); + } + let result = result?; + cleanup.map_err(E::from)?; + Ok(result) +} + +/// Validate one ordered captured catalog before opening its external object cursor. /// -/// The caller retains these sources against GC. Checkpoint rows stream through -/// bounded spill; each packed delta retains its existing format-bounded decoder. -/// Immutable conflicts, file-size conflicts and incomplete final dependencies fail -/// before any result is returned. No origin object is written. -pub async fn inspect_pointer_catalog_shard_hashes( +/// Checkpoint rows stream through bounded spill; each packed delta retains its +/// existing format-bounded decoder. Conflicts and incomplete final dependencies +/// fail before a cursor is returned. Await cursor `finish` on every returned path. +pub async fn inspect_pointer_catalog( layout: &StoreLayout, checkpoint: Option<&CheckpointCatalog>, capsules: &[Capsule], controls: &[CapsuleControl], maximum: u64, -) -> Result> { +) -> Result { let maximum = maximum.min(MAX_CATALOG_BYTES); let mut inspector = Inspector::new(maximum); let checked = async { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs index d6d446a7c..973457d72 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -31,7 +31,7 @@ async fn inspect(deltas: Vec) -> Result> { Ok(()) } .await; - inspector.finish(checked).await + inspector.finish(checked).await?.into_shard_hashes().await } #[tokio::test] @@ -246,7 +246,13 @@ async fn checkpoint_rows_stream_without_per_file_dependency_joins() { let mut inspector = Inspector::new(maximum); let checked = cold.inspect_into(&store, maximum, &mut inspector).await; assert_eq!( - inspector.finish(checked).await.unwrap(), + inspector + .finish(checked) + .await + .unwrap() + .into_shard_hashes() + .await + .unwrap(), catalog.shards.into_keys().collect::>() ); let io = observations.0.lock().unwrap(); @@ -259,3 +265,161 @@ async fn checkpoint_rows_stream_without_per_file_dependency_joins() { assert!(streamed <= 4 * full, "{streamed} versus {full}"); assert!(io.iter().all(|op| op.bytes_written == 0)); } + +async fn cursor_fixture( + files: u32, +) -> ( + tempfile::TempDir, + PointerCatalogInspection, + Vec, +) { + let parent = tempfile::tempdir().unwrap(); + let space = ScratchSpace::create(parent.path(), 16 * 1024 * 1024) + .await + .unwrap(); + let mut inspector = Inspector { + space: Some(space), + rows: Reducer::new(), + maximum: 1024 * 1024, + }; + let xorb = XorbCatalogEntry::new( + 80, + "d".repeat(64), + vec![ + XorbChunkEntry::new("c".repeat(64), 3), + XorbChunkEntry::new("b".repeat(64), 9), + XorbChunkEntry::new("c".repeat(64), 7), + ], + ); + let shard = ShardCatalogEntry::new(7, vec!["a".repeat(64)]); + let expected = vec![ + PointerCatalogObject::Shard { + hash: format!("{:064x}", 1), + entry: shard.clone(), + }, + PointerCatalogObject::Xorb { + hash: "a".repeat(64), + entry: xorb.clone(), + }, + ]; + let mut catalog = PointerCatalog::new(); + for file in 0..files { + catalog.merge_delta(&binding(file, 1, 19)).unwrap(); + } + catalog.insert_shard(format!("{:064x}", 1), shard).unwrap(); + catalog.insert_xorb("a".repeat(64), xorb).unwrap(); + inspector.delta(catalog, 0).await.unwrap(); + let cursor = inspector.finish(Ok(())).await.unwrap(); + (parent, cursor, expected) +} + +#[tokio::test] +async fn verified_cursor_preserves_complete_descriptors_across_spilled_file_rows() { + let (_parent, mut cursor, expected) = cursor_fixture(5000).await; + let result: Result<_> = async { + let mut values = Vec::new(); + while let Some(object) = cursor.next_object().await? { + values.push(object); + } + Ok(values) + } + .await; + assert_eq!(cursor.finish(result).await.unwrap(), expected); +} + +#[tokio::test] +async fn verified_cursor_retires_spill_after_early_stop_error_abandonment_and_tampering() { + for outcome in ["early", "error", "drop", "tamper"] { + let (parent, mut cursor, _) = cursor_fixture(64).await; + match outcome { + "early" => { + cursor.next_object().await.unwrap(); + cursor.finish(Ok::<_, MetadataError>(())).await.unwrap(); + } + "error" => { + let result: Result<()> = Err(contract_error("body check failed")); + assert!(matches!( + cursor.finish(result).await, + Err(MetadataError::CapsuleContract { .. }) + )); + } + "drop" => drop(cursor), + "tamper" => { + let directory = std::fs::read_dir(parent.path()) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + for file in std::fs::read_dir(directory).unwrap() { + std::fs::write(file.unwrap().path(), b"changed").unwrap(); + } + let result = cursor.next_object().await; + assert!(matches!( + result, + Err(MetadataError::Storage { + source: crab_storage::StorageError::CorruptObject { .. } + }) + )); + assert!(cursor.next_object().await.is_err()); + assert!(cursor.finish(result).await.is_err()); + } + _ => unreachable!(), + } + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while std::fs::read_dir(parent.path()).unwrap().count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } +} + +#[tokio::test] +async fn verified_cursor_preserves_operation_errors_when_cleanup_also_fails() { + let (parent, cursor, _) = cursor_fixture(1).await; + let directory = std::fs::read_dir(parent.path()) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + std::fs::remove_dir_all(&directory).unwrap(); + std::fs::write(&directory, b"obstruct cleanup").unwrap(); + let result: Result<()> = Err(contract_error("body check failed")); + assert!( + matches!(cursor.finish(result).await, Err(MetadataError::CapsuleContract { reason, .. }) if reason == "body check failed") + ); +} + +#[test] +fn cancelled_pending_cursor_read_resumes_at_the_same_object() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + runtime.block_on(async { + let (_parent, mut cursor, mut expected) = cursor_fixture(64).await; + let (release, gate) = std::sync::mpsc::channel(); + let (started, entered) = tokio::sync::oneshot::channel(); + let blocker = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + gate.recv().unwrap(); + }); + entered.await.unwrap(); + let mut pending = Box::pin(cursor.next_object()); + std::future::poll_fn(|context| { + assert!(std::future::Future::poll(pending.as_mut(), context).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + drop(pending); + release.send(()).unwrap(); + blocker.await.unwrap(); + let actual = cursor.next_object().await.unwrap(); + cursor.finish(Ok::<_, MetadataError>(())).await.unwrap(); + assert_eq!(actual, Some(expected.remove(0))); + }); +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs index 45d12bd98..6500905dc 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -332,7 +332,16 @@ async fn maintenance_streams_shared_recipes_without_materializing_the_catalog() .unwrap(); let mut logical = crate::capsule_protocol::pointer::inspection::Inspector::new(maximum); let checked = cold.inspect_into(&store, maximum, &mut logical).await; - assert!(logical.finish(checked).await.unwrap().is_empty()); + assert!( + logical + .finish(checked) + .await + .unwrap() + .into_shard_hashes() + .await + .unwrap() + .is_empty() + ); } #[tokio::test] @@ -599,7 +608,16 @@ async fn streamed_inspection_bounds_io_for_distinct_recipes_beyond_the_page_cach .unwrap(); let mut logical = crate::capsule_protocol::pointer::inspection::Inspector::new(maximum); let checked = cold.inspect_into(&store, maximum, &mut logical).await; - assert!(logical.finish(checked).await.unwrap().is_empty()); + assert!( + logical + .finish(checked) + .await + .unwrap() + .into_shard_hashes() + .await + .unwrap() + .is_empty() + ); let io = observations.0.lock().unwrap(); let logical = io.iter().map(|op| op.bytes_read).sum::(); tracing::debug!( diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs b/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs index bc22edd05..88e4b57a7 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs @@ -13,10 +13,20 @@ pub(super) trait Row: Sized { fn decode(bytes: Bytes) -> Result>; } +#[derive(Clone, Default)] pub(super) struct Run { files: Vec, } +impl Run { + pub(super) async fn remove(self, space: &ScratchSpace) -> Result<()> { + for file in self.files { + space.remove(file).await?; + } + Ok(()) + } +} + pub(super) struct Cursor { files: VecDeque, rows: std::vec::IntoIter, @@ -34,17 +44,21 @@ impl Cursor { pub(super) async fn next(&mut self, space: &ScratchSpace) -> Result> { loop { - if let Some(row) = self.rows.next() { + if let Some(row) = self.rows.as_slice().first() { if self.previous.is_some_and(|previous| previous >= *row.key()) { return Err(corrupt("inspection spill rows are not strictly ordered")); } self.previous = Some(*row.key()); - return Ok(Some(row)); + return Ok(self.rows.next()); } - let Some(file) = self.files.pop_front() else { + let Some(file) = self.files.front() else { return Ok(None); }; - self.rows = R::decode(space.read(&file, R::MAX_BYTES).await?)?.into_iter(); + // A caller can cancel and resume this read. Commit progression only + // after verified I/O and decoding, or the unread block is lost. + let rows = R::decode(space.read(file, R::MAX_BYTES).await?)?; + let _ = self.files.pop_front(); + self.rows = rows.into_iter(); } } } @@ -165,9 +179,7 @@ impl Reducer { if !rows.is_empty() { output.files.push(space.write(R::encode(&rows)?).await?); } - for file in retired { - space.remove(file).await?; - } + Run { files: retired }.remove(space).await?; Ok(output) } @@ -180,6 +192,6 @@ impl Reducer { None => run, }); } - Ok(combined.unwrap_or(Run { files: Vec::new() })) + Ok(combined.unwrap_or_default()) } } diff --git a/crates/crab-read/AGENTS.md b/crates/crab-read/AGENTS.md index a597f1648..ab35a3713 100644 --- a/crates/crab-read/AGENTS.md +++ b/crates/crab-read/AGENTS.md @@ -27,6 +27,7 @@ For object download changes continue into `crates/crab-read/src/store_client.rs` | --- | --- | --- | | Hydration or range integrity | `crates/crab-read/src/hydrator.rs` | `crates/crab-vfs/src/hydration.rs` | | Fetch authorization | `crates/crab-read/src/fetch_admission.rs` | `crab/src/git/remote_helper.rs` | +| Replica readiness | `crates/crab-read/src/selection.rs` | `crab/src/replication/mod.rs` | | Term resolution | `crates/crab-read/src/term_resolver.rs` | `crates/crab-metadata/src/file_index_lookup.rs` | ## Invariants diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index b7e996abd..294c2ebbc 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -223,12 +223,22 @@ Its error source is Tokio's `JoinError`, so diagnostic consumers can distinguish worker panic from task cancellation without parsing log text. The CLI preserves that source while retaining its internal-error diagnostic classification. -Replica readiness compares the exact authenticated capsule view before reading -and validating every cataloged shard and xorb body. Large-body hashing and -parsing run outside the async executor; worker failures remain typed as -`ReadError::ReadinessTask`. Product caches may skip repeated immutable-body -validation, but must recheck the replica's authenticated view digest before -selection. +Replica readiness compares the exact authenticated capsule view and completes +catalog conflict/dependency validation before checking any payload. It owns a +`PointerCatalogInspection` cursor and verifies one shard/xorb descriptor at a time. +Full probes validate every cataloged body; sampled probes preserve canonical +shard-before-xorb order and stop at their existing limit. The complete metadata +proof is required even for a zero-object sample. Every returned path finishes the +cursor, including missing bodies, errors and early sample limits; abandonment +retains local cleanup behind blocking workers. + +Large-body hashing and parsing run outside the async executor; worker failures +remain typed as `ReadError::ReadinessTask`. Streamed recipes move into that worker; +already-built publication catalogs clone borrowed recipes only at the worker +boundary. Both inputs share the physical body verifiers. Product caches may skip +repeated immutable-body validation, but must recheck the replica's authenticated +view digest before selection; a sampled probe does not publish full-readiness +cache state. ## Boundaries diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 330f45ab9..065d724dd 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -847,12 +847,22 @@ impl CapsuleRepositoryView { Ok(catalog.finish()?) } - /// Inspect complete captured catalog dependencies without retaining the merged catalog. - /// - /// File bindings and immutable conflicts retain capture order. Fetch-only - /// captures remain incomplete; origin state is never recaptured or written. - /// The caller retains these captured sources against GC through completion. + /// Collect complete captured shard roots without retaining the merged catalog. pub async fn pointer_catalog_shard_hashes(&self) -> Result> { + Ok(self + .pointer_catalog_inspection() + .await? + .into_shard_hashes() + .await?) + } + + /// Validate the complete captured catalog before streaming its external object descriptors. + /// + /// Fetch-only captures remain incomplete. Retain captured origin sources + /// through construction, then await the cursor's `finish` on every returned path. + pub async fn pointer_catalog_inspection( + &self, + ) -> Result { self.catalog_sources .as_ref() .ok_or(ReadError::IncompleteCatalogView)?; @@ -860,18 +870,16 @@ impl CapsuleRepositoryView { .metadata_layout .as_ref() .ok_or_else(|| ReadError::internal("catalog inspection has no captured origin"))?; - Ok( - crab_metadata::capsule_protocol::inspect_pointer_catalog_shard_hashes( - layout, - self.checkpoint - .as_ref() - .and_then(LayeredCheckpoint::catalog), - &self.capsules, - &self.capsule_controls, - crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, - ) - .await?, + Ok(crab_metadata::capsule_protocol::inspect_pointer_catalog( + layout, + self.checkpoint + .as_ref() + .and_then(LayeredCheckpoint::catalog), + &self.capsules, + &self.capsule_controls, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, ) + .await?) } /// Open the authenticated embedded Git packs as a filesystem-free repository. diff --git a/crates/crab-read/src/selection.rs b/crates/crab-read/src/selection.rs index dc5fcea13..fb3f08169 100644 --- a/crates/crab-read/src/selection.rs +++ b/crates/crab-read/src/selection.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::future::Future; use std::io::Write; @@ -479,10 +480,31 @@ pub async fn check_capsule_read_replica_readiness( } let mut stats = ReadinessProbeStats::default(); - let catalog = replica.pointer_catalog().await?; - if let Some(reason) = - capsule_pointer_object_gap(replica_router, &catalog, &mut stats, options).await? - { + let mut catalog = replica.pointer_catalog_inspection().await?; + // Metadata proof is complete before probing bodies. Keep the spill owner + // through every exit, including an early sample limit or provider error. + let result: Result> = async { + while !readiness_probe_budget_exhausted(&stats, options) { + let Some(object) = catalog.next_object().await? else { + break; + }; + let gap = match object { + crab_metadata::capsule_protocol::PointerCatalogObject::Shard { hash, entry } => { + verify_shard_object(replica_router, &hash, Cow::Owned(entry), &mut stats) + .await? + } + crab_metadata::capsule_protocol::PointerCatalogObject::Xorb { hash, entry } => { + verify_xorb_object(replica_router, &hash, Cow::Owned(entry), &mut stats).await? + } + }; + if gap.is_some() { + return Ok(gap); + } + } + Ok(None) + } + .await; + if let Some(reason) = catalog.finish(result).await? { let mut readiness = ReadReplicaReadiness::not_ready( primary_generation, Some(replica_generation), @@ -528,17 +550,7 @@ async fn capsule_pointer_object_gap( if readiness_probe_budget_exhausted(stats, options) { return Ok(None); } - let hash = parse_merkle_hash(hash, "shard")?; - let path = router.shard_path(&hash); - if let Some(reason) = verify_shard_object( - router, - &path, - hash, - shard.encoded_size(), - shard.xorb_hashes(), - stats, - ) - .await? + if let Some(reason) = verify_shard_object(router, hash, Cow::Borrowed(shard), stats).await? { return Ok(Some(reason)); } @@ -547,19 +559,7 @@ async fn capsule_pointer_object_gap( if readiness_probe_budget_exhausted(stats, options) { return Ok(None); } - let hash = parse_merkle_hash(hash, "xorb")?; - let path = router.xorb_path(&hash); - if let Some(reason) = verify_xorb_object( - router, - &path, - hash, - xorb.encoded_size(), - xorb.body_digest(), - xorb.chunks(), - stats, - ) - .await? - { + if let Some(reason) = verify_xorb_object(router, hash, Cow::Borrowed(xorb), stats).await? { return Ok(Some(reason)); } } @@ -568,16 +568,17 @@ async fn capsule_pointer_object_gap( async fn verify_shard_object( router: &StoreLayout, - path: &ObjectPath, - expected_hash: MerkleHash, - expected_size: u64, - expected_xorbs: &[String], + hash: &str, + entry: Cow<'_, crab_metadata::capsule_protocol::ShardCatalogEntry>, stats: &mut ReadinessProbeStats, ) -> Result> { + let expected_hash = parse_merkle_hash(hash, "shard")?; + let path = router.shard_path(&expected_hash); + let expected_size = entry.encoded_size(); stats.object_read_count = stats.object_read_count.saturating_add(1); let body = match router .store() - .get_with_etag_bounded(path, expected_size) + .get_with_etag_bounded(&path, expected_size) .await { Ok((body, _)) => body, @@ -594,9 +595,11 @@ async fn verify_shard_object( ))); } let path = path.to_string(); - let expected_xorbs = expected_xorbs.to_vec(); + // Streamed descriptors already own their recipe. Borrowed publication + // catalogs clone only at this blocking-worker lifetime boundary. + let entry = entry.into_owned(); tokio::task::spawn_blocking(move || { - verify_shard_body(&path, body, expected_hash, &expected_xorbs) + verify_shard_body(&path, body, expected_hash, entry.xorb_hashes()) }) .await .map_err(ReadError::ReadinessTask)? @@ -630,17 +633,17 @@ fn verify_shard_body( async fn verify_xorb_object( router: &StoreLayout, - path: &ObjectPath, - expected_hash: MerkleHash, - expected_size: u64, - expected_body_digest: &str, - expected_chunks: &[crab_metadata::capsule_protocol::XorbChunkEntry], + hash: &str, + entry: Cow<'_, crab_metadata::capsule_protocol::XorbCatalogEntry>, stats: &mut ReadinessProbeStats, ) -> Result> { + let expected_hash = parse_merkle_hash(hash, "xorb")?; + let path = router.xorb_path(&expected_hash); + let expected_size = entry.encoded_size(); stats.object_read_count = stats.object_read_count.saturating_add(1); let body = match router .store() - .get_with_etag_bounded(path, expected_size) + .get_with_etag_bounded(&path, expected_size) .await { Ok((body, _)) => body, @@ -657,15 +660,14 @@ async fn verify_xorb_object( ))); } let path = path.to_string(); - let expected_body_digest = expected_body_digest.to_owned(); - let expected_chunks = expected_chunks.to_vec(); + let entry = entry.into_owned(); tokio::task::spawn_blocking(move || { verify_xorb_body( &path, body, expected_hash, - &expected_body_digest, - &expected_chunks, + entry.body_digest(), + entry.chunks(), ) }) .await @@ -1177,6 +1179,83 @@ mod tests { .unwrap() } + async fn capsule_catalog_view( + router: &StoreLayout, + catalog: &PointerCatalog, + ) -> crate::capsule_protocol::CapsuleRepositoryView { + use crab_metadata::capsule_protocol::{ + Capsule, CapsulePointer, CapsuleRefEdit, CapsuleRun, CapsuleSection, + CapsuleSectionKind, CapsuleTransaction, + }; + let initial = RootRecord::encode( + RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap(), + ) + .unwrap(); + let oid = "2".repeat(40); + let transaction = CapsuleTransaction::new( + initial.digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some(oid.clone()), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )], + ) + .unwrap(); + let run = CapsuleRun::leaf(capsule).unwrap(); + router + .store() + .create_strict(&router.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let transaction_id = run.transaction_ids()[0].clone(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let root = initial + .root() + .advance( + initial.digest(), + std::collections::BTreeMap::from([("refs/heads/main".to_owned(), oid)]), + std::collections::BTreeMap::new(), + vec![pointer], + &transaction_id, + ) + .unwrap(); + let root = RootRecord::encode(root).unwrap(); + router + .store() + .create_strict(&router.capsule_root_path(), root.bytes().clone()) + .await + .unwrap(); + crate::capsule_protocol::open_view( + router, + crate::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 1024 * 1024, + }, + ) + .await + .unwrap() + } + #[tokio::test] async fn capsule_readiness_requires_the_exact_primary_view() { let (_primary_store, primary_router) = memory_store_with_layout("org/repo"); @@ -1346,6 +1425,59 @@ mod tests { assert!(gap.is_none()); assert_eq!(stats.object_read_count, 2); + + let primary = capsule_catalog_view(&router, &catalog).await; + let readiness = + check_capsule_read_replica_readiness(&primary, &router, ReadinessCheckOptions::deep()) + .await + .unwrap(); + assert!(readiness.ready); + assert_eq!(readiness.stats.object_read_count, 2); + store.delete(&router.xorb_path(&xorb.hash)).await.unwrap(); + let sampled = check_capsule_read_replica_readiness( + &primary, + &router, + ReadinessCheckOptions::sampled(1), + ) + .await + .unwrap(); + assert!(sampled.ready); + assert_eq!(sampled.stats.object_read_count, 1); + let complete = + check_capsule_read_replica_readiness(&primary, &router, ReadinessCheckOptions::deep()) + .await + .unwrap(); + assert!(!complete.ready); + assert_eq!(complete.stats.object_read_count, 2); + assert!(complete.reason.unwrap().contains("xorb missing")); + } + + #[tokio::test] + async fn sampled_capsule_readiness_requires_complete_metadata_before_any_payload_probe() { + let (_store, router) = memory_store_with_layout("org/repo"); + let mut catalog = PointerCatalog::new(); + catalog + .insert_file( + "a".repeat(64), + crab_metadata::capsule_protocol::FileCatalogEntry::new(3, "b".repeat(64)), + ) + .unwrap(); + let primary = capsule_catalog_view(&router, &catalog).await; + let result = check_capsule_read_replica_readiness( + &primary, + &router, + ReadinessCheckOptions { + max_object_probes: Some(0), + ..ReadinessCheckOptions::deep() + }, + ) + .await; + assert!(matches!( + result, + Err(ReadError::Metadata( + crab_metadata::error::MetadataError::CorruptObject { .. } + )) + )); } #[tokio::test] From 727da99396c63bc65be80c7f45942ea643ce3f6a Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 04:39:12 -0700 Subject: [PATCH 123/127] perf(read): stream deep dependency catalogs Verify captured catalog descriptors through the spill owner and resolve reachable file bindings in bounded batches. Share the Git scan and whole-file proof with recovery-owned catalogs; reuse proof cardinalities in metadata diagnosis and rebuild summaries. Admit complete catalog authority before source I/O so fetch-only views retain their early incomplete-view error and perform no provider reads. --- .../docs/architecture/indexed-remote-reads.md | 8 +- crab/src/cmd/metadb.rs | 29 +- crates/crab-metadata/README.md | 3 +- .../crab-metadata/src/capsule_protocol/mod.rs | 3 +- .../src/capsule_protocol/pointer.rs | 4 +- .../capsule_protocol/pointer/inspection.rs | 33 +- .../pointer/inspection/tests.rs | 12 +- crates/crab-read/AGENTS.md | 1 + crates/crab-read/README.md | 8 + crates/crab-read/src/capsule_protocol.rs | 248 ++----------- .../src/capsule_protocol/dependency.rs | 336 ++++++++++++++++++ .../src/capsule_protocol/dependency_tests.rs | 320 ++++++++++++++--- crates/crab-read/src/selection.rs | 124 +++++-- 13 files changed, 803 insertions(+), 326 deletions(-) create mode 100644 crates/crab-read/src/capsule_protocol/dependency.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 80ec8ecd3..5a696eff9 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2546,8 +2546,12 @@ shard/xorb descriptors through the shared physical body verifiers. No body probe starts before the final metadata proof; all returned paths close the cursor, including provider errors and sample limits. Pending reads advance only after verified I/O and decoding, so cancellation followed by resume cannot skip a block. -Current fsck payload verification, restore accounting and catalog-rewriting tools -also retain full logical catalogs. Materialization admission continues to +Deep reachable-dependency verification now streams that same proof and physical +body checks before batching reachable file bindings from the captured roots. +Metadata diagnosis and rebuild summaries consume verified cardinalities without +a separate materialization. Recovery's assembled replacement catalogs share the +same Git scan and whole-file/LFS verification owner. Current fsck payload +verification, restore accounting and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to protect those callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before diff --git a/crab/src/cmd/metadb.rs b/crab/src/cmd/metadb.rs index 1cf983cb2..9aa9bf79d 100644 --- a/crab/src/cmd/metadb.rs +++ b/crab/src/cmd/metadb.rs @@ -1737,7 +1737,6 @@ async fn diagnose_capsule( }, ) .await?; - let catalog = view.pointer_catalog().await?; view.git_visibility_index().await?; // Verification must read layered sources and prove reachable content, // not rebuild a complete pack or accept catalog membership as file proof. @@ -1774,13 +1773,11 @@ async fn diagnose_capsule( git_closure_verified: true, file_entries: db .includes_file_index() - .then(|| diagnosis_count(catalog.files().len(), "capsule file entry")) - .transpose()?, - shard_entries: diagnosis_count(catalog.shards().len(), "capsule shard entry")?, + .then_some(dependencies.catalog_files), + shard_entries: dependencies.catalog_shards, xorb_entries: db .includes_chunk_index() - .then(|| diagnosis_count(catalog.xorbs().len(), "capsule xorb entry")) - .transpose()?, + .then_some(dependencies.catalog_xorbs), pointer_objects_read: dependencies.catalog_objects_read, verdict: "OK — root, ref heads, capsules, checkpoint, catalogs, visibility, and Git packs authenticated", }), @@ -2662,11 +2659,10 @@ async fn run_capsule_rebuild_in( }, ) .await?; - let catalog = view.pointer_catalog().await?; view.git_visibility_index().await?; // Authenticate the complete closure before either a no-op success or a new // checkpoint; immutable publication alone does not verify reachable files. - crab_read::capsule_protocol::verify_reachable_dependencies( + let dependencies = crab_read::capsule_protocol::verify_reachable_dependencies( layout, &view, crab_read::capsule_protocol::CapsuleDependencyLimits { @@ -2720,21 +2716,12 @@ async fn run_capsule_rebuild_in( protocol: "capsule-v2", repo_prefix: repo_prefix.to_owned(), checkpoint_published: Some(checkpoint_published), - catalog_files_verified: Some(diagnosis_count( - catalog.files().len(), - "capsule file entry", - )?), - catalog_shards_verified: Some(diagnosis_count( - catalog.shards().len(), - "capsule shard entry", - )?), - catalog_xorbs_verified: Some(diagnosis_count( - catalog.xorbs().len(), - "capsule xorb entry", - )?), + catalog_files_verified: Some(dependencies.catalog_files), + catalog_shards_verified: Some(dependencies.catalog_shards), + catalog_xorbs_verified: Some(dependencies.catalog_xorbs), file_index_entries_written: 0, chunk_index_entries_written: 0, - shards_processed: diagnosis_count(catalog.shards().len(), "capsule shard entry")?, + shards_processed: dependencies.catalog_shards, shards_failed: 0, git_packs_processed: git_packs, git_packs_failed: 0, diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 70c95d713..af455ce1e 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -101,7 +101,8 @@ shard identities from this owner; replica readiness streams full shard/xorb descriptors without a joined catalog in memory. The cursor retains the normalized run until `finish` drains workers and removes it, preserving the operation's primary error if cleanup also fails. A cancelled pending read can resume at the same -object. Physical page identities, spill handles and the largest recipe still +object. Complete cardinalities remain bound to the proof even after cursor +consumption; sampled physical checks cannot stand in for these counts. Physical page identities, spill handles and the largest recipe still consume memory; each packed delta retains its format-bounded decoder. Logical reduction has a live disk ceiling of sixteen times the admitted catalog limit; normal lookup and materialization budgets remain unchanged. diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 23e972442..8595677ec 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -73,7 +73,8 @@ pub use pointer::{ }; #[cfg(feature = "storage")] pub use pointer::{ - PointerCatalogInspection, PointerCatalogLookup, PointerCatalogObject, inspect_pointer_catalog, + PointerCatalogCounts, PointerCatalogInspection, PointerCatalogLookup, PointerCatalogObject, + inspect_pointer_catalog, }; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index d97c88495..bae7c2a6b 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -18,7 +18,9 @@ mod relations; #[cfg(feature = "storage")] pub(super) mod spill; #[cfg(feature = "storage")] -pub use inspection::{PointerCatalogInspection, PointerCatalogObject, inspect_pointer_catalog}; +pub use inspection::{ + PointerCatalogCounts, PointerCatalogInspection, PointerCatalogObject, inspect_pointer_catalog, +}; pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] pub use paged::PointerCatalogLookup; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs index 8d53a6f43..f4c53a1d0 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs @@ -238,15 +238,26 @@ impl Inspector { return Ok(PointerCatalogInspection { space: None, rows: Cursor::new(Run::default()), + counts: PointerCatalogCounts::default(), }); }; - let result: Result = async { + let result: Result<(Run, PointerCatalogCounts)> = async { prior?; let run = self.rows.finish(&space).await?; let mut cursor = Cursor::::new(run.clone()); let mut dependencies = Reducer::::new(); + let mut counts = PointerCatalogCounts::default(); while let Some(row) = cursor.next(&space).await? { let identity = &row.key[1..]; + let count = match row.key[0] { + 0 => &mut counts.files, + 1 => &mut counts.shards, + 2 => &mut counts.xorbs, + _ => return Err(corrupt("logical catalog row kind differs")), + }; + *count = count + .checked_add(1) + .ok_or_else(|| corrupt("catalog count overflows"))?; match row.key[0] { 0 => { dependencies @@ -277,13 +288,14 @@ impl Inspector { row.validate()?; } dependencies.remove(&space).await?; - Ok(run) + Ok((run, counts)) } .await; match result { - Ok(run) => Ok(PointerCatalogInspection { + Ok((run, counts)) => Ok(PointerCatalogInspection { space: Some(space), rows: Cursor::new(run), + counts, }), Err(error) => finish(Some(space), Err(error)).await, } @@ -303,6 +315,14 @@ pub enum PointerCatalogObject { }, } +/// Cardinalities of one complete, precedence-resolved captured catalog. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PointerCatalogCounts { + pub files: u64, + pub shards: u64, + pub xorbs: u64, +} + /// A complete catalog proof with bounded iteration over its external object descriptors. /// /// The private spill is owned until `finish` drains and removes it. Dropping the @@ -311,9 +331,16 @@ pub enum PointerCatalogObject { pub struct PointerCatalogInspection { space: Option, rows: Cursor, + counts: PointerCatalogCounts, } impl PointerCatalogInspection { + /// Return complete catalog cardinalities, independent of cursor consumption. + #[must_use] + pub fn counts(&self) -> PointerCatalogCounts { + self.counts + } + async fn next_entry(&mut self) -> Result> { match &self.space { Some(space) => self.rows.next(space).await, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs index 973457d72..ed0e0469b 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -316,6 +316,7 @@ async fn cursor_fixture( #[tokio::test] async fn verified_cursor_preserves_complete_descriptors_across_spilled_file_rows() { let (_parent, mut cursor, expected) = cursor_fixture(5000).await; + let before = cursor.counts(); let result: Result<_> = async { let mut values = Vec::new(); while let Some(object) = cursor.next_object().await? { @@ -324,7 +325,16 @@ async fn verified_cursor_preserves_complete_descriptors_across_spilled_file_rows Ok(values) } .await; - assert_eq!(cursor.finish(result).await.unwrap(), expected); + let after = cursor.counts(); + let counts = PointerCatalogCounts { + files: 5000, + shards: 1, + xorbs: 1, + }; + assert_eq!( + (before, after, cursor.finish(result).await.unwrap()), + (counts, counts, expected) + ); } #[tokio::test] diff --git a/crates/crab-read/AGENTS.md b/crates/crab-read/AGENTS.md index ab35a3713..f472bc904 100644 --- a/crates/crab-read/AGENTS.md +++ b/crates/crab-read/AGENTS.md @@ -27,6 +27,7 @@ For object download changes continue into `crates/crab-read/src/store_client.rs` | --- | --- | --- | | Hydration or range integrity | `crates/crab-read/src/hydrator.rs` | `crates/crab-vfs/src/hydration.rs` | | Fetch authorization | `crates/crab-read/src/fetch_admission.rs` | `crab/src/git/remote_helper.rs` | +| Deep dependency proof | `crates/crab-read/src/capsule_protocol/dependency.rs` | CLI metadata diagnosis, recovery and HTTP integrity | | Replica readiness | `crates/crab-read/src/selection.rs` | `crab/src/replication/mod.rs` | | Term resolution | `crates/crab-read/src/term_resolver.rs` | `crates/crab-metadata/src/file_index_lookup.rs` | diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 294c2ebbc..c5228b31d 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -498,3 +498,11 @@ reconciled logical delta state; both catalog APIs return `ReadError::IncompleteCatalogView` before I/O rather than treating that state as an authenticated miss. A complete capture with an empty suffix can still return a legitimate miss. + +Deep `capsule_protocol::verify_reachable_dependencies` streams the complete +captured catalog proof and physical descriptor verification, then resolves +reachable pointer bindings in batches of 64 from those same roots. Catalog +cardinalities come from the completed metadata reduction. Recovery's assembled +replacement catalogs share the Git scan and whole-file/LFS proof in +`capsule_protocol/dependency.rs`; binding source selection does not change the +integrity checks. Current fsck's additional cross-recipe checks remain separate. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 065d724dd..5011492dc 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -23,6 +23,9 @@ use tokio_util::sync::CancellationToken; use crate::{ReadError, Result}; +mod dependency; +pub use dependency::{verify_installed_dependencies, verify_reachable_dependencies}; + #[cfg(test)] mod visibility_tests; @@ -1663,230 +1666,6 @@ pub async fn install_git_packs( install_git_packs_with_candidates(view, &[], git_dir, max_input_bytes).await } -/// Verify every Git and external content dependency reachable from one view. -/// -/// This is a deep administrative proof, not a foreground read-path check. It -/// authenticates complete immutable sources, validates the shard/xorb catalog, -/// installs all Git packs in an isolated database, and hashes every reachable -/// Crab file and LFS object at origin. Source verification and pack installation -/// have separate intake phases bounded by `max_git_bytes`. -/// Callers must signal token cancellation and await completion before shutting -/// down the runtime; native installation and Git scan workers drain before return. -pub async fn verify_reachable_dependencies( - layout: &StoreLayout, - view: &CapsuleRepositoryView, - limits: CapsuleDependencyLimits, - cancellation: &CancellationToken, -) -> Result { - if cancellation.is_cancelled() { - return Err(ReadError::Cancelled); - } - let catalog = view.pointer_catalog().await?; - let mut sources = BTreeMap::new(); - for source in view - .layered_checkpoint() - .into_iter() - .flat_map(LayeredCheckpoint::sources) - .chain(view.capsule_run_sources()) - { - let path = match source.kind() { - PackSourceKind::CapsuleRun => layout.capsule_path(source.object_hash()), - PackSourceKind::PackLayer => layout.capsule_pack_layer_path(source.object_hash()), - }; - if let Some(previous) = sources.insert(path.clone(), source) - && previous != source - { - return Err(corrupt_path( - path.to_string(), - "immutable source has conflicting descriptors", - )); - } - } - let source_bytes = sources.values().try_fold(0_u64, |total, source| { - total - .checked_add(source.object_size()) - .ok_or_else(|| ReadError::internal("layered source byte count overflowed")) - })?; - if limits.max_git_bytes > 0 && source_bytes > limits.max_git_bytes { - return Err(ReadError::CapsuleReadLimit { - resource: "layered source bodies", - maximum: limits.max_git_bytes, - }); - } - let pointers = view - .capsule_run_pointers() - .iter() - .map(|pointer| (pointer.hash(), pointer)) - .collect::>(); - // Range installation proves Git members, not unused source framing. Deep - // administrative proof authenticates each complete source once, separately. - for source in sources.into_values() { - let retained_run = match source.kind() { - PackSourceKind::CapsuleRun => pointers.get(source.object_hash()).copied(), - PackSourceKind::PackLayer => None, - }; - tokio::select! { - biased; - () = cancellation.cancelled() => return Err(ReadError::Cancelled), - result = verify_layered_source(layout, source, retained_run, source.object_size()) => { - result?; - } - } - } - if cancellation.is_cancelled() { - return Err(ReadError::Cancelled); - } - let workspace = tempfile::tempdir()?; - crab_git::initialize_bare_git_dir(workspace.path()).map_err(std::io::Error::other)?; - let installation = install_git_packs_from_store( - view, - layout, - workspace.path(), - limits.max_git_bytes, - None, - cancellation, - ); - tokio::pin!(installation); - tokio::select! { - biased; - () = cancellation.cancelled() => { - // A started blocking installer cannot be aborted. Keep its database - // alive until it finishes instead of detaching writes into a removed path. - let _ = installation.await; - return Err(ReadError::Cancelled); - } - result = &mut installation => { - result?; - } - } - verify_installed_dependencies( - layout, - workspace.path(), - view.refs(), - &catalog, - limits.pointer_scan, - cancellation, - ) - .await -} - -/// Verify catalog bodies and reachable Crab/LFS pointers in an installed Git database. -/// -/// Callers must install authenticated packs, pin the supplied refs and catalog, -/// and keep the database alive until this future completes. Token cancellation -/// drains its Git scan before returning; catalog and whole-file bytes are checked at origin. -pub async fn verify_installed_dependencies( - layout: &StoreLayout, - git_dir: &Path, - refs: &BTreeMap, - catalog: &PointerCatalog, - limits: crab_git::walk::PointerScanLimits, - cancellation: &CancellationToken, -) -> Result { - let catalog_stats = tokio::select! { - biased; - () = cancellation.cancelled() => return Err(ReadError::Cancelled), - result = crate::verify_capsule_pointer_catalog_objects(layout, catalog) => { - result? - } - }; - let refs = refs - .iter() - .map(|(name, oid)| (name.clone(), oid.clone())) - .collect::>(); - let git_dir = git_dir.to_owned(); - let scan_cancel = cancellation.child_token(); - let worker_cancel = scan_cancel.clone(); - let scan = tokio::task::spawn_blocking(move || -> Result<_> { - let scan = crab_git::walk::scan_pointers(&git_dir, &refs, limits, &|| { - worker_cancel.is_cancelled() - })?; - crab_git::batch::verify_git_dir_blobs(&git_dir, &scan.unchecked_blobs, &|| { - worker_cancel.is_cancelled() - })?; - Ok(scan) - }); - tokio::pin!(scan); - let scan = tokio::select! { - biased; - () = cancellation.cancelled() => { - scan_cancel.cancel(); - // The worker borrows the temporary Git database by path. Drain it - // before that database and any caller-owned admission are released. - let _ = scan.await; - return Err(ReadError::Cancelled); - } - result = &mut scan => result - .map_err(|error| ReadError::Io(std::io::Error::other(error)))??, - }; - - let mut verified_files = BTreeSet::new(); - for pointer in &scan.pointers { - let hash = crab_xet::hash::MerkleHash::from(pointer.file_hash).hex(); - let Some(entry) = catalog.files().get(&hash) else { - return Err(ReadError::CorruptObject { - path: gix_hash::ObjectId::Sha1(pointer.oid).to_string(), - reason: format!("reachable Crab pointer {hash} is absent from the catalog"), - }); - }; - if entry.size() != pointer.size { - return Err(ReadError::CorruptObject { - path: gix_hash::ObjectId::Sha1(pointer.oid).to_string(), - reason: format!( - "reachable Crab pointer {hash} declares size {}, catalog declares {}", - pointer.size, - entry.size() - ), - }); - } - // Different Git pointer blobs can name the same content. Validate every - // declared size above, but reconstruct each file version only once. - if verified_files.insert(pointer.file_hash) { - let pointer = crab_types::pointer::Pointer { - file_hash: pointer.file_hash, - size: pointer.size, - shard_hint: None, - }; - crate::verify_catalog_file_recipe(layout, &pointer, entry, cancellation).await?; - } - } - - let lfs = crab_lfs::LfsObjectStore::new(layout.store().clone(), layout.repo_prefix()); - let mut lfs_objects = BTreeMap::new(); - for blob in scan.lfs_pointers { - if let Some(previous) = lfs_objects.insert(blob.pointer.oid, blob.pointer.size) - && previous != blob.pointer.size - { - return Err(ReadError::CorruptObject { - path: gix_hash::ObjectId::Sha1(blob.oid).to_string(), - reason: "reachable LFS pointers declare conflicting sizes for one object" - .to_owned(), - }); - } - } - for (oid, size) in &lfs_objects { - tokio::select! { - biased; - () = cancellation.cancelled() => return Err(ReadError::Cancelled), - result = lfs.verify_origin(oid, *size) => result?, - } - } - - Ok(CapsuleDependencyProof { - catalog_files: catalog.files().len() as u64, - catalog_shards: catalog.shards().len() as u64, - catalog_xorbs: catalog.xorbs().len() as u64, - catalog_objects_read: catalog_stats.object_read_count, - reachable_crab_pointers: scan.pointers.len() as u64, - reachable_lfs_objects: lfs_objects.len() as u64, - reachable_lfs_bytes: lfs_objects.values().try_fold(0_u64, |total, size| { - total - .checked_add(*size) - .ok_or_else(|| ReadError::internal("LFS dependency byte count overflowed")) - })?, - }) -} - /// Install the pinned view plus additional verified candidate capsules. /// /// The caller must separately bind each candidate transaction to the pinned @@ -5424,10 +5203,6 @@ fn corrupt_path(path: impl Into, reason: impl Into) -> ReadError } } -#[cfg(test)] -#[path = "capsule_protocol/dependency_tests.rs"] -mod dependency_tests; - #[cfg(test)] #[expect(clippy::unwrap_used, reason = "test assertions")] mod tests { @@ -6135,6 +5910,23 @@ mod tests { fetch.pointer_catalog_shard_hashes().await, Err(ReadError::IncompleteCatalogView) )); + assert!(matches!( + verify_reachable_dependencies( + &router, + &fetch, + CapsuleDependencyLimits { + max_git_bytes: 1024 * 1024, + pointer_scan: crab_git::walk::PointerScanLimits { + objects: 16, + lookups: 64, + allocation_bytes: 1024 * 1024 + }, + }, + &CancellationToken::new(), + ) + .await, + Err(ReadError::IncompleteCatalogView) + )); assert!(observer.observations.lock().unwrap().is_empty()); let complete = open_git_view_from_root(&router, root, TEST_LIMITS) .await diff --git a/crates/crab-read/src/capsule_protocol/dependency.rs b/crates/crab-read/src/capsule_protocol/dependency.rs new file mode 100644 index 000000000..9e47ffa6b --- /dev/null +++ b/crates/crab-read/src/capsule_protocol/dependency.rs @@ -0,0 +1,336 @@ +//! Complete dependency proof with streamed captured catalogs and selected file bindings. +use super::{ + CapsuleDependencyLimits, CapsuleDependencyProof, CapsuleRepositoryView, LayeredCheckpoint, + PackSourceKind, corrupt_path, install_git_packs_from_store, verify_layered_source, +}; +use crate::{ReadError, Result}; +use crab_metadata::capsule_protocol::{ + FileCatalogEntry, PointerCatalog, PointerCatalogCounts, PointerCatalogLookup, +}; +use crab_storage::{Store, StoreLayout}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use tokio_util::sync::CancellationToken; + +// Recovery owns an already assembled replacement catalog; captured views own +// authenticated lookup roots. Both feed the same Git scan and file-byte proof. +enum DependencyCatalog<'a> { + Assembled(&'a PointerCatalog), + Captured { + view: &'a CapsuleRepositoryView, + lookup: &'a PointerCatalogLookup, + }, +} + +impl DependencyCatalog<'_> { + async fn verify( + &self, + layout: &StoreLayout, + cancel: &CancellationToken, + ) -> Result<(PointerCatalogCounts, crate::ReadinessProbeStats)> { + match self { + Self::Assembled(catalog) => { + let stats = tokio::select! { + biased; + () = cancel.cancelled() => return Err(ReadError::Cancelled), + result = crate::verify_capsule_pointer_catalog_objects(layout, catalog) => result?, + }; + Ok(( + PointerCatalogCounts { + files: catalog.files().len() as u64, + shards: catalog.shards().len() as u64, + xorbs: catalog.xorbs().len() as u64, + }, + stats, + )) + } + Self::Captured { view, .. } => { + let inspection = view.pointer_catalog_inspection().await?; + crate::selection::verify_capsule_pointer_inspection(layout, inspection, cancel) + .await + } + } + } + + async fn files( + &self, + layout: &StoreLayout, + hashes: &[String], + ) -> Result> { + match self { + Self::Assembled(catalog) => Ok(hashes + .iter() + .filter_map(|hash| { + catalog + .files() + .get(hash) + .map(|entry| (hash.clone(), entry.clone())) + }) + .collect()), + Self::Captured { lookup, .. } => { + let selected = lookup.files(layout.store(), hashes).await?; + Ok(selected.files().clone()) + } + } + } +} + +/// Verify every Git and external content dependency reachable from one view. +/// +/// This is a deep administrative proof, not a foreground read-path check. It +/// authenticates complete immutable sources, validates the shard/xorb catalog, +/// installs all Git packs in an isolated database, and hashes every reachable +/// Crab file and LFS object at origin. Source verification and pack installation +/// have separate intake phases bounded by `max_git_bytes`. +/// Callers must signal token cancellation and await completion before shutting +/// down the runtime; native installation and Git scan workers drain before return. +pub async fn verify_reachable_dependencies( + layout: &StoreLayout, + view: &CapsuleRepositoryView, + limits: CapsuleDependencyLimits, + cancellation: &CancellationToken, +) -> Result { + if cancellation.is_cancelled() { + return Err(ReadError::Cancelled); + } + // Fetch-only views have no complete catalog authority. Reject them before + // source I/O or temporary Git installation, even when their packs are invalid. + let lookup = + view.pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; + let mut sources = BTreeMap::new(); + for source in view + .layered_checkpoint() + .into_iter() + .flat_map(LayeredCheckpoint::sources) + .chain(view.capsule_run_sources()) + { + let path = match source.kind() { + PackSourceKind::CapsuleRun => layout.capsule_path(source.object_hash()), + PackSourceKind::PackLayer => layout.capsule_pack_layer_path(source.object_hash()), + }; + if let Some(previous) = sources.insert(path.clone(), source) + && previous != source + { + return Err(corrupt_path( + path.to_string(), + "immutable source has conflicting descriptors", + )); + } + } + let source_bytes = sources.values().try_fold(0_u64, |total, source| { + total + .checked_add(source.object_size()) + .ok_or_else(|| ReadError::internal("layered source byte count overflowed")) + })?; + if limits.max_git_bytes > 0 && source_bytes > limits.max_git_bytes { + return Err(ReadError::CapsuleReadLimit { + resource: "layered source bodies", + maximum: limits.max_git_bytes, + }); + } + let pointers = view + .capsule_run_pointers() + .iter() + .map(|pointer| (pointer.hash(), pointer)) + .collect::>(); + // Range installation proves Git members, not unused source framing. Deep + // administrative proof authenticates each complete source once, separately. + for source in sources.into_values() { + let retained_run = match source.kind() { + PackSourceKind::CapsuleRun => pointers.get(source.object_hash()).copied(), + PackSourceKind::PackLayer => None, + }; + tokio::select! { + biased; + () = cancellation.cancelled() => return Err(ReadError::Cancelled), + result = verify_layered_source(layout, source, retained_run, source.object_size()) => { + result?; + } + } + } + if cancellation.is_cancelled() { + return Err(ReadError::Cancelled); + } + let workspace = tempfile::tempdir()?; + crab_git::initialize_bare_git_dir(workspace.path()).map_err(std::io::Error::other)?; + let installation = install_git_packs_from_store( + view, + layout, + workspace.path(), + limits.max_git_bytes, + None, + cancellation, + ); + tokio::pin!(installation); + tokio::select! { + biased; + () = cancellation.cancelled() => { + // A started blocking installer cannot be aborted. Keep its database + // alive until it finishes instead of detaching writes into a removed path. + let _ = installation.await; + return Err(ReadError::Cancelled); + } + result = &mut installation => { + result?; + } + } + verify_installed_dependencies_from( + layout, + workspace.path(), + view.refs(), + DependencyCatalog::Captured { + view, + lookup: &lookup, + }, + limits.pointer_scan, + cancellation, + ) + .await +} + +/// Verify catalog bodies and reachable Crab/LFS pointers in an installed Git database. +/// +/// Callers must install authenticated packs, pin the supplied refs and catalog, +/// and keep the database alive until this future completes. Token cancellation +/// drains its Git scan before returning; catalog and whole-file bytes are checked at origin. +pub async fn verify_installed_dependencies( + layout: &StoreLayout, + git_dir: &Path, + refs: &BTreeMap, + catalog: &PointerCatalog, + limits: crab_git::walk::PointerScanLimits, + cancellation: &CancellationToken, +) -> Result { + verify_installed_dependencies_from( + layout, + git_dir, + refs, + DependencyCatalog::Assembled(catalog), + limits, + cancellation, + ) + .await +} + +async fn verify_installed_dependencies_from( + layout: &StoreLayout, + git_dir: &Path, + refs: &BTreeMap, + catalog: DependencyCatalog<'_>, + limits: crab_git::walk::PointerScanLimits, + cancellation: &CancellationToken, +) -> Result { + let (counts, catalog_stats) = catalog.verify(layout, cancellation).await?; + let refs = refs + .iter() + .map(|(name, oid)| (name.clone(), oid.clone())) + .collect::>(); + let git_dir = git_dir.to_owned(); + let scan_cancel = cancellation.child_token(); + let worker_cancel = scan_cancel.clone(); + let scan = tokio::task::spawn_blocking(move || -> Result<_> { + let scan = crab_git::walk::scan_pointers(&git_dir, &refs, limits, &|| { + worker_cancel.is_cancelled() + })?; + crab_git::batch::verify_git_dir_blobs(&git_dir, &scan.unchecked_blobs, &|| { + worker_cancel.is_cancelled() + })?; + Ok(scan) + }); + tokio::pin!(scan); + let scan = tokio::select! { + biased; + () = cancellation.cancelled() => { + scan_cancel.cancel(); + // The worker borrows the temporary Git database by path. Drain it + // before that database and any caller-owned admission are released. + let _ = scan.await; + return Err(ReadError::Cancelled); + } + result = &mut scan => result + .map_err(|error| ReadError::Io(std::io::Error::other(error)))??, + }; + + let mut verified_files = BTreeSet::new(); + for batch in scan.pointers.chunks(64) { + let hashes = batch + .iter() + .map(|pointer| crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) + .collect::>(); + // Selected bindings share the captured roots with the complete proof. + // Bound each query's recipe expansion independently of reachable history. + let entries = tokio::select! { + biased; + () = cancellation.cancelled() => return Err(ReadError::Cancelled), + result = catalog.files(layout, &hashes) => result?, + }; + for pointer in batch { + let hash = crab_xet::hash::MerkleHash::from(pointer.file_hash).hex(); + let Some(entry) = entries.get(&hash) else { + return Err(ReadError::CorruptObject { + path: gix_hash::ObjectId::Sha1(pointer.oid).to_string(), + reason: format!("reachable Crab pointer {hash} is absent from the catalog"), + }); + }; + if entry.size() != pointer.size { + return Err(ReadError::CorruptObject { + path: gix_hash::ObjectId::Sha1(pointer.oid).to_string(), + reason: format!( + "reachable Crab pointer {hash} declares size {}, catalog declares {}", + pointer.size, + entry.size() + ), + }); + } + // Different Git pointer blobs can name the same content. Validate every + // declared size above, but reconstruct each file version only once. + if verified_files.insert(pointer.file_hash) { + let pointer = crab_types::pointer::Pointer { + file_hash: pointer.file_hash, + size: pointer.size, + shard_hint: None, + }; + crate::verify_catalog_file_recipe(layout, &pointer, entry, cancellation).await?; + } + } + } + + let lfs = crab_lfs::LfsObjectStore::new(layout.store().clone(), layout.repo_prefix()); + let mut lfs_objects = BTreeMap::new(); + for blob in scan.lfs_pointers { + if let Some(previous) = lfs_objects.insert(blob.pointer.oid, blob.pointer.size) + && previous != blob.pointer.size + { + return Err(ReadError::CorruptObject { + path: gix_hash::ObjectId::Sha1(blob.oid).to_string(), + reason: "reachable LFS pointers declare conflicting sizes for one object" + .to_owned(), + }); + } + } + for (oid, size) in &lfs_objects { + tokio::select! { + biased; + () = cancellation.cancelled() => return Err(ReadError::Cancelled), + result = lfs.verify_origin(oid, *size) => result?, + } + } + + Ok(CapsuleDependencyProof { + catalog_files: counts.files, + catalog_shards: counts.shards, + catalog_xorbs: counts.xorbs, + catalog_objects_read: catalog_stats.object_read_count, + reachable_crab_pointers: scan.pointers.len() as u64, + reachable_lfs_objects: lfs_objects.len() as u64, + reachable_lfs_bytes: lfs_objects.values().try_fold(0_u64, |total, size| { + total + .checked_add(*size) + .ok_or_else(|| ReadError::internal("LFS dependency byte count overflowed")) + })?, + }) +} + +#[cfg(test)] +#[path = "dependency_tests.rs"] +mod tests; diff --git a/crates/crab-read/src/capsule_protocol/dependency_tests.rs b/crates/crab-read/src/capsule_protocol/dependency_tests.rs index 5d94fb7fd..99bc8cf28 100644 --- a/crates/crab-read/src/capsule_protocol/dependency_tests.rs +++ b/crates/crab-read/src/capsule_protocol/dependency_tests.rs @@ -1,6 +1,7 @@ #![expect(clippy::unwrap_used, reason = "test assertions")] -use super::*; +use super::super::*; +use super::{DependencyCatalog, verify_installed_dependencies_from}; use crab_metadata::capsule_protocol::{ FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, }; @@ -100,12 +101,14 @@ async fn fixture( ShardCatalogEntry::new(shard_body.len() as u64, vec![xorb.hash.hex()]), ) .unwrap(); - catalog - .insert_file( - MerkleHash::from(pointer.file_hash).hex(), - FileCatalogEntry::new(pointer.size, shard_hash.hex()), - ) - .unwrap(); + if recipe_case != "missing_binding" { + catalog + .insert_file( + MerkleHash::from(pointer.file_hash).hex(), + FileCatalogEntry::new(pointer.size, shard_hash.hex()), + ) + .unwrap(); + } layout .store() .put(&layout.xorb_path(&xorb.hash), xorb.bytes) @@ -129,6 +132,117 @@ async fn fixture( ) } +async fn captured_catalog( + layout: &StoreLayout, + catalog: &PointerCatalog, + refs: &BTreeMap, +) -> CapsuleRepositoryView { + use crab_metadata::capsule_protocol::{ + CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, RepositoryRoot, + }; + let initial = + RootRecord::encode(RepositoryRoot::initial(&"1".repeat(64), "refs/heads/main").unwrap()) + .unwrap(); + let transaction = CapsuleTransaction::new( + initial.digest(), + refs.iter() + .map(|(name, oid)| CapsuleRefEdit::new(name, None, Some(oid.clone()), None)) + .collect(), + ) + .unwrap(); + let run = CapsuleRun::leaf( + Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )], + ) + .unwrap(), + ) + .unwrap(); + layout + .store() + .put(&layout.capsule_path(run.hash()), run.bytes().clone()) + .await + .unwrap(); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let root = initial + .root() + .advance( + initial.digest(), + refs.clone(), + BTreeMap::new(), + vec![pointer], + &transaction.id().unwrap(), + ) + .unwrap(); + let root = RootRecord::encode(root).unwrap(); + layout + .store() + .put_overwrite(&layout.capsule_root_path(), root.bytes().clone()) + .await + .unwrap(); + open_view( + layout, + CapsuleReadLimits { + max_capsule_bytes: 16 * 1024 * 1024, + max_frontier_bytes: 16 * 1024 * 1024, + }, + ) + .await + .unwrap() +} + +async fn verify_fixture( + layout: &StoreLayout, + workspace: &Path, + refs: &BTreeMap, + catalog: &PointerCatalog, + captured: bool, +) -> Result { + let cancel = CancellationToken::new(); + let git_dir = workspace.join("repository.git"); + if !captured { + return verify_installed_dependencies( + layout, + &git_dir, + refs, + catalog, + SCAN_LIMITS, + &cancel, + ) + .await; + } + let view = captured_catalog(layout, catalog, refs).await; + let lookup = view + .pointer_catalog_lookup(layout, 16 * 1024 * 1024) + .unwrap(); + verify_installed_dependencies_from( + layout, + &git_dir, + refs, + DependencyCatalog::Captured { + view: &view, + lookup: &lookup, + }, + SCAN_LIMITS, + &cancel, + ) + .await +} + fn write_pointer(workspace: &Path, pointer: &Pointer) -> String { let file = workspace.join("pointer"); std::fs::write(&file, pointer.serialize()).unwrap(); @@ -146,17 +260,23 @@ fn write_pointer(workspace: &Path, pointer: &Pointer) -> String { #[tokio::test] async fn installed_dependency_proof_accepts_reordered_repeated_file_chunks() { let (layout, catalog, workspace, refs) = fixture("valid").await; - let proof = verify_installed_dependencies( - &layout, - &workspace.path().join("repository.git"), - &refs, - &catalog, - SCAN_LIMITS, - &CancellationToken::new(), - ) - .await - .unwrap(); - assert_eq!(proof.reachable_crab_pointers, 1); + for captured in [false, true] { + let proof = verify_fixture(&layout, workspace.path(), &refs, &catalog, captured) + .await + .unwrap(); + assert_eq!( + proof, + CapsuleDependencyProof { + catalog_files: 1, + catalog_shards: 1, + catalog_xorbs: 1, + catalog_objects_read: 2, + reachable_crab_pointers: 1, + reachable_lfs_objects: 0, + reachable_lfs_bytes: 0 + } + ); + } } #[tokio::test] @@ -165,19 +285,13 @@ async fn installed_dependency_proof_rejects_a_valid_catalog_with_wrong_file_hash crate::verify_capsule_pointer_catalog_objects(&layout, &catalog) .await .unwrap(); - let result = verify_installed_dependencies( - &layout, - &workspace.path().join("repository.git"), - &refs, - &catalog, - SCAN_LIMITS, - &CancellationToken::new(), - ) - .await; - assert!( - matches!(result, Err(ReadError::HashMismatch { .. })), - "{result:?}" - ); + for captured in [false, true] { + let result = verify_fixture(&layout, workspace.path(), &refs, &catalog, captured).await; + assert!( + matches!(result, Err(ReadError::HashMismatch { .. })), + "{captured}: {result:?}" + ); + } } #[tokio::test] @@ -187,22 +301,16 @@ async fn installed_dependency_proof_rejects_missing_short_or_reordered_recipes() crate::verify_capsule_pointer_catalog_objects(&layout, &catalog) .await .unwrap(); - let result = verify_installed_dependencies( - &layout, - &workspace.path().join("repository.git"), - &refs, - &catalog, - SCAN_LIMITS, - &CancellationToken::new(), - ) - .await; - assert!( - matches!( - result, - Err(ReadError::HashMismatch { .. } | ReadError::CorruptObject { .. }) - ), - "{case}: {result:?}", - ); + for captured in [false, true] { + let result = verify_fixture(&layout, workspace.path(), &refs, &catalog, captured).await; + assert!( + matches!( + result, + Err(ReadError::HashMismatch { .. } | ReadError::CorruptObject { .. }) + ), + "{case}/{captured}: {result:?}" + ); + } } } @@ -472,3 +580,121 @@ async fn git_view_catalog_materialization_uses_the_captured_checkpoint_after_roo .unwrap(); assert_eq!(view.pointer_catalog().await.unwrap(), catalog); } + +#[tokio::test] +async fn dependency_proof_rejects_missing_bindings_on_both_sources() { + let (layout, catalog, workspace, refs) = fixture("missing_binding").await; + for captured in [false, true] { + let result = verify_fixture(&layout, workspace.path(), &refs, &catalog, captured).await; + assert!( + matches!(result, Err(ReadError::CorruptObject { reason, .. }) if reason.contains("absent from the catalog")) + ); + } +} + +#[tokio::test] +async fn captured_dependency_proof_keeps_pinned_bindings_after_root_replacement() { + use crab_metadata::capsule_protocol::RepositoryRoot; + let (layout, catalog, workspace, refs) = fixture("valid").await; + let view = captured_catalog(&layout, &catalog, &refs).await; + let lookup = view + .pointer_catalog_lookup(&layout, 16 * 1024 * 1024) + .unwrap(); + let replacement = + RootRecord::encode(RepositoryRoot::initial(&"2".repeat(64), "refs/heads/main").unwrap()) + .unwrap(); + layout + .store() + .put_overwrite(&layout.capsule_root_path(), replacement.bytes().clone()) + .await + .unwrap(); + let proof = verify_installed_dependencies_from( + &layout, + &workspace.path().join("repository.git"), + &refs, + DependencyCatalog::Captured { + view: &view, + lookup: &lookup, + }, + SCAN_LIMITS, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(proof.reachable_crab_pointers, 1); +} + +#[tokio::test] +async fn captured_dependency_proof_rejects_unrelated_incomplete_catalog_dependencies() { + let (layout, mut catalog, workspace, refs) = fixture("valid").await; + catalog + .insert_file("a".repeat(64), FileCatalogEntry::new(7, "b".repeat(64))) + .unwrap(); + let result = verify_fixture(&layout, workspace.path(), &refs, &catalog, true).await; + assert!(matches!( + result, + Err(ReadError::Metadata( + crab_metadata::error::MetadataError::CorruptObject { .. } + )) + )); +} + +#[tokio::test] +async fn captured_binding_batches_do_not_repeat_file_reconstruction() { + let (layout, catalog, workspace, mut refs) = fixture("valid").await; + let reads = Arc::new(ReadCount::default()); + let layout = StoreLayout::new( + layout.store().clone().with_storage_observer(reads.clone()), + layout.repo_prefix().to_owned(), + ); + let git_dir = workspace.path().join("repository.git"); + verify_installed_dependencies( + &layout, + &git_dir, + &refs, + &catalog, + SCAN_LIMITS, + &CancellationToken::new(), + ) + .await + .unwrap(); + let baseline = reads.0.swap(0, Ordering::Relaxed); + let mut pointer = + Pointer::parse(&std::fs::read(workspace.path().join("pointer")).unwrap()).unwrap(); + for hint in 1..=65u8 { + pointer.shard_hint = Some([hint; 32]); + refs.insert( + format!("refs/tags/hint-{hint}"), + write_pointer(workspace.path(), &pointer), + ); + } + let view = captured_catalog(&layout, &catalog, &refs).await; + let lookup = view + .pointer_catalog_lookup(&layout, 16 * 1024 * 1024) + .unwrap(); + reads.0.store(0, Ordering::Relaxed); + let proof = verify_installed_dependencies_from( + &layout, + &git_dir, + &refs, + DependencyCatalog::Captured { + view: &view, + lookup: &lookup, + }, + crab_git::walk::PointerScanLimits { + objects: 256, + lookups: 512, + allocation_bytes: 1024 * 1024, + }, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + ( + proof.reachable_crab_pointers, + reads.0.load(Ordering::Relaxed) + ), + (66, baseline) + ); +} diff --git a/crates/crab-read/src/selection.rs b/crates/crab-read/src/selection.rs index fb3f08169..c8ffae8ad 100644 --- a/crates/crab-read/src/selection.rs +++ b/crates/crab-read/src/selection.rs @@ -2,6 +2,9 @@ use std::borrow::Cow; use std::future::Future; use std::io::Write; +use crab_metadata::capsule_protocol::{ + PointerCatalogCounts, PointerCatalogInspection, PointerCatalogObject, +}; use crab_metadata::manifest_store; use crab_metadata::manifests::Manifest; use crab_metadata::ref_journal::list_active_transactions; @@ -483,27 +486,8 @@ pub async fn check_capsule_read_replica_readiness( let mut catalog = replica.pointer_catalog_inspection().await?; // Metadata proof is complete before probing bodies. Keep the spill owner // through every exit, including an early sample limit or provider error. - let result: Result> = async { - while !readiness_probe_budget_exhausted(&stats, options) { - let Some(object) = catalog.next_object().await? else { - break; - }; - let gap = match object { - crab_metadata::capsule_protocol::PointerCatalogObject::Shard { hash, entry } => { - verify_shard_object(replica_router, &hash, Cow::Owned(entry), &mut stats) - .await? - } - crab_metadata::capsule_protocol::PointerCatalogObject::Xorb { hash, entry } => { - verify_xorb_object(replica_router, &hash, Cow::Owned(entry), &mut stats).await? - } - }; - if gap.is_some() { - return Ok(gap); - } - } - Ok(None) - } - .await; + let result = + inspection_pointer_object_gap(replica_router, &mut catalog, &mut stats, options).await; if let Some(reason) = catalog.finish(result).await? { let mut readiness = ReadReplicaReadiness::not_ready( primary_generation, @@ -540,6 +524,56 @@ pub async fn verify_capsule_pointer_catalog_objects( Ok(stats) } +pub(crate) async fn verify_capsule_pointer_inspection( + router: &StoreLayout, + mut catalog: PointerCatalogInspection, + cancel: &tokio_util::sync::CancellationToken, +) -> Result<(PointerCatalogCounts, ReadinessProbeStats)> { + let counts = catalog.counts(); + let mut stats = ReadinessProbeStats::default(); + let result = tokio::select! { + biased; + () = cancel.cancelled() => Err(ReadError::Cancelled), + result = async { + if let Some(reason) = inspection_pointer_object_gap( + router, &mut catalog, &mut stats, ReadinessCheckOptions::deep(), + ).await? { + return Err(ReadError::CorruptObject { + path: "capsule-protocol pointer catalog".to_owned(), + reason, + }); + } + Ok((counts, stats)) + } => result, + }; + catalog.finish(result).await +} + +async fn inspection_pointer_object_gap( + router: &StoreLayout, + catalog: &mut PointerCatalogInspection, + stats: &mut ReadinessProbeStats, + options: ReadinessCheckOptions, +) -> Result> { + while !readiness_probe_budget_exhausted(stats, options) { + let Some(object) = catalog.next_object().await? else { + break; + }; + let gap = match object { + PointerCatalogObject::Shard { hash, entry } => { + verify_shard_object(router, &hash, Cow::Owned(entry), stats).await? + } + PointerCatalogObject::Xorb { hash, entry } => { + verify_xorb_object(router, &hash, Cow::Owned(entry), stats).await? + } + }; + if gap.is_some() { + return Ok(gap); + } + } + Ok(None) +} + async fn capsule_pointer_object_gap( router: &StoreLayout, catalog: &crab_metadata::capsule_protocol::PointerCatalog, @@ -1452,6 +1486,54 @@ mod tests { assert!(complete.reason.unwrap().contains("xorb missing")); } + #[tokio::test] + async fn catalog_body_verification_cancellation_interrupts_pending_origin_read() { + use object_store::throttle::{ThrottleConfig, ThrottledStore}; + use std::sync::Arc; + use std::time::Duration; + let (_store, router) = memory_store_with_layout("org/repo"); + let mut catalog = PointerCatalog::new(); + catalog + .insert_file( + "a".repeat(64), + crab_metadata::capsule_protocol::FileCatalogEntry::new(3, "b".repeat(64)), + ) + .unwrap(); + catalog + .insert_shard( + "b".repeat(64), + crab_metadata::capsule_protocol::ShardCatalogEntry::new(7, Vec::new()), + ) + .unwrap(); + let view = capsule_catalog_view(&router, &catalog).await; + let inspection = view.pointer_catalog_inspection().await.unwrap(); + let slow = StoreLayout::new( + Store::new(Arc::new(ThrottledStore::new( + Arc::clone(router.store().inner()), + ThrottleConfig { + wait_get_per_call: Duration::from_secs(10), + ..Default::default() + }, + ))), + router.repo_prefix().to_owned(), + ); + let cancel = tokio_util::sync::CancellationToken::new(); + let verification = verify_capsule_pointer_inspection(&slow, inspection, &cancel); + tokio::pin!(verification); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut verification) + .await + .is_err() + ); + cancel.cancel(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(1), &mut verification) + .await + .unwrap(), + Err(ReadError::Cancelled) + )); + } + #[tokio::test] async fn sampled_capsule_readiness_requires_complete_metadata_before_any_payload_probe() { let (_store, router) = memory_store_with_layout("org/repo"); From 341cf16f6b99f1f58a384309705a9c326d8a2e86 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 05:46:16 -0700 Subject: [PATCH 124/127] perf(fsck): stream shard-grouped catalog checks --- .../docs/architecture/indexed-remote-reads.md | 14 +- crab/src/cmd/fsck_store.rs | 360 ++++------------- .../cmd/fsck_store/capsule_history_tests.rs | 14 +- crab/src/cmd/fsck_store/catalog.rs | 361 ++++++++++++++++++ crab/src/cmd/fsck_store/catalog/tests.rs | 227 +++++++++++ crates/crab-metadata/AGENTS.md | 2 + crates/crab-metadata/README.md | 7 + .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../src/capsule_protocol/pointer.rs | 3 +- .../capsule_protocol/pointer/inspection.rs | 129 ++++--- .../pointer/inspection/grouped.rs | 157 ++++++++ .../pointer/inspection/tests.rs | 183 +++++++++ .../src/capsule_protocol/pointer/relations.rs | 1 + .../src/capsule_protocol/pointer/spill.rs | 37 +- crates/crab-read/AGENTS.md | 3 + crates/crab-read/README.md | 12 +- crates/crab-read/src/capsule_protocol.rs | 29 +- .../src/capsule_protocol/dependency.rs | 73 +++- .../src/capsule_protocol/dependency_tests.rs | 22 ++ crates/crab-xet/README.md | 4 + crates/crab-xet/src/shard.rs | 42 +- 21 files changed, 1304 insertions(+), 380 deletions(-) create mode 100644 crab/src/cmd/fsck_store/catalog.rs create mode 100644 crab/src/cmd/fsck_store/catalog/tests.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 5a696eff9..e50bcada2 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2550,8 +2550,18 @@ Deep reachable-dependency verification now streams that same proof and physical body checks before batching reachable file bindings from the captured roots. Metadata diagnosis and rebuild summaries consume verified cardinalities without a separate materialization. Recovery's assembled replacement catalogs share the -same Git scan and whole-file/LFS verification owner. Current fsck payload -verification, restore accounting and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to +same Git scan and whole-file/LFS verification owner. Current fsck also streams +its complete metadata proof. A private external reduction groups final file +bindings by shard, so one shard body and its dependency metadata serve batches +of 64 bindings. Xorb decoded-chunk checks, exact shard/xorb metadata comparisons, +file headers, sizes and reconstruction term validation remain required. The +subsequent canonical Git/whole-file/LFS phase does not repeat the weaker physical +descriptor pass. Constructor admission completes and closes its own inspection; +a later full-data phase opens a fresh inspection of the same pinned sources. +This can scan metadata twice, but avoids retaining operation spill across checker +phases and prevents an unrelated phase from reporting success before full proof. +Ordinary GC and replica readiness do not perform shard grouping. Restore +accounting and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to protect those callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index f8aa72dd0..f64eb676b 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -3,8 +3,7 @@ //! Wraps the real `Store`, ref store, and manifest state to perform //! actual storage queries and repairs for `crab fsck`. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::io::Cursor; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -30,7 +29,7 @@ use crab_storage::repo_pack_path; use crab_types::pointer::Pointer; use crab_xet::hash::MerkleHash; use crab_xet::shard::ShardReader; -use crab_xet::xorb::format::MAX_XORB_SIZE; +#[cfg(test)] use crab_xet::xorb::parser::XorbParser; const MAX_FSCK_SHARD_BYTES: u64 = 512 * 1024 * 1024; @@ -93,7 +92,7 @@ pub struct StoreChecker { struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, - catalog: crab_metadata::capsule_protocol::PointerCatalog, + lookup: crab_metadata::capsule_protocol::PointerCatalogLookup, source_lifetime: Arc, browse_lifetime: Arc>, @@ -136,7 +135,14 @@ impl StoreChecker { }, ) .await?; - let catalog = view.pointer_catalog().await?; + let lookup = view + .pointer_catalog_lookup(&layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; + // Validate the complete capture before any phase can report success. + // Keep spill ownership within the operation; later phases retain roots. + view.pointer_catalog_inspection() + .await? + .finish(Ok::<_, crab_metadata::error::MetadataError>(())) + .await?; let source_lifetime = verify_capsule_history(&layout, view.root().root()).await?; Ok(Self { store, @@ -145,7 +151,7 @@ impl StoreChecker { multipart_journal: None, capsule: Some(CapsuleFsckState { view, - catalog, + lookup, source_lifetime: Arc::new(source_lifetime), browse_lifetime: Arc::new(tokio::sync::OnceCell::new()), git_workspace: tokio::sync::OnceCell::new(), @@ -176,75 +182,7 @@ impl StoreChecker { ) }) .await?; - let mut issues = Vec::new(); - - for (hash, entry) in state.catalog.xorbs() { - if entry.encoded_size() > MAX_XORB_SIZE as u64 { - issues.push(FsckIssue::corrupt_xorb( - hash, - format!( - "catalog size {} exceeds the xorb limit {MAX_XORB_SIZE}", - entry.encoded_size() - ), - )); - continue; - } - let xorb_hash = MerkleHash::from_hex(hash).map_err(|error| { - CrabError::Internal(format!( - "validated xorb catalog hash became invalid: {error}" - )) - })?; - let path = self.router.xorb_path(&xorb_hash); - let body = match self - .store - .get_with_etag_bounded(&path, entry.encoded_size()) - .await - { - Ok((body, _)) => body, - Err(CrabError::NotFound { .. }) => { - issues.push(FsckIssue::missing_xorb(hash)); - continue; - } - Err(error) => return Err(error), - }; - if let Err(detail) = validate_catalog_xorb(hash, entry, body) { - issues.push(FsckIssue::corrupt_xorb(hash, detail)); - } - } - - for (hash, entry) in state.catalog.shards() { - if entry.encoded_size() > MAX_FSCK_SHARD_BYTES { - issues.push(FsckIssue::corrupt_shard( - hash, - format!( - "catalog size {} exceeds the shard limit {MAX_FSCK_SHARD_BYTES}", - entry.encoded_size() - ), - )); - continue; - } - let shard_hash = MerkleHash::from_hex(hash).map_err(|error| { - CrabError::Internal(format!( - "validated shard catalog hash became invalid: {error}" - )) - })?; - let path = self.router.shard_path(&shard_hash); - let body = match self - .store - .get_with_etag_bounded(&path, entry.encoded_size()) - .await - { - Ok((body, _)) => body, - Err(CrabError::NotFound { .. }) => { - issues.push(FsckIssue::missing_shard(hash)); - continue; - } - Err(error) => return Err(error), - }; - if let Err(detail) = validate_catalog_shard(hash, entry, &state.catalog, body) { - issues.push(FsckIssue::corrupt_shard(hash, detail)); - } - } + let issues = catalog::check_objects(&layout, &state.view, &state.lookup).await?; if issues.is_empty() { // Keep the authenticated Git database across phases. Catalog bodies @@ -255,11 +193,12 @@ impl StoreChecker { check_capsule_git_connectivity(&self.store, &self.router, state) }) .await?; - crab_read::capsule_protocol::verify_installed_dependencies( + // Object/catalog proof above is stronger than the shared descriptor + // pass. Reuse only the canonical Git and whole-file/LFS byte phase. + crab_read::capsule_protocol::verify_captured_installed_pointer_bytes( &layout, workspace.path(), - state.view.refs(), - &state.catalog, + &state.view, CAPSULE_GIT_SCAN_LIMITS, &CancellationToken::new(), ) @@ -336,60 +275,74 @@ impl StoreChecker { let mut issues = Vec::new(); let mut verified = 0u64; let mut recipes = blake3::Hasher::new_derive_key("crab verified pointer recipes v1"); - for (file_hash, expected_size) in expected { - crate::core::error::check_cancelled(cancel)?; - let Some(entry) = state.catalog.files().get(&file_hash.hex()) else { - issues.push(PointerDataIssue { - file_hash: file_hash.hex(), - expected_size, - kind: PointerDataIssueKind::Missing, - detail: format!( - "pointer {} has no recipe in the captured capsule catalog", - file_hash.hex() - ), - }); - continue; - }; - let shard_hash = MerkleHash::from_hex(entry.shard_hash()).map_err(|error| { - CrabError::CorruptObject { - path: entry.shard_hash().to_owned(), - reason: format!("capsule catalog shard identity is invalid: {error}"), - } - })?; - let pointer = Pointer { - file_hash: file_hash.into(), - size: expected_size, - shard_hint: None, + let mut expected = expected.into_iter(); + loop { + let batch = expected.by_ref().take(64).collect::>(); + if batch.is_empty() { + break; + } + let hashes = batch.iter().map(|(hash, _)| hash.hex()).collect::>(); + let selected = tokio::select! { + biased; + () = cancel.cancelled() => return Err(CrabError::Cancelled), + result = state.lookup.files(origin_layout.store(), &hashes) => result?, }; - match crab_read::verify_catalog_file_recipe(&origin_layout, &pointer, entry, cancel) - .await - { - Ok(file_info) => { - verified += 1; - recipes.update(&pointer.file_hash); - recipes.update(&pointer.size.to_le_bytes()); - recipes.update(shard_hash.as_bytes()); - let mut recipe = blake3::Hasher::new_derive_key("crab shard file recipe v1"); - file_info.serialize(&mut recipe)?; - recipes.update(recipe.finalize().as_bytes()); - } - Err(crab_read::ReadError::Cancelled) => return Err(CrabError::Cancelled), - Err(error) => { - let kind = match &error { - crab_read::ReadError::Xet(_) => PointerDataIssueKind::Corrupt, - _ => PointerDataIssueKind::Unverifiable, - }; - let error = CrabError::from(error); + for (file_hash, expected_size) in batch { + crate::core::error::check_cancelled(cancel)?; + let Some(entry) = selected.files().get(&file_hash.hex()) else { issues.push(PointerDataIssue { file_hash: file_hash.hex(), expected_size, - kind: if kind == PointerDataIssueKind::Corrupt { - kind - } else { - PointerDataIssueKind::from_error(&error) - }, - detail: error.to_string(), + kind: PointerDataIssueKind::Missing, + detail: format!( + "pointer {} has no recipe in the captured capsule catalog", + file_hash.hex() + ), }); + continue; + }; + let shard_hash = MerkleHash::from_hex(entry.shard_hash()).map_err(|error| { + CrabError::CorruptObject { + path: entry.shard_hash().to_owned(), + reason: format!("capsule catalog shard identity is invalid: {error}"), + } + })?; + let pointer = Pointer { + file_hash: file_hash.into(), + size: expected_size, + shard_hint: None, + }; + match crab_read::verify_catalog_file_recipe(&origin_layout, &pointer, entry, cancel) + .await + { + Ok(file_info) => { + verified += 1; + recipes.update(&pointer.file_hash); + recipes.update(&pointer.size.to_le_bytes()); + recipes.update(shard_hash.as_bytes()); + let mut recipe = + blake3::Hasher::new_derive_key("crab shard file recipe v1"); + file_info.serialize(&mut recipe)?; + recipes.update(recipe.finalize().as_bytes()); + } + Err(crab_read::ReadError::Cancelled) => return Err(CrabError::Cancelled), + Err(error) => { + let kind = match &error { + crab_read::ReadError::Xet(_) => PointerDataIssueKind::Corrupt, + _ => PointerDataIssueKind::Unverifiable, + }; + let error = CrabError::from(error); + issues.push(PointerDataIssue { + file_hash: file_hash.hex(), + expected_size, + kind: if kind == PointerDataIssueKind::Corrupt { + kind + } else { + PointerDataIssueKind::from_error(&error) + }, + detail: error.to_string(), + }); + } } } } @@ -1086,157 +1039,6 @@ fn insert_history_pointer( Ok(()) } -fn validate_catalog_xorb( - hash: &str, - entry: &crab_metadata::capsule_protocol::XorbCatalogEntry, - body: bytes::Bytes, -) -> std::result::Result<(), String> { - if body.len() as u64 != entry.encoded_size() { - return Err(format!( - "stored size is {}, catalog declares {}", - body.len(), - entry.encoded_size() - )); - } - let body_digest = blake3::hash(&body).to_hex().to_string(); - if body_digest != entry.body_digest() { - return Err(format!( - "stored body digest is {body_digest}, catalog declares {}", - entry.body_digest() - )); - } - let parser = XorbParser::parse(body).map_err(|error| error.to_string())?; - if parser.hash().hex() != hash { - return Err(format!( - "stored logical identity is {}, catalog key is {hash}", - parser.hash().hex() - )); - } - parser - .verify_payload_digest() - .and_then(|()| parser.verify_all_chunks()) - .map_err(|error| error.to_string())?; - if parser.num_chunks() as usize != entry.chunks().len() { - return Err(format!( - "stored chunk count is {}, catalog declares {}", - parser.num_chunks(), - entry.chunks().len() - )); - } - for (index, expected) in entry.chunks().iter().enumerate() { - let index = u32::try_from(index) - .map_err(|_| "catalog xorb chunk index cannot be represented".to_owned())?; - let actual = parser - .chunk_meta(index) - .map_err(|error| error.to_string())?; - if actual.hash.hex() != expected.hash() - || actual.uncompressed_len != expected.uncompressed_size() - { - return Err(format!( - "stored chunk {index} does not match its catalog hash and size" - )); - } - } - Ok(()) -} - -fn validate_catalog_shard( - hash: &str, - entry: &crab_metadata::capsule_protocol::ShardCatalogEntry, - catalog: &crab_metadata::capsule_protocol::PointerCatalog, - body: bytes::Bytes, -) -> std::result::Result<(), String> { - if body.len() as u64 != entry.encoded_size() { - return Err(format!( - "stored size is {}, catalog declares {}", - body.len(), - entry.encoded_size() - )); - } - let shard_hash = MerkleHash::from_hex(hash).map_err(|error| error.to_string())?; - let actual_hash = crab_xet::hash::compute_data_hash(&body); - if actual_hash != shard_hash { - return Err(format!( - "stored content identity is {}, catalog key is {hash}", - actual_hash.hex() - )); - } - - let mut actual_xorbs = BTreeSet::new(); - let mut cursor = Cursor::new(crab_xet::shard_parse::strip_bloom_trailer(&body)); - crab_xet::shard_parse::visit_xorb_chunks_from_reader(&mut cursor, |xorb_hash, _, _, _| { - actual_xorbs.insert(xorb_hash.hex()); - Ok(()) - }) - .map_err(|error| error.to_string())?; - let expected_xorbs = entry.xorb_hashes().iter().cloned().collect::>(); - if actual_xorbs != expected_xorbs { - return Err("stored xorb closure differs from the authenticated catalog".to_owned()); - } - - let reader = ShardReader::from_bytes(body, shard_hash); - let mut dependencies = Vec::with_capacity(entry.xorb_hashes().len()); - for xorb_hash in entry.xorb_hashes() { - let parsed_hash = MerkleHash::from_hex(xorb_hash).map_err(|error| error.to_string())?; - let dependency = reader - .get_xorb_info(&parsed_hash) - .map_err(|error| error.to_string())? - .ok_or_else(|| format!("stored shard lacks catalogued xorb {xorb_hash}"))?; - let expected = catalog - .xorbs() - .get(xorb_hash) - .ok_or_else(|| format!("catalog lacks shard xorb {xorb_hash}"))?; - if dependency.chunks.len() != expected.chunks().len() - || dependency - .chunks - .iter() - .zip(expected.chunks()) - .any(|(actual, expected)| { - actual.chunk_hash.hex() != expected.hash() - || actual.unpacked_segment_bytes != expected.uncompressed_size() - }) - { - return Err(format!( - "stored shard metadata for xorb {xorb_hash} differs from the catalog" - )); - } - dependencies.push(dependency); - } - - for (file_hash, expected) in catalog - .files() - .iter() - .filter(|(_, file)| file.shard_hash() == hash) - { - let parsed_hash = MerkleHash::from_hex(file_hash).map_err(|error| error.to_string())?; - let file = reader - .get_file_info(&parsed_hash) - .map_err(|error| error.to_string())? - .ok_or_else(|| format!("stored shard lacks catalogued file {file_hash}"))?; - if file.metadata.file_hash != parsed_hash - || file.metadata.num_entries as usize != file.segments.len() - { - return Err(format!( - "stored recipe header for file {file_hash} is inconsistent" - )); - } - let size = file.segments.iter().try_fold(0_u64, |total, segment| { - total - .checked_add(u64::from(segment.unpacked_segment_bytes)) - .ok_or_else(|| format!("stored recipe size for file {file_hash} overflowed")) - })?; - if size != expected.size() { - return Err(format!( - "stored recipe for file {file_hash} covers {size} bytes, catalog declares {}", - expected.size() - )); - } - crab_xet::shard::validate_file_bundle(&file, &dependencies) - .map_err(|error| error.to_string())?; - } - Ok(()) -} - async fn install_capsule_git_packs( view: crab_read::capsule_protocol::CapsuleRepositoryView, layout: crab_storage::StoreLayout, @@ -1800,6 +1602,8 @@ fn unix_now() -> i64 { .unwrap_or(0) } +mod catalog; + #[cfg(test)] mod verification_tests; diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index 311ecefb4..40d91e969 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -316,17 +316,11 @@ async fn capsule_recipe_probe_does_not_require_browse_maintenance() { let checker = StoreChecker::for_capsule_repository(store, prefix, root) .await .unwrap(); - let (hash, file) = checker - .capsule - .as_ref() - .unwrap() - .catalog - .files() - .first_key_value() - .unwrap(); + // The fixture's original user bytes define the expected pointer, independent + // of how the checker stores or queries its catalog. let pointer = Pointer { - file_hash: MerkleHash::from_hex(hash).unwrap().into(), - size: file.size(), + file_hash: blake3::hash(&[0x61; 1024]).into(), + size: 1024, shard_hint: None, }; let proof = checker diff --git a/crab/src/cmd/fsck_store/catalog.rs b/crab/src/cmd/fsck_store/catalog.rs new file mode 100644 index 000000000..ce53092f1 --- /dev/null +++ b/crab/src/cmd/fsck_store/catalog.rs @@ -0,0 +1,361 @@ +//! Complete object and per-file recipe inspection from a locally grouped capture. +use super::{CrabError, FsckIssue, MAX_FSCK_SHARD_BYTES, Result}; +use crab_metadata::capsule_protocol::{ + FileCatalogEntry, PointerCatalogEntry, PointerCatalogLookup, PointerCatalogObject, +}; +use crab_read::capsule_protocol::CapsuleRepositoryView; +use crab_storage::{StorageError, Store, StoreLayout}; +use crab_xet::{ + hash::MerkleHash, + shard::ShardReader, + xorb::{format::MAX_XORB_SIZE, parser::XorbParser}, +}; +use std::{collections::BTreeSet, io::Cursor, sync::Arc}; + +pub(super) async fn check_objects( + layout: &StoreLayout, + view: &CapsuleRepositoryView, + lookup: &PointerCatalogLookup, +) -> Result> { + let mut catalog = view + .pointer_catalog_inspection() + .await? + .group_by_shard() + .await?; + let result = check_entries(layout, lookup, &mut catalog).await; + catalog.finish(result).await +} + +async fn check_entries( + layout: &StoreLayout, + lookup: &PointerCatalogLookup, + catalog: &mut crab_metadata::capsule_protocol::PointerCatalogShardInspection, +) -> Result> { + let expected = catalog.counts(); + let mut seen = crab_metadata::capsule_protocol::PointerCatalogCounts::default(); + let mut issues = Vec::new(); + let mut current: Option = None; + let mut files = Vec::new(); + while let Some(entry) = catalog.next_entry().await? { + match entry { + PointerCatalogEntry::File { hash, entry } => { + seen.files += 1; + let state = current.as_ref().ok_or_else(|| { + CrabError::Protocol("grouped file precedes its shard".to_owned()) + })?; + if state.hash != entry.shard_hash() { + return Err(CrabError::Protocol( + "grouped file identifies a different shard".to_owned(), + )); + } + if state.validated.is_some() { + files.push((hash, entry)); + if files.len() == 64 { + flush_files(&mut current, &mut files, &mut issues).await?; + } + } + } + PointerCatalogEntry::Object(object) => { + flush_files(&mut current, &mut files, &mut issues).await?; + match object { + PointerCatalogObject::Xorb { hash, entry } => { + seen.xorbs += 1; + if entry.encoded_size() > MAX_XORB_SIZE as u64 { + issues.push(FsckIssue::corrupt_xorb( + &hash, + format!( + "catalog size {} exceeds the xorb limit {MAX_XORB_SIZE}", + entry.encoded_size() + ), + )); + continue; + } + let parsed = MerkleHash::from_hex(&hash) + .map_err(|source| CrabError::Io(std::io::Error::other(source)))?; + let body = match layout + .store() + .get_with_etag_bounded(&layout.xorb_path(&parsed), entry.encoded_size()) + .await + { + Ok((body, _)) => body, + Err(StorageError::NotFound { .. }) => { + issues.push(FsckIssue::missing_xorb(&hash)); + continue; + } + Err(source) => return Err(source.into()), + }; + let checked_hash = hash.clone(); + let result = tokio::task::spawn_blocking(move || { + validate_catalog_xorb(&checked_hash, &entry, body) + }) + .await + .map_err(|source| CrabError::Io(std::io::Error::other(source)))?; + if let Err(detail) = result { + issues.push(FsckIssue::corrupt_xorb(&hash, detail)); + } + } + PointerCatalogObject::Shard { hash, entry } => { + seen.shards += 1; + current = Some(ShardState { + hash: hash.clone(), + validated: None, + }); + if entry.encoded_size() > MAX_FSCK_SHARD_BYTES { + issues.push(FsckIssue::corrupt_shard(&hash, format!("catalog size {} exceeds the shard limit {MAX_FSCK_SHARD_BYTES}", entry.encoded_size()))); + continue; + } + let parsed = MerkleHash::from_hex(&hash) + .map_err(|source| CrabError::Io(std::io::Error::other(source)))?; + let body = match layout + .store() + .get_with_etag_bounded( + &layout.shard_path(&parsed), + entry.encoded_size(), + ) + .await + { + Ok((body, _)) => body, + Err(StorageError::NotFound { .. }) => { + issues.push(FsckIssue::missing_shard(&hash)); + continue; + } + Err(source) => return Err(source.into()), + }; + // One selected closure per shard, never per file binding. + // These are the same captured roots used by the full metadata proof. + let dependencies = lookup + .dependencies(layout.store(), &[], std::slice::from_ref(&hash), &[]) + .await?; + if dependencies.shards().get(&hash) != Some(&entry) { + return Err(CrabError::Protocol( + "captured shard descriptor differs across inspection paths" + .to_owned(), + )); + } + let checked_hash = hash.clone(); + let result = tokio::task::spawn_blocking(move || { + prepare_shard(&checked_hash, &entry, &dependencies, body) + }) + .await + .map_err(|source| CrabError::Io(std::io::Error::other(source)))?; + match result { + Ok(shard) => { + current = Some(ShardState { + hash, + validated: Some(Arc::new(shard)), + }); + } + Err(detail) => issues.push(FsckIssue::corrupt_shard(&hash, detail)), + } + } + } + } + } + } + flush_files(&mut current, &mut files, &mut issues).await?; + if seen != expected { + return Err(CrabError::Protocol( + "grouped catalog cardinalities differ from the complete proof".to_owned(), + )); + } + Ok(issues) +} + +struct ValidatedShard { + reader: ShardReader, + dependencies: Vec, +} + +impl ValidatedShard { + fn files(&self, files: &[(String, FileCatalogEntry)]) -> std::result::Result<(), String> { + let validator = crab_xet::shard::FileBundleValidator::new(&self.dependencies); + for (file_hash, expected) in files { + let parsed_hash = MerkleHash::from_hex(file_hash).map_err(|error| error.to_string())?; + let file = self + .reader + .get_file_info(&parsed_hash) + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("stored shard lacks catalogued file {file_hash}"))?; + if file.metadata.file_hash != parsed_hash + || file.metadata.num_entries as usize != file.segments.len() + { + return Err(format!( + "stored recipe header for file {file_hash} is inconsistent" + )); + } + let size = file.segments.iter().try_fold(0_u64, |total, segment| { + total + .checked_add(u64::from(segment.unpacked_segment_bytes)) + .ok_or_else(|| format!("stored recipe size for file {file_hash} overflowed")) + })?; + if size != expected.size() { + return Err(format!( + "stored recipe for file {file_hash} covers {size} bytes, catalog declares {}", + expected.size() + )); + } + validator + .validate(&file) + .map_err(|error| error.to_string())?; + } + Ok(()) + } +} + +struct ShardState { + hash: String, + // A missing/corrupt shard still owns its following bindings. Report it once, + // then consume those rows without claiming their recipes were verified. + validated: Option>, +} + +async fn flush_files( + current: &mut Option, + files: &mut Vec<(String, FileCatalogEntry)>, + issues: &mut Vec, +) -> Result<()> { + if files.is_empty() { + return Ok(()); + } + let state = current + .as_mut() + .ok_or_else(|| CrabError::Protocol("grouped bindings lack a shard".to_owned()))?; + let shard = state.validated.clone().ok_or_else(|| { + CrabError::Protocol("unverifiable shard retained pending bindings".to_owned()) + })?; + let files = std::mem::take(files); + let result = tokio::task::spawn_blocking(move || shard.files(&files)) + .await + .map_err(|source| CrabError::Io(std::io::Error::other(source)))?; + if let Err(detail) = result { + issues.push(FsckIssue::corrupt_shard(&state.hash, detail)); + state.validated = None; + } + Ok(()) +} + +fn validate_catalog_xorb( + hash: &str, + entry: &crab_metadata::capsule_protocol::XorbCatalogEntry, + body: bytes::Bytes, +) -> std::result::Result<(), String> { + if body.len() as u64 != entry.encoded_size() { + return Err(format!( + "stored size is {}, catalog declares {}", + body.len(), + entry.encoded_size() + )); + } + let body_digest = blake3::hash(&body).to_hex().to_string(); + if body_digest != entry.body_digest() { + return Err(format!( + "stored body digest is {body_digest}, catalog declares {}", + entry.body_digest() + )); + } + let parser = XorbParser::parse(body).map_err(|error| error.to_string())?; + if parser.hash().hex() != hash { + return Err(format!( + "stored logical identity is {}, catalog key is {hash}", + parser.hash().hex() + )); + } + parser + .verify_payload_digest() + .and_then(|()| parser.verify_all_chunks()) + .map_err(|error| error.to_string())?; + if parser.num_chunks() as usize != entry.chunks().len() { + return Err(format!( + "stored chunk count is {}, catalog declares {}", + parser.num_chunks(), + entry.chunks().len() + )); + } + for (index, expected) in entry.chunks().iter().enumerate() { + let index = u32::try_from(index) + .map_err(|_| "catalog xorb chunk index cannot be represented".to_owned())?; + let actual = parser + .chunk_meta(index) + .map_err(|error| error.to_string())?; + if actual.hash.hex() != expected.hash() + || actual.uncompressed_len != expected.uncompressed_size() + { + return Err(format!( + "stored chunk {index} does not match its catalog hash and size" + )); + } + } + Ok(()) +} + +fn prepare_shard( + hash: &str, + entry: &crab_metadata::capsule_protocol::ShardCatalogEntry, + catalog: &crab_metadata::capsule_protocol::PointerCatalog, + body: bytes::Bytes, +) -> std::result::Result { + if body.len() as u64 != entry.encoded_size() { + return Err(format!( + "stored size is {}, catalog declares {}", + body.len(), + entry.encoded_size() + )); + } + let shard_hash = MerkleHash::from_hex(hash).map_err(|error| error.to_string())?; + let actual_hash = crab_xet::hash::compute_data_hash(&body); + if actual_hash != shard_hash { + return Err(format!( + "stored content identity is {}, catalog key is {hash}", + actual_hash.hex() + )); + } + + let mut actual_xorbs = BTreeSet::new(); + let mut cursor = Cursor::new(crab_xet::shard_parse::strip_bloom_trailer(&body)); + crab_xet::shard_parse::visit_xorb_chunks_from_reader(&mut cursor, |xorb_hash, _, _, _| { + actual_xorbs.insert(xorb_hash.hex()); + Ok(()) + }) + .map_err(|error| error.to_string())?; + let expected_xorbs = entry.xorb_hashes().iter().cloned().collect::>(); + if actual_xorbs != expected_xorbs { + return Err("stored xorb closure differs from the authenticated catalog".to_owned()); + } + + let reader = ShardReader::from_bytes(body, shard_hash); + let mut dependencies = Vec::with_capacity(entry.xorb_hashes().len()); + for xorb_hash in entry.xorb_hashes() { + let parsed_hash = MerkleHash::from_hex(xorb_hash).map_err(|error| error.to_string())?; + let dependency = reader + .get_xorb_info(&parsed_hash) + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("stored shard lacks catalogued xorb {xorb_hash}"))?; + let expected = catalog + .xorbs() + .get(xorb_hash) + .ok_or_else(|| format!("catalog lacks shard xorb {xorb_hash}"))?; + if dependency.chunks.len() != expected.chunks().len() + || dependency + .chunks + .iter() + .zip(expected.chunks()) + .any(|(actual, expected)| { + actual.chunk_hash.hex() != expected.hash() + || actual.unpacked_segment_bytes != expected.uncompressed_size() + }) + { + return Err(format!( + "stored shard metadata for xorb {xorb_hash} differs from the catalog" + )); + } + dependencies.push(dependency); + } + + Ok(ValidatedShard { + reader, + dependencies, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crab/src/cmd/fsck_store/catalog/tests.rs b/crab/src/cmd/fsck_store/catalog/tests.rs new file mode 100644 index 000000000..ef9b4d307 --- /dev/null +++ b/crab/src/cmd/fsck_store/catalog/tests.rs @@ -0,0 +1,227 @@ +#![expect(clippy::unwrap_used, reason = "test assertions")] +use super::*; +use bytes::Bytes; +use crab_metadata::capsule_protocol::{ + Capsule, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, + FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, +}; +use crab_storage::{StorageObservation, StorageObserver, StorageOperation}; +use crab_xet::{ + shard::{ShardWriter, file_info_from_placements, xorb_info_from_placements}, + xorb::{ + builder::{RunId, XorbBuilder}, + format::Chunk, + }, +}; +use std::sync::Mutex; + +#[derive(Default)] +struct Observations(Mutex>); +impl StorageObserver for Observations { + fn started(&self, _: StorageOperation) {} + fn finished(&self, observation: StorageObservation) { + self.0.lock().unwrap().push(observation); + } +} + +async fn fixture( + wrong_size: bool, +) -> ( + super::super::StoreChecker, + StoreLayout, + Arc, + Option, +) { + let observations = Arc::new(Observations::default()); + let storage = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(observations.clone()); + let layout = StoreLayout::new(storage.clone(), "grouped-fsck".to_owned()); + let base = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let mut builder = XorbBuilder::new(); + let mut chunks = Vec::new(); + for index in 0..129u64 { + let bytes = index.to_le_bytes(); + let chunk = Chunk::new(Bytes::copy_from_slice(&bytes)); + builder.push(&chunk, RunId(0)).unwrap(); + chunks.push((MerkleHash::from(*blake3::hash(&bytes).as_bytes()), chunk)); + } + let xorb = builder.finalize().unwrap().pop().unwrap(); + let placements = xorb + .placements + .iter() + .cloned() + .map(|placement| (placement.chunk_hash, placement)) + .collect(); + let mut shard = ShardWriter::new(); + shard + .add_xorb(Arc::new( + xorb_info_from_placements(xorb.hash, &xorb.placements).unwrap(), + )) + .unwrap(); + for (file_hash, chunk) in &chunks { + shard + .add_file(file_info_from_placements(*file_hash, &[chunk.hash], &placements).unwrap()) + .unwrap(); + } + let (shard_body, shard_hash) = shard.finalize().unwrap(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hash.hex(), + XorbCatalogEntry::new( + xorb.bytes.len() as u64, + blake3::hash(&xorb.bytes).to_hex().to_string(), + chunks + .iter() + .map(|(_, chunk)| XorbChunkEntry::new(chunk.hash.hex(), 8)) + .collect(), + ), + ) + .unwrap(); + catalog + .insert_shard( + shard_hash.hex(), + ShardCatalogEntry::new(shard_body.len() as u64, vec![xorb.hash.hex()]), + ) + .unwrap(); + let mut hashes = chunks + .iter() + .map(|(hash, _)| hash.hex()) + .collect::>(); + hashes.sort(); + let wrong = wrong_size.then(|| hashes[100].clone()); + for hash in hashes { + let size = if wrong.as_ref() == Some(&hash) { 9 } else { 8 }; + catalog + .insert_file(hash, FileCatalogEntry::new(size, shard_hash.hex())) + .unwrap(); + } + storage + .put(&layout.xorb_path(&xorb.hash), xorb.bytes) + .await + .unwrap(); + storage + .put(&layout.shard_path(&shard_hash), shard_body.into()) + .await + .unwrap(); + let transaction = CapsuleTransaction::new( + base.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )], + ) + .unwrap(); + let root = crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) + .await + .unwrap(); + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + Some("2".repeat(40)), + None, + None, + )], + ) + .unwrap(); + let capsule = Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap(); + let root = crab_write::capsule_protocol::publish(&layout, root, &transaction, &capsule) + .await + .unwrap(); + let checker = super::super::StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(storage), + layout.repo_prefix().to_owned(), + root, + ) + .await + .unwrap(); + observations.0.lock().unwrap().clear(); + (checker, layout, observations, wrong) +} + +#[tokio::test] +async fn shard_group_reads_each_body_once_across_multiple_file_batches() { + let (checker, layout, observations, _) = fixture(false).await; + let state = checker.capsule.as_ref().unwrap(); + let issues = check_objects(&layout, &state.view, &state.lookup) + .await + .unwrap(); + let operations = observations.0.lock().unwrap(); + assert_eq!( + ( + issues.len(), + operations + .iter() + .filter(|op| op.operation == StorageOperation::Get) + .count(), + operations.iter().any(|op| op.bytes_written > 0) + ), + (0, 2, false) + ); +} + +#[tokio::test] +async fn shard_group_rejects_a_bad_recipe_after_the_first_file_batch() { + let (checker, layout, observations, wrong) = fixture(true).await; + let state = checker.capsule.as_ref().unwrap(); + let issues = check_objects(&layout, &state.view, &state.lookup) + .await + .unwrap(); + assert_eq!(issues.len(), 1); + assert!( + matches!(&issues[0].kind, crate::cmd::fsck::IssueKind::CorruptShard { detail, .. } if detail.contains(wrong.as_ref().unwrap()) && detail.contains("covers 8 bytes")) + ); + let operations = observations.0.lock().unwrap(); + assert_eq!( + operations + .iter() + .filter(|op| op.operation == StorageOperation::Get) + .count(), + 2 + ); +} + +#[tokio::test] +async fn shard_header_verification_retains_chunk_metadata_comparison() { + let (checker, layout, _, _) = fixture(false).await; + let state = checker.capsule.as_ref().unwrap(); + let catalog = state.view.pointer_catalog().await.unwrap(); + let (hash, entry) = catalog.shards().iter().next().unwrap(); + let body = layout + .store() + .get_with_etag_bounded( + &layout.shard_path(&MerkleHash::from_hex(hash).unwrap()), + entry.encoded_size(), + ) + .await + .unwrap() + .0; + let (xorb_hash, xorb) = catalog.xorbs().iter().next().unwrap(); + let mut chunks = xorb.chunks().to_vec(); + chunks[0] = XorbChunkEntry::new(chunks[0].hash(), chunks[0].uncompressed_size() + 1); + let mut inconsistent = PointerCatalog::new(); + inconsistent + .insert_xorb( + xorb_hash.clone(), + XorbCatalogEntry::new(xorb.encoded_size(), xorb.body_digest(), chunks), + ) + .unwrap(); + assert!( + matches!(prepare_shard(hash, entry, &inconsistent, body), Err(reason) if reason.contains("differs from the catalog")) + ); +} diff --git a/crates/crab-metadata/AGENTS.md b/crates/crab-metadata/AGENTS.md index fd2844570..01a5873ff 100644 --- a/crates/crab-metadata/AGENTS.md +++ b/crates/crab-metadata/AGENTS.md @@ -55,6 +55,8 @@ Trace one path: `crates/crab-write/src/journal.rs` → `commit_ref_transaction` - `PointerCatalogInspection` exposes external descriptors only after complete precedence/conflict/dependency proof. Await `finish` on all returned paths; cancelled reads must resume without losing a block or object. + Consuming `group_by_shard` replays the complete capture and retires its input + run before returning; grouped cursors retain the same finish/cleanup contract. Source: `crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs`. ## Features and platform diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index af455ce1e..f2e14dcc9 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -106,6 +106,13 @@ consumption; sampled physical checks cannot stand in for these counts. Physical consume memory; each packed delta retains its format-bounded decoder. Logical reduction has a live disk ceiling of sixteen times the admitted catalog limit; normal lookup and materialization budgets remain unchanged. +`PointerCatalogInspection::group_by_shard` consumes the complete verified capture, +including rows already visited by its object cursor. It reorders only private +spill blocks: xorbs first, then each shard and its final file bindings. The same +disk ceiling and cleanup owner apply; the original run retires before the grouped +cursor returns. Ordinary GC/readiness consumers do not pay this extra reduction. +Fsck consumes bounded file-binding batches while retaining one shard body and its +xorb metadata, rather than a complete file map or a scan of every file per shard. GC traverses catalog pages from current, historical and retained browse roots. Protected keys and authenticated closure traversal are tracked separately. Retired unshipped checkpoint formats through 8 are rejected without fallback. diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 8595677ec..51360bab1 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -73,8 +73,8 @@ pub use pointer::{ }; #[cfg(feature = "storage")] pub use pointer::{ - PointerCatalogCounts, PointerCatalogInspection, PointerCatalogLookup, PointerCatalogObject, - inspect_pointer_catalog, + PointerCatalogCounts, PointerCatalogEntry, PointerCatalogInspection, PointerCatalogLookup, + PointerCatalogObject, PointerCatalogShardInspection, inspect_pointer_catalog, }; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index bae7c2a6b..7d491f3d0 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -19,7 +19,8 @@ mod relations; pub(super) mod spill; #[cfg(feature = "storage")] pub use inspection::{ - PointerCatalogCounts, PointerCatalogInspection, PointerCatalogObject, inspect_pointer_catalog, + PointerCatalogCounts, PointerCatalogEntry, PointerCatalogInspection, PointerCatalogObject, + PointerCatalogShardInspection, inspect_pointer_catalog, }; pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs index f4c53a1d0..b4eb835bd 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs @@ -15,6 +15,70 @@ pub(super) struct Entry { } impl Entry { + fn external_object(self) -> Result { + let hash = hex(&self.key[1..])?; + let size = u64::from_be_bytes( + self.value[..8] + .try_into() + .map_err(|_| corrupt("object size is truncated"))?, + ); + let object = match self.key[0] { + 1 => PointerCatalogObject::Shard { + hash, + entry: ShardCatalogEntry::new( + size, + self.value[8..] + .as_chunks::<32>() + .0 + .iter() + .map(|value| hex(value)) + .collect::>()?, + ), + }, + 2 => PointerCatalogObject::Xorb { + hash, + entry: XorbCatalogEntry::new( + size, + hex(&self.value[8..40])?, + self.value[40..] + .as_chunks::<36>() + .0 + .iter() + .map(|chunk| { + Ok(XorbChunkEntry::new( + hex(&chunk[..32])?, + u32::from_be_bytes( + chunk[32..] + .try_into() + .map_err(|_| corrupt("chunk size is truncated"))?, + ), + )) + }) + .collect::>()?, + ), + }, + _ => return Err(corrupt("logical catalog object kind differs")), + }; + Ok(object) + } + fn encode_rows<'a>(rows: impl ExactSizeIterator) -> Result { + let count = rows.len(); + let mut bytes = MAGIC.to_vec(); + bytes.extend_from_slice(&(count as u32).to_be_bytes()); + for row in rows { + row.validate()?; + bytes.extend_from_slice(&row.key); + bytes.extend_from_slice(&row.rank.to_be_bytes()); + let size = u32::try_from(row.value.len()) + .map_err(|_| corrupt("logical spill value exceeds its length field"))?; + bytes.extend_from_slice(&size.to_be_bytes()); + bytes.extend_from_slice(&row.value); + } + if count == 0 || count > 4096 || bytes.len() as u64 > Self::MAX_BYTES { + return Err(corrupt("logical catalog spill block exceeds its bound")); + } + Ok(bytes.into()) + } fn new(kind: u8, identity: &str, rank: u64, value: Vec) -> Result { let mut key = [0; 33]; key[0] = kind; @@ -62,6 +126,7 @@ impl Entry { } impl Row for Entry { + type Key = [u8; 33]; const TARGET_BYTES: usize = 512 * 1024; const MAX_BYTES: u64 = MAX_CATALOG_BYTES + 128; fn key(&self) -> &[u8; 33] { @@ -91,21 +156,7 @@ impl Row for Entry { Ok(()) } fn encode(rows: &[Self]) -> Result { - let mut bytes = MAGIC.to_vec(); - bytes.extend_from_slice(&(rows.len() as u32).to_be_bytes()); - for row in rows { - row.validate()?; - bytes.extend_from_slice(&row.key); - bytes.extend_from_slice(&row.rank.to_be_bytes()); - let size = u32::try_from(row.value.len()) - .map_err(|_| corrupt("logical spill value exceeds its length field"))?; - bytes.extend_from_slice(&size.to_be_bytes()); - bytes.extend_from_slice(&row.value); - } - if rows.is_empty() || rows.len() > 4096 || bytes.len() as u64 > Self::MAX_BYTES { - return Err(corrupt("logical catalog spill block exceeds its bound")); - } - Ok(bytes.into()) + Self::encode_rows(rows.iter()) } fn decode(bytes: Bytes) -> Result> { if bytes.len() < 12 || bytes.get(..8) != Some(MAGIC.as_slice()) { @@ -358,50 +409,7 @@ impl PointerCatalogInspection { if row.key[0] == 0 { continue; } - let hash = hex(&row.key[1..])?; - let size = u64::from_be_bytes( - row.value[..8] - .try_into() - .map_err(|_| corrupt("object size is truncated"))?, - ); - let object = match row.key[0] { - 1 => PointerCatalogObject::Shard { - hash, - entry: ShardCatalogEntry::new( - size, - row.value[8..] - .as_chunks::<32>() - .0 - .iter() - .map(|value| hex(value)) - .collect::>()?, - ), - }, - 2 => PointerCatalogObject::Xorb { - hash, - entry: XorbCatalogEntry::new( - size, - hex(&row.value[8..40])?, - row.value[40..] - .as_chunks::<36>() - .0 - .iter() - .map(|chunk| { - Ok(XorbChunkEntry::new( - hex(&chunk[..32])?, - u32::from_be_bytes( - chunk[32..] - .try_into() - .map_err(|_| corrupt("chunk size is truncated"))?, - ), - )) - }) - .collect::>()?, - ), - }, - _ => return Err(corrupt("logical catalog object kind differs")), - }; - return Ok(Some(object)); + return row.external_object().map(Some); } Ok(None) } @@ -493,5 +501,8 @@ pub async fn inspect_pointer_catalog( inspector.finish(checked).await } +mod grouped; +pub use grouped::{PointerCatalogEntry, PointerCatalogShardInspection}; + #[cfg(test)] mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs new file mode 100644 index 000000000..49afb7a34 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs @@ -0,0 +1,157 @@ +//! Local ordering of a verified capture for one-shard-at-a-time integrity checks. +use super::*; + +struct Grouped { + key: [u8; 66], + entry: Entry, +} + +impl Grouped { + fn new(entry: Entry) -> Result { + let mut key = [0; 66]; + match entry.key[0] { + 0 => { + key[0] = 1; + key[1..33].copy_from_slice(&entry.value[8..]); + key[33] = 1; + key[34..].copy_from_slice(&entry.key[1..]); + } + 1 => { + key[0] = 1; + key[1..33].copy_from_slice(&entry.key[1..]); + } + 2 => key[1..33].copy_from_slice(&entry.key[1..]), + _ => return Err(corrupt("grouped catalog row kind differs")), + } + Ok(Self { key, entry }) + } +} + +impl Row for Grouped { + type Key = [u8; 66]; + const TARGET_BYTES: usize = Entry::TARGET_BYTES; + const MAX_BYTES: u64 = Entry::MAX_BYTES; + fn key(&self) -> &Self::Key { + &self.key + } + fn bytes(&self) -> usize { + self.entry.bytes().saturating_add(self.key.len()) + } + fn merge(&mut self, _: Self) -> Result<()> { + Err(corrupt("normalized catalog grouping repeated an entry")) + } + fn encode(rows: &[Self]) -> Result { + Entry::encode_rows(rows.iter().map(|row| &row.entry)) + } + fn decode(bytes: Bytes) -> Result> { + Entry::decode(bytes)?.into_iter().map(Self::new).collect() + } +} + +/// One record in a verified catalog ordered for shard integrity inspection. +#[derive(Debug, PartialEq, Eq)] +pub enum PointerCatalogEntry { + Object(PointerCatalogObject), + File { + hash: String, + entry: FileCatalogEntry, + }, +} + +/// A verified capture ordered as xorbs followed by each shard and its file bindings. +/// +/// File bindings within a shard follow file-hash order. Await `finish` on all +/// returned paths; dropping retains cleanup behind blocking workers. +#[must_use] +pub struct PointerCatalogShardInspection { + space: Option, + rows: Cursor, + counts: PointerCatalogCounts, +} + +impl PointerCatalogInspection { + /// Reorder the complete verified capture locally, grouping all final file bindings by shard. + /// + /// Prior cursor consumption does not omit records. The same private disk + /// ceiling applies; input spill files retire before this method returns. + /// No origin object is read or written. One recipe may reach its admitted bound. + pub async fn group_by_shard(self) -> Result { + let Self { + space, + rows, + counts, + } = self; + let Some(space) = space else { + return Ok(PointerCatalogShardInspection { + space: None, + rows: Cursor::new(Run::default()), + counts, + }); + }; + let result: Result = async { + let mut rows = rows.restart(); + let mut grouped = Reducer::::new(); + while let Some(entry) = rows.next(&space).await? { + grouped.add(&space, Grouped::new(entry)?).await?; + } + let run = grouped.finish(&space).await?; + rows.into_run().remove(&space).await?; + Ok(run) + } + .await; + match result { + Ok(run) => Ok(PointerCatalogShardInspection { + space: Some(space), + rows: Cursor::new(run), + counts, + }), + Err(error) => finish(Some(space), Err(error)).await, + } + } +} + +impl PointerCatalogShardInspection { + /// Return complete cardinalities, independent of iteration progress. + #[must_use] + pub fn counts(&self) -> PointerCatalogCounts { + self.counts + } + + /// Return the next external object or final file binding in shard-grouped order. + /// + /// A pending read can be cancelled and resumed without losing its next entry. + pub async fn next_entry(&mut self) -> Result> { + let row = match &self.space { + Some(space) => self.rows.next(space).await?, + None => None, + }; + let Some(row) = row else { + return Ok(None); + }; + let entry = row.entry; + if entry.key[0] != 0 { + return entry + .external_object() + .map(PointerCatalogEntry::Object) + .map(Some); + } + let hash = hex(&entry.key[1..])?; + let size = u64::from_be_bytes( + entry.value[..8] + .try_into() + .map_err(|_| corrupt("grouped file size is truncated"))?, + ); + Ok(Some(PointerCatalogEntry::File { + hash, + entry: FileCatalogEntry::new(size, hex(&entry.value[8..])?), + })) + } + + /// Drain and remove the private spill, preserving the operation's primary error. + pub async fn finish>( + self, + result: std::result::Result, + ) -> std::result::Result { + finish(self.space, result).await + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs index ed0e0469b..7b86f16da 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -433,3 +433,186 @@ fn cancelled_pending_cursor_read_resumes_at_the_same_object() { assert_eq!(actual, Some(expected.remove(0))); }); } + +#[tokio::test] +async fn grouping_restarts_a_partially_consumed_capture_and_retires_input_spill() { + let (parent, mut cursor, mut objects) = cursor_fixture(5000).await; + cursor.next_object().await.unwrap(); + let directory = std::fs::read_dir(parent.path()) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let input = std::fs::read_dir(&directory) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + let mut grouped = cursor.group_by_shard().await.unwrap(); + let output = std::fs::read_dir(&directory) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert!(input.is_disjoint(&output)); + let mut expected = vec![ + PointerCatalogEntry::Object(objects.remove(1)), + PointerCatalogEntry::Object(objects.remove(0)), + ]; + expected.extend((0..5000).map(|file| PointerCatalogEntry::File { + hash: format!("{file:064x}"), + entry: FileCatalogEntry::new(19, format!("{:064x}", 1)), + })); + let result: Result<_> = async { + let mut entries = Vec::new(); + while let Some(entry) = grouped.next_entry().await? { + entries.push(entry); + } + Ok(entries) + } + .await; + assert_eq!(grouped.finish(result).await.unwrap(), expected); + assert_eq!(std::fs::read_dir(parent.path()).unwrap().count(), 0); +} + +#[tokio::test] +async fn grouped_file_bindings_match_assembly_after_shard_rebinding() { + let parent = tempfile::tempdir().unwrap(); + let mut inspector = Inspector { + space: Some( + ScratchSpace::create(parent.path(), 16 * 1024 * 1024) + .await + .unwrap(), + ), + rows: Reducer::new(), + maximum: 1024 * 1024, + }; + let mut initial = binding(0, 99, 3); + for file in 1..13_000 { + initial + .merge_delta(&binding(file, file % 3 + 1, 3)) + .unwrap(); + } + let deltas = vec![ + initial, + binding(0, 3, 3), + shard(1, Vec::new()), + shard(2, Vec::new()), + shard(3, Vec::new()), + ]; + let mut oracle = PointerCatalogAssembly::new(PointerCatalog::new()); + for (rank, delta) in deltas.into_iter().enumerate() { + oracle.apply(&delta).unwrap(); + inspector.delta(delta, rank as u64).await.unwrap(); + } + let oracle = oracle.finish().unwrap(); + let mut expected = Vec::new(); + for (hash, entry) in oracle.shards() { + expected.push(PointerCatalogEntry::Object(PointerCatalogObject::Shard { + hash: hash.clone(), + entry: entry.clone(), + })); + expected.extend( + oracle + .files() + .iter() + .filter(|(_, entry)| entry.shard_hash() == hash) + .map(|(hash, entry)| PointerCatalogEntry::File { + hash: hash.clone(), + entry: entry.clone(), + }), + ); + } + let mut grouped = inspector + .finish(Ok(())) + .await + .unwrap() + .group_by_shard() + .await + .unwrap(); + let result: Result<_> = async { + let mut entries = Vec::new(); + while let Some(entry) = grouped.next_entry().await? { + entries.push(entry); + } + Ok(entries) + } + .await; + assert_eq!(grouped.finish(result).await.unwrap(), expected); +} + +#[tokio::test] +async fn grouped_inspection_retires_spill_on_early_error_drop_and_tampering() { + for outcome in ["early", "error", "drop", "tamper", "input-tamper"] { + let (parent, cursor, _) = cursor_fixture(64).await; + let directory = std::fs::read_dir(parent.path()) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + if outcome == "input-tamper" { + for entry in std::fs::read_dir(directory).unwrap() { + std::fs::write(entry.unwrap().path(), b"changed").unwrap(); + } + assert!(cursor.group_by_shard().await.is_err()); + } else { + let mut grouped = cursor.group_by_shard().await.unwrap(); + match outcome { + "early" => { + grouped.next_entry().await.unwrap(); + grouped.finish(Ok::<_, MetadataError>(())).await.unwrap(); + } + "error" => { + assert!( + grouped + .finish(Err::<(), _>(contract_error("body failed"))) + .await + .is_err() + ); + } + "drop" => drop(grouped), + "tamper" => { + for entry in std::fs::read_dir(directory).unwrap() { + std::fs::write(entry.unwrap().path(), b"changed").unwrap(); + } + let result = grouped.next_entry().await; + assert!(result.is_err()); + assert!(grouped.next_entry().await.is_err()); + assert!(grouped.finish(result).await.is_err()); + } + _ => unreachable!(), + } + } + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while std::fs::read_dir(parent.path()).unwrap().count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } +} + +#[tokio::test] +async fn grouping_disk_admission_failure_removes_the_workspace() { + let parent = tempfile::tempdir().unwrap(); + let mut inspector = Inspector { + space: Some( + ScratchSpace::create(parent.path(), 1024 * 1024) + .await + .unwrap(), + ), + rows: Reducer::new(), + maximum: 1024 * 1024, + }; + for file in 0..5000 { + inspector.delta(binding(file, 1, 3), 0).await.unwrap(); + } + inspector.delta(shard(1, Vec::new()), 0).await.unwrap(); + let cursor = inspector.finish(Ok(())).await.unwrap(); + assert!(matches!( + cursor.group_by_shard().await, + Err(MetadataError::Storage { .. }) + )); + assert_eq!(std::fs::read_dir(parent.path()).unwrap().count(), 0); +} diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/relations.rs b/crates/crab-metadata/src/capsule_protocol/pointer/relations.rs index 7e6f418d7..0fcc30ff1 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/relations.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/relations.rs @@ -157,6 +157,7 @@ fn decode(bytes: &[u8]) -> Result> { } impl Row for Record { + type Key = [u8; 33]; const TARGET_BYTES: usize = BLOCK_ROWS * ROW_BYTES; const MAX_BYTES: u64 = MAX_BLOCK_BYTES; fn key(&self) -> &[u8; 33] { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs b/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs index 88e4b57a7..c851ca694 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/spill.rs @@ -1,12 +1,12 @@ //! Bounded sorted reductions over authenticated private spill blocks. use super::*; use crab_storage::scratch::{ScratchFile, ScratchSpace}; -use std::collections::VecDeque; pub(super) trait Row: Sized { + type Key: Ord + Clone; const TARGET_BYTES: usize; const MAX_BYTES: u64; - fn key(&self) -> &[u8; 33]; + fn key(&self) -> &Self::Key; fn bytes(&self) -> usize; fn merge(&mut self, other: Self) -> Result<()>; fn encode(rows: &[Self]) -> Result; @@ -27,37 +27,54 @@ impl Run { } } -pub(super) struct Cursor { - files: VecDeque, +pub(super) struct Cursor { + files: Vec, + next_file: usize, rows: std::vec::IntoIter, - previous: Option<[u8; 33]>, + previous: Option, } impl Cursor { pub(super) fn new(run: Run) -> Self { Self { - files: run.files.into(), + files: run.files, + next_file: 0, rows: Vec::new().into_iter(), previous: None, } } + pub(super) fn restart(mut self) -> Self { + self.next_file = 0; + self.rows = Vec::new().into_iter(); + self.previous = None; + self + } + + pub(super) fn into_run(self) -> Run { + Run { files: self.files } + } + pub(super) async fn next(&mut self, space: &ScratchSpace) -> Result> { loop { if let Some(row) = self.rows.as_slice().first() { - if self.previous.is_some_and(|previous| previous >= *row.key()) { + if self + .previous + .as_ref() + .is_some_and(|previous| previous >= row.key()) + { return Err(corrupt("inspection spill rows are not strictly ordered")); } - self.previous = Some(*row.key()); + self.previous = Some(row.key().clone()); return Ok(self.rows.next()); } - let Some(file) = self.files.front() else { + let Some(file) = self.files.get(self.next_file) else { return Ok(None); }; // A caller can cancel and resume this read. Commit progression only // after verified I/O and decoding, or the unread block is lost. let rows = R::decode(space.read(file, R::MAX_BYTES).await?)?; - let _ = self.files.pop_front(); + self.next_file += 1; self.rows = rows.into_iter(); } } diff --git a/crates/crab-read/AGENTS.md b/crates/crab-read/AGENTS.md index f472bc904..3c1bd2afa 100644 --- a/crates/crab-read/AGENTS.md +++ b/crates/crab-read/AGENTS.md @@ -35,6 +35,9 @@ For object download changes continue into `crates/crab-read/src/store_client.rs` - Requests naming hidden refs remain rejected even when arbitrary wants are enabled. This does not imply that the same SHA is denied under every other name; inspect policy and advertisement together. Source: `crates/crab-read/src/fetch_admission.rs`. +- `verify_captured_installed_pointer_bytes` proves reachable Git/Crab/LFS bytes + against captured selected closures. Complete integrity callers separately + prove all catalog descriptors; do not treat this phase as global catalog proof. - Preserve the difference between full-file hash verification and range/chunk verification; partial output is not proof of the complete file. Source: `crates/crab-read/src/hydrator.rs`. - TermResolver construction rejects zero/oversized permit counts. Its semaphore diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index c5228b31d..472375a21 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -482,9 +482,9 @@ tracks physical memory separately from this admission limit. Git views capture footer-rooted checkpoint visibility readers and load requested membership pages separately from retained-object presence. Complete visibility inspection is an explicit async operation. `CapsuleRepositoryView::pointer_catalog` -is also async: hydration, publication and maintenance materialize the complete -catalog from the captured checkpoint when its body was omitted during opening. -Shared preparation and hydration still need selected-lookup integration. +is also async: explicit bulk maintenance materializes the complete catalog from +the captured checkpoint. Publication preparation and hydration use selected +lookups; complete fsck streams its catalog proof. `CapsuleRepositoryView::pointer_catalog_lookup` captures the checkpoint's paged catalog together with authenticated run roots and exactly the selected @@ -505,4 +505,8 @@ reachable pointer bindings in batches of 64 from those same roots. Catalog cardinalities come from the completed metadata reduction. Recovery's assembled replacement catalogs share the Git scan and whole-file/LFS proof in `capsule_protocol/dependency.rs`; binding source selection does not change the -integrity checks. Current fsck's additional cross-recipe checks remain separate. +integrity checks. Fsck composes its stronger decoded-chunk and cross-recipe proof +with `verify_captured_installed_pointer_bytes`, the same canonical Git scan and +whole-file/LFS byte phase. That API validates selected dependency closures only; +it does not certify unrelated catalog health. Complete fsck must first finish +its full catalog/object proof against the same captured sources. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 5011492dc..3c7475a54 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -24,7 +24,10 @@ use tokio_util::sync::CancellationToken; use crate::{ReadError, Result}; mod dependency; -pub use dependency::{verify_installed_dependencies, verify_reachable_dependencies}; +pub use dependency::{ + verify_captured_installed_pointer_bytes, verify_installed_dependencies, + verify_reachable_dependencies, +}; #[cfg(test)] mod visibility_tests; @@ -5927,6 +5930,30 @@ mod tests { .await, Err(ReadError::IncompleteCatalogView) )); + let workspace = tempfile::tempdir().unwrap(); + for cancelled in [false, true] { + let cancel = CancellationToken::new(); + if cancelled { + cancel.cancel(); + } + let result = verify_captured_installed_pointer_bytes( + &router, + &workspace.path().join("absent.git"), + &fetch, + crab_git::walk::PointerScanLimits { + objects: 16, + lookups: 64, + allocation_bytes: 1024 * 1024, + }, + &cancel, + ) + .await; + if cancelled { + assert!(matches!(result, Err(ReadError::Cancelled))); + } else { + assert!(matches!(result, Err(ReadError::IncompleteCatalogView))); + } + } assert!(observer.observations.lock().unwrap().is_empty()); let complete = open_git_view_from_root(&router, root, TEST_LIMITS) .await diff --git a/crates/crab-read/src/capsule_protocol/dependency.rs b/crates/crab-read/src/capsule_protocol/dependency.rs index 9e47ffa6b..d0470a175 100644 --- a/crates/crab-read/src/capsule_protocol/dependency.rs +++ b/crates/crab-read/src/capsule_protocol/dependency.rs @@ -221,6 +221,67 @@ async fn verify_installed_dependencies_from( cancellation: &CancellationToken, ) -> Result { let (counts, catalog_stats) = catalog.verify(layout, cancellation).await?; + let proof = + verify_installed_pointer_bytes_from(layout, git_dir, refs, catalog, limits, cancellation) + .await?; + Ok(CapsuleDependencyProof { + catalog_files: counts.files, + catalog_shards: counts.shards, + catalog_xorbs: counts.xorbs, + catalog_objects_read: catalog_stats.object_read_count, + reachable_crab_pointers: proof.crab, + reachable_lfs_objects: proof.lfs_objects, + reachable_lfs_bytes: proof.lfs_bytes, + }) +} + +struct PointerProof { + crab: u64, + lfs_objects: u64, + lfs_bytes: u64, +} + +/// Verify reachable Git, Crab and LFS bytes in an installed database against captured bindings. +/// +/// Callers install authenticated packs matching this view and retain the database +/// and captured sources through completion. This checks selected dependency closures; +/// complete repository inspection must separately validate all catalog descriptors. +/// Token cancellation drains the Git worker before returning. +pub async fn verify_captured_installed_pointer_bytes( + layout: &StoreLayout, + git_dir: &Path, + view: &CapsuleRepositoryView, + limits: crab_git::walk::PointerScanLimits, + cancellation: &CancellationToken, +) -> Result<()> { + if cancellation.is_cancelled() { + return Err(ReadError::Cancelled); + } + let lookup = + view.pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; + verify_installed_pointer_bytes_from( + layout, + git_dir, + view.refs(), + DependencyCatalog::Captured { + view, + lookup: &lookup, + }, + limits, + cancellation, + ) + .await + .map(|_| ()) +} + +async fn verify_installed_pointer_bytes_from( + layout: &StoreLayout, + git_dir: &Path, + refs: &BTreeMap, + catalog: DependencyCatalog<'_>, + limits: crab_git::walk::PointerScanLimits, + cancellation: &CancellationToken, +) -> Result { let refs = refs .iter() .map(|(name, oid)| (name.clone(), oid.clone())) @@ -316,14 +377,10 @@ async fn verify_installed_dependencies_from( } } - Ok(CapsuleDependencyProof { - catalog_files: counts.files, - catalog_shards: counts.shards, - catalog_xorbs: counts.xorbs, - catalog_objects_read: catalog_stats.object_read_count, - reachable_crab_pointers: scan.pointers.len() as u64, - reachable_lfs_objects: lfs_objects.len() as u64, - reachable_lfs_bytes: lfs_objects.values().try_fold(0_u64, |total, size| { + Ok(PointerProof { + crab: scan.pointers.len() as u64, + lfs_objects: lfs_objects.len() as u64, + lfs_bytes: lfs_objects.values().try_fold(0_u64, |total, size| { total .checked_add(*size) .ok_or_else(|| ReadError::internal("LFS dependency byte count overflowed")) diff --git a/crates/crab-read/src/capsule_protocol/dependency_tests.rs b/crates/crab-read/src/capsule_protocol/dependency_tests.rs index 99bc8cf28..878ed7ae3 100644 --- a/crates/crab-read/src/capsule_protocol/dependency_tests.rs +++ b/crates/crab-read/src/capsule_protocol/dependency_tests.rs @@ -698,3 +698,25 @@ async fn captured_binding_batches_do_not_repeat_file_reconstruction() { (66, baseline) ); } + +#[tokio::test] +async fn pointer_byte_phase_checks_selected_closures_without_claiming_global_catalog_health() { + let (layout, mut catalog, workspace, refs) = fixture("valid").await; + catalog + .insert_file("a".repeat(64), FileCatalogEntry::new(7, "b".repeat(64))) + .unwrap(); + let view = captured_catalog(&layout, &catalog, &refs).await; + verify_captured_installed_pointer_bytes( + &layout, + &workspace.path().join("repository.git"), + &view, + SCAN_LIMITS, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + verify_fixture(&layout, workspace.path(), &refs, &catalog, true).await, + Err(ReadError::Metadata(_)) + )); +} diff --git a/crates/crab-xet/README.md b/crates/crab-xet/README.md index 30ac02bb3..a1ed79ab3 100644 --- a/crates/crab-xet/README.md +++ b/crates/crab-xet/README.md @@ -68,6 +68,10 @@ For a lower-level read path, parse a complete serialized Xorb with `xorb::builder::XorbBuilder` and upload each `XorbResult` through the owning storage or staging layer. +`shard::FileBundleValidator` indexes a borrowed xorb metadata bundle once for +multiple file-term checks. It preserves the same range and byte-coverage checks +as `validate_file_bundle`; neither API proves payload hashes or whole-file bytes. + ## Xorb verification | API | What it verifies | diff --git a/crates/crab-xet/src/shard.rs b/crates/crab-xet/src/shard.rs index bfa2731d0..10a022e9f 100644 --- a/crates/crab-xet/src/shard.rs +++ b/crates/crab-xet/src/shard.rs @@ -322,13 +322,32 @@ fn validate_file_terms<'a>( Ok(()) } +/// Reusable file-term validation against one borrowed xorb metadata bundle. +pub struct FileBundleValidator<'a> { + by_hash: HashMap, +} + +impl<'a> FileBundleValidator<'a> { + /// Index the supplied dependencies once for multiple file checks. + #[must_use] + pub fn new(dependencies: &'a [MDBXorbInfo]) -> Self { + Self { + by_hash: dependencies + .iter() + .map(|dependency| (dependency.metadata.xorb_hash, dependency)) + .collect(), + } + } + + /// Validate one file's ranges and byte coverage against this exact bundle. + pub fn validate(&self, file_info: &MDBFileInfo) -> Result<()> { + validate_file_terms(file_info, |hash| self.by_hash.get(hash).copied()) + } +} + /// Validate one file entry against the exact xorb metadata bundled with it. pub fn validate_file_bundle(file_info: &MDBFileInfo, dependencies: &[MDBXorbInfo]) -> Result<()> { - let by_hash = dependencies - .iter() - .map(|dependency| (dependency.metadata.xorb_hash, dependency)) - .collect::>(); - validate_file_terms(file_info, |hash| by_hash.get(hash).copied()) + FileBundleValidator::new(dependencies).validate(file_info) } /// Lazy-loading shard reader backed by raw bytes. @@ -823,6 +842,19 @@ mod tests { } } + #[test] + fn bundle_validator_keeps_dependencies_after_a_failed_file() { + let dependency = make_xorb(7, 2); + let dependencies = vec![dependency.as_ref().clone()]; + let validator = FileBundleValidator::new(&dependencies); + let mut invalid = make_file(1, 7); + invalid.segments[0].unpacked_segment_bytes += 1; + assert!(validator.validate(&invalid).is_err()); + assert!(validator.validate(&make_file(2, 7)).is_ok()); + assert!(validator.validate(&make_file(3, 8)).is_err()); + assert!(validator.validate(&make_file(4, 7)).is_ok()); + } + #[test] fn writer_empty_by_default() { let w = ShardWriter::new(); From d796b3dff581195da251f57ea05e8bbc9492a8ff Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 06:30:45 -0700 Subject: [PATCH 125/127] perf(history): stream checkpoint dependency proof --- .../docs/architecture/indexed-remote-reads.md | 6 +- crab/src/cmd/history_recovery_v2.rs | 26 +-- .../layered_catalog_tests.rs | 178 ++++++++++++++++++ .../cmd/history_recovery_v2/layered_tests.rs | 60 ++++-- crates/crab-metadata/README.md | 4 +- .../capsule_protocol/pointer/inspection.rs | 29 ++- .../pointer/inspection/grouped.rs | 1 + .../pointer/inspection/tests.rs | 22 ++- crates/crab-read/AGENTS.md | 2 +- crates/crab-read/README.md | 7 + crates/crab-read/src/capsule_protocol.rs | 6 +- .../src/capsule_protocol/dependency.rs | 69 ++++++- .../src/capsule_protocol/dependency_tests.rs | 10 + crates/crab-read/src/selection.rs | 5 +- 14 files changed, 368 insertions(+), 57 deletions(-) create mode 100644 crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index e50bcada2..ad07873cf 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2560,8 +2560,10 @@ descriptor pass. Constructor admission completes and closes its own inspection; a later full-data phase opens a fresh inspection of the same pinned sources. This can scan metadata twice, but avoids retaining operation spill across checker phases and prevents an unrelated phase from reporting success before full proof. -Ordinary GC and replica readiness do not perform shard grouping. Restore -accounting and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to +Ordinary GC and replica readiness do not perform shard grouping. Historical +verification uses the same streamed complete catalog/body proof plus checkpoint- +bound file lookups, preserving checked dependency counts and byte accounting. +Restore's catalog merge and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to protect those callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index d04677c8c..9d40a7911 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -206,25 +206,20 @@ async fn verify_history( verify_history_sources(layout, &segment, &checkpoint, cancel).await?; check_cancelled(cancel)?; - let catalog = checkpoint - .pointer_catalog_from_store(layout, u64::MAX) - .await?; validate_visibility(&segment, &checkpoint)?; let workspace = verify_git_checkpoint(layout, &segment, &checkpoint).await?; - let dependencies = crab_read::capsule_protocol::verify_installed_dependencies( + let dependencies = crab_read::capsule_protocol::verify_installed_checkpoint_dependencies( layout, &workspace.path().join("repository.git"), segment.refs(), - &catalog, + &checkpoint, crate::cmd::fsck_store::CAPSULE_GIT_SCAN_LIMITS, cancel, ) .await?; check_cancelled(cancel)?; - let shard_count = u64::try_from(catalog.shards().len()) - .map_err(|_| CrabError::Internal("history shard count overflowed".to_owned()))?; - let xorb_count = u64::try_from(catalog.xorbs().len()) - .map_err(|_| CrabError::Internal("history xorb count overflowed".to_owned()))?; + let shard_count = dependencies.catalog_shards; + let xorb_count = dependencies.catalog_xorbs; let catalog_objects = checkpoint.catalog_objects(layout, u64::MAX).await?; let dependency_objects = 2_u64 .checked_add(catalog_objects.len() as u64) @@ -245,18 +240,7 @@ async fn verify_history( .checked_add(checkpoint_bytes) .and_then(|total| total.checked_add(source_bytes)) .and_then(|total| total.checked_add(dependencies.reachable_lfs_bytes)) - .and_then(|total| { - catalog - .shards() - .values() - .try_fold(total, |sum, shard| sum.checked_add(shard.encoded_size())) - }) - .and_then(|total| { - catalog - .xorbs() - .values() - .try_fold(total, |sum, xorb| sum.checked_add(xorb.encoded_size())) - }) + .and_then(|total| total.checked_add(dependencies.catalog_payload_bytes)) .ok_or_else(|| CrabError::Internal("history dependency bytes overflowed".to_owned()))?; let verification = HistoryVerificationPayload { generation: segment.checkpoint().covered_generation(), diff --git a/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs b/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs new file mode 100644 index 000000000..2da05d627 --- /dev/null +++ b/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs @@ -0,0 +1,178 @@ +use super::*; + +async fn xet_history_fixture() -> ( + CapsuleStoreLayout, + RootSnapshot, + Vec<(object_store::path::Path, Bytes)>, +) { + use crab_metadata::capsule_protocol::{ + FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + }; + use crab_xet::{ + hash::MerkleHash, + shard::{ShardWriter, file_info_from_placements, xorb_info_from_placements}, + xorb::{ + builder::{RunId, XorbBuilder}, + format::Chunk, + }, + }; + let layout = empty_fixture().await; + let content = Bytes::from_static(b"historical xet payload"); + let chunk = Chunk::new(content.clone()); + let mut builder = XorbBuilder::new(); + builder.push(&chunk, RunId(0)).unwrap(); + let xorb = builder.finalize().unwrap().pop().unwrap(); + let file_hash = MerkleHash::from(*blake3::hash(&content).as_bytes()); + let placements = xorb + .placements + .iter() + .cloned() + .map(|p| (p.chunk_hash, p)) + .collect(); + let mut writer = ShardWriter::new(); + writer + .add_xorb(Arc::new( + xorb_info_from_placements(xorb.hash, &xorb.placements).unwrap(), + )) + .unwrap(); + writer + .add_file(file_info_from_placements(file_hash, &[chunk.hash], &placements).unwrap()) + .unwrap(); + let (shard, shard_hash) = writer.finalize().unwrap(); + let bodies = vec![ + (layout.xorb_path(&xorb.hash), xorb.bytes), + (layout.shard_path(&shard_hash), Bytes::from(shard)), + ]; + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hash.hex(), + XorbCatalogEntry::new( + bodies[0].1.len() as u64, + blake3::hash(&bodies[0].1).to_hex().to_string(), + vec![XorbChunkEntry::new(chunk.hash.hex(), content.len() as u32)], + ), + ) + .unwrap(); + catalog + .insert_shard( + shard_hash.hex(), + ShardCatalogEntry::new(bodies[1].1.len() as u64, vec![xorb.hash.hex()]), + ) + .unwrap(); + catalog + .insert_file( + file_hash.hex(), + FileCatalogEntry::new(content.len() as u64, shard_hash.hex()), + ) + .unwrap(); + for (path, body) in &bodies { + layout.store().put(path, body.clone()).await.unwrap(); + } + let pointer = crab_types::pointer::Pointer { + file_hash: file_hash.into(), + size: content.len() as u64, + shard_hint: None, + }; + publish_blob(&layout, "refs/tags/xet", &pointer.serialize(), catalog).await; + assert!( + crab_remote::checkpoint::publish_capsule_checkpoint( + &layout, + 1, + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap() + .published + ); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + (layout, root, bodies) +} + +#[tokio::test] +async fn historical_catalog_accounting_counts_distinct_encoded_bodies() { + let (layout, root, bodies) = xet_history_fixture().await; + let chain = history_chain(&layout, &root).await.unwrap(); + let segment = &chain[0]; + let verified = verify_history( + &layout, + &root, + segment.checkpoint().covered_generation(), + Some(segment.hash()), + &CancellationToken::new(), + ) + .await + .unwrap(); + let pages = verified + .checkpoint + .catalog_objects(&layout, LIMIT) + .await + .unwrap(); + let expected_bytes = segment.bytes().len() as u64 + + verified.checkpoint.bytes().len() as u64 + + pages.iter().map(|(_, bytes)| *bytes).sum::() + + verified + .checkpoint + .sources() + .iter() + .map(|source| source.object_size()) + .sum::() + + bodies + .iter() + .map(|(_, body)| body.len() as u64) + .sum::(); + assert_eq!( + ( + verified.verification.shards, + verified.verification.xorbs, + verified.verification.dependency_objects, + verified.verification.dependency_bytes + ), + ( + 1, + 1, + 2 + pages.len() as u64 + + verified.checkpoint.sources().len() as u64 + + bodies.len() as u64, + expected_bytes + ) + ); +} + +#[tokio::test] +async fn historical_checkpoint_proof_rejects_corrupt_catalog_payload_bodies() { + let (layout, root, bodies) = xet_history_fixture().await; + let chain = history_chain(&layout, &root).await.unwrap(); + let segment = &chain[0]; + for (path, original) in bodies { + let mut corrupt = original.to_vec(); + corrupt[0] ^= 1; + layout + .store() + .inner() + .put(&path, Bytes::from(corrupt).into()) + .await + .unwrap(); + let result = verify_history( + &layout, + &root, + segment.checkpoint().covered_generation(), + Some(segment.hash()), + &CancellationToken::new(), + ) + .await; + layout + .store() + .inner() + .put(&path, original.into()) + .await + .unwrap(); + assert!( + result.is_err(), + "accepted corrupt historical payload {path}" + ); + } +} diff --git a/crab/src/cmd/history_recovery_v2/layered_tests.rs b/crab/src/cmd/history_recovery_v2/layered_tests.rs index 9c7018fae..014ad89f5 100644 --- a/crab/src/cmd/history_recovery_v2/layered_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_tests.rs @@ -4,7 +4,7 @@ use super::*; use bytes::Bytes; use crab_metadata::capsule_protocol::{ Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, - CapsuleTransaction, CapsuleVisibilityDelta, + CapsuleTransaction, CapsuleVisibilityDelta, PointerCatalog, }; use crab_metadata::git_visibility::GitVisibilityEdit; use object_store::ObjectStoreExt; @@ -16,6 +16,7 @@ async fn publish_blob( layout: &CapsuleStoreLayout, name: &str, body: &[u8], + catalog: PointerCatalog, ) -> String { let kind = gix_object::Kind::Blob; let oid = crab_remote::objects::object_id(kind, body) @@ -51,7 +52,7 @@ async fn publish_blob( 1, ) .unwrap(); - publish_pack(layout, name, pack, &oid).await; + publish_pack(layout, name, pack, &oid, catalog).await; oid } @@ -60,6 +61,7 @@ async fn publish_pack( name: &str, pack: CapsuleGitPack, oid: &str, + catalog: PointerCatalog, ) { let root = crab_write::capsule_protocol::open_root(layout) .await @@ -74,15 +76,17 @@ async fn publish_pack( GitVisibilityEdit::from_replacement_objects(None, oid.to_owned(), vec![oid.to_owned()]), )])) .unwrap(); - let capsule = Capsule::build( - &transaction, - vec![pack], - vec![CapsuleSection::new( - CapsuleSectionKind::VisibilityDelta, - visibility.encode().unwrap(), - )], - ) - .unwrap(); + let mut sections = vec![CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + )]; + if !catalog.is_empty() { + sections.push(CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )); + } + let capsule = Capsule::build(&transaction, vec![pack], sections).unwrap(); crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) .await .unwrap(); @@ -105,7 +109,7 @@ async fn fixture() -> (CapsuleStoreLayout, RootSnapshot) { ("refs/tags/first", b"first version".as_slice()), ("refs/tags/second", b"second version".as_slice()), ] { - publish_blob(&layout, name, body).await; + publish_blob(&layout, name, body, PointerCatalog::new()).await; assert!( crab_remote::checkpoint::publish_capsule_checkpoint( &layout, @@ -168,7 +172,7 @@ async fn historical_thin_member_is_repaired_against_its_retained_base() { let layout = empty_fixture().await; let base = vec![b'a'; 32 * 1024]; - let base_oid = publish_blob(&layout, "refs/tags/base", &base).await; + let base_oid = publish_blob(&layout, "refs/tags/base", &base, PointerCatalog::new()).await; let base_oid = gix_hash::ObjectId::from_hex(base_oid.as_bytes()).unwrap(); let mut target = base.clone(); target[1024] = b'b'; @@ -211,7 +215,14 @@ async fn historical_thin_member_is_repaired_against_its_retained_base() { vec![base_oid.to_string()], ) .unwrap(); - publish_pack(&layout, "refs/tags/target", pack, &oid.to_string()).await; + publish_pack( + &layout, + "refs/tags/target", + pack, + &oid.to_string(), + PointerCatalog::new(), + ) + .await; assert!( crab_remote::checkpoint::publish_capsule_checkpoint( &layout, @@ -309,7 +320,13 @@ async fn layered_restore_preserves_sources_and_allows_new_epoch_publication() { .unwrap(); assert_eq!(retained_proof.verification.refs, 2); - let new_oid = publish_blob(&layout, "refs/tags/after-restore", b"after restore").await; + let new_oid = publish_blob( + &layout, + "refs/tags/after-restore", + b"after restore", + PointerCatalog::new(), + ) + .await; let view = crab_read::capsule_protocol::open_view( &layout, crab_read::capsule_protocol::CapsuleReadLimits { @@ -415,7 +432,7 @@ async fn history_verification_rejects_unavailable_external_pointer_content() { .serialize(); for (is_crab, bytes) in [(true, crab), (false, lfs)] { let layout = empty_fixture().await; - publish_blob(&layout, "refs/tags/pointer", &bytes).await; + publish_blob(&layout, "refs/tags/pointer", &bytes, PointerCatalog::new()).await; assert!( crab_remote::checkpoint::publish_capsule_checkpoint( &layout, @@ -468,7 +485,13 @@ async fn history_verification_counts_and_hashes_reachable_lfs_content() { size: content.len() as u64, extensions: Vec::new(), }; - publish_blob(&layout, "refs/tags/lfs", &pointer.serialize()).await; + publish_blob( + &layout, + "refs/tags/lfs", + &pointer.serialize(), + PointerCatalog::new(), + ) + .await; assert!( crab_remote::checkpoint::publish_capsule_checkpoint( &layout, @@ -522,3 +545,6 @@ async fn history_verification_counts_and_hashes_reachable_lfs_content() { .is_err() ); } + +#[path = "layered_catalog_tests.rs"] +mod catalog_tests; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index f2e14dcc9..5fd57b510 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -102,7 +102,9 @@ descriptors without a joined catalog in memory. The cursor retains the normalize run until `finish` drains workers and removes it, preserving the operation's primary error if cleanup also fails. A cancelled pending read can resume at the same object. Complete cardinalities remain bound to the proof even after cursor -consumption; sampled physical checks cannot stand in for these counts. Physical page identities, spill handles and the largest recipe still +consumption; sampled physical checks cannot stand in for these counts. The +cursor also retains the checked encoded-byte total of distinct shard/xorb +bodies. This is metadata accounting; physical verification remains required. Physical page identities, spill handles and the largest recipe still consume memory; each packed delta retains its format-bounded decoder. Logical reduction has a live disk ceiling of sixteen times the admitted catalog limit; normal lookup and materialization budgets remain unchanged. diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs index b4eb835bd..0de531ff5 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs @@ -290,14 +290,16 @@ impl Inspector { space: None, rows: Cursor::new(Run::default()), counts: PointerCatalogCounts::default(), + payload_bytes: 0, }); }; - let result: Result<(Run, PointerCatalogCounts)> = async { + let result: Result<(Run, PointerCatalogCounts, u64)> = async { prior?; let run = self.rows.finish(&space).await?; let mut cursor = Cursor::::new(run.clone()); let mut dependencies = Reducer::::new(); let mut counts = PointerCatalogCounts::default(); + let mut payload_bytes = 0_u64; while let Some(row) = cursor.next(&space).await? { let identity = &row.key[1..]; let count = match row.key[0] { @@ -309,6 +311,16 @@ impl Inspector { *count = count .checked_add(1) .ok_or_else(|| corrupt("catalog count overflows"))?; + if row.key[0] != 0 { + let size = u64::from_be_bytes( + row.value[..8] + .try_into() + .map_err(|_| corrupt("catalog payload size is truncated"))?, + ); + payload_bytes = payload_bytes + .checked_add(size) + .ok_or_else(|| corrupt("catalog payload byte count overflows"))?; + } match row.key[0] { 0 => { dependencies @@ -339,14 +351,15 @@ impl Inspector { row.validate()?; } dependencies.remove(&space).await?; - Ok((run, counts)) + Ok((run, counts, payload_bytes)) } .await; match result { - Ok((run, counts)) => Ok(PointerCatalogInspection { + Ok((run, counts, payload_bytes)) => Ok(PointerCatalogInspection { space: Some(space), rows: Cursor::new(run), counts, + payload_bytes, }), Err(error) => finish(Some(space), Err(error)).await, } @@ -383,6 +396,7 @@ pub struct PointerCatalogInspection { space: Option, rows: Cursor, counts: PointerCatalogCounts, + payload_bytes: u64, } impl PointerCatalogInspection { @@ -392,6 +406,15 @@ impl PointerCatalogInspection { self.counts } + /// Return the declared encoded bytes of all distinct shard and xorb bodies. + /// + /// The total comes from the complete normalized capture, independent of + /// iteration progress. It does not certify that those bodies are readable. + #[must_use] + pub fn encoded_payload_bytes(&self) -> u64 { + self.payload_bytes + } + async fn next_entry(&mut self) -> Result> { match &self.space { Some(space) => self.rows.next(space).await, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs index 49afb7a34..9c3037049 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/grouped.rs @@ -80,6 +80,7 @@ impl PointerCatalogInspection { space, rows, counts, + .. } = self; let Some(space) = space else { return Ok(PointerCatalogShardInspection { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs index 7b86f16da..c568f2252 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -316,7 +316,7 @@ async fn cursor_fixture( #[tokio::test] async fn verified_cursor_preserves_complete_descriptors_across_spilled_file_rows() { let (_parent, mut cursor, expected) = cursor_fixture(5000).await; - let before = cursor.counts(); + let before = (cursor.counts(), cursor.encoded_payload_bytes()); let result: Result<_> = async { let mut values = Vec::new(); while let Some(object) = cursor.next_object().await? { @@ -325,7 +325,7 @@ async fn verified_cursor_preserves_complete_descriptors_across_spilled_file_rows Ok(values) } .await; - let after = cursor.counts(); + let after = (cursor.counts(), cursor.encoded_payload_bytes()); let counts = PointerCatalogCounts { files: 5000, shards: 1, @@ -333,7 +333,7 @@ async fn verified_cursor_preserves_complete_descriptors_across_spilled_file_rows }; assert_eq!( (before, after, cursor.finish(result).await.unwrap()), - (counts, counts, expected) + ((counts, 87), (counts, 87), expected) ); } @@ -616,3 +616,19 @@ async fn grouping_disk_admission_failure_removes_the_workspace() { )); assert_eq!(std::fs::read_dir(parent.path()).unwrap().count(), 0); } + +#[tokio::test] +async fn complete_catalog_payload_accounting_rejects_overflow() { + let mut catalog = PointerCatalog::new(); + for (identity, bytes) in [(1, u64::MAX), (2, 1)] { + catalog + .insert_shard( + format!("{identity:064x}"), + ShardCatalogEntry::new(bytes, Vec::new()), + ) + .unwrap(); + } + assert!(matches!(inspect(vec![catalog]).await, + Err(MetadataError::CorruptObject { reason, .. }) + if reason == "catalog payload byte count overflows")); +} diff --git a/crates/crab-read/AGENTS.md b/crates/crab-read/AGENTS.md index 3c1bd2afa..057723812 100644 --- a/crates/crab-read/AGENTS.md +++ b/crates/crab-read/AGENTS.md @@ -27,7 +27,7 @@ For object download changes continue into `crates/crab-read/src/store_client.rs` | --- | --- | --- | | Hydration or range integrity | `crates/crab-read/src/hydrator.rs` | `crates/crab-vfs/src/hydration.rs` | | Fetch authorization | `crates/crab-read/src/fetch_admission.rs` | `crab/src/git/remote_helper.rs` | -| Deep dependency proof | `crates/crab-read/src/capsule_protocol/dependency.rs` | CLI metadata diagnosis, recovery and HTTP integrity | +| Deep dependency proof | `crates/crab-read/src/capsule_protocol/dependency.rs` | CLI metadata diagnosis, historical checkpoint verification, recovery and HTTP integrity | | Replica readiness | `crates/crab-read/src/selection.rs` | `crab/src/replication/mod.rs` | | Term resolution | `crates/crab-read/src/term_resolver.rs` | `crates/crab-metadata/src/file_index_lookup.rs` | diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index 472375a21..def45b954 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -510,3 +510,10 @@ with `verify_captured_installed_pointer_bytes`, the same canonical Git scan and whole-file/LFS byte phase. That API validates selected dependency closures only; it does not certify unrelated catalog health. Complete fsck must first finish its full catalog/object proof against the same captured sources. + +Historical verification uses `verify_installed_checkpoint_dependencies` after +source authentication, pack installation and ref-visibility validation. It +streams the full checkpoint catalog, resolves reachable file bindings from +that same immutable checkpoint and shares the Git/whole-file/LFS byte proof. +Proof counts include the encoded bytes of distinct verified shard/xorb bodies; +logical recipe occurrences and repeated references do not increase that total. diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index 3c7475a54..b731abb70 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -25,8 +25,8 @@ use crate::{ReadError, Result}; mod dependency; pub use dependency::{ - verify_captured_installed_pointer_bytes, verify_installed_dependencies, - verify_reachable_dependencies, + verify_captured_installed_pointer_bytes, verify_installed_checkpoint_dependencies, + verify_installed_dependencies, verify_reachable_dependencies, }; #[cfg(test)] @@ -74,6 +74,8 @@ pub struct CapsuleDependencyProof { pub catalog_xorbs: u64, /// Logical shard/xorb body reads for catalog verification, excluding recipe reads and retries. pub catalog_objects_read: u64, + /// Encoded bytes of all distinct, verified catalog shard and xorb bodies. + pub catalog_payload_bytes: u64, /// Reachable Crab pointer blobs whose complete file bytes were verified. pub reachable_crab_pointers: u64, /// Distinct reachable LFS bodies read and hash-verified from origin. diff --git a/crates/crab-read/src/capsule_protocol/dependency.rs b/crates/crab-read/src/capsule_protocol/dependency.rs index d0470a175..b78a7ba0a 100644 --- a/crates/crab-read/src/capsule_protocol/dependency.rs +++ b/crates/crab-read/src/capsule_protocol/dependency.rs @@ -12,14 +12,18 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use tokio_util::sync::CancellationToken; -// Recovery owns an already assembled replacement catalog; captured views own -// authenticated lookup roots. Both feed the same Git scan and file-byte proof. +// Recovery owns assembled catalogs; current and historical captures own +// authenticated lookup roots. All feed the same Git scan and file-byte proof. enum DependencyCatalog<'a> { Assembled(&'a PointerCatalog), Captured { view: &'a CapsuleRepositoryView, lookup: &'a PointerCatalogLookup, }, + Checkpoint { + checkpoint: &'a LayeredCheckpoint, + lookup: &'a PointerCatalogLookup, + }, } impl DependencyCatalog<'_> { @@ -27,7 +31,7 @@ impl DependencyCatalog<'_> { &self, layout: &StoreLayout, cancel: &CancellationToken, - ) -> Result<(PointerCatalogCounts, crate::ReadinessProbeStats)> { + ) -> Result<(PointerCatalogCounts, crate::ReadinessProbeStats, u64)> { match self { Self::Assembled(catalog) => { let stats = tokio::select! { @@ -42,6 +46,15 @@ impl DependencyCatalog<'_> { xorbs: catalog.xorbs().len() as u64, }, stats, + catalog + .shards() + .values() + .map(|entry| entry.encoded_size()) + .chain(catalog.xorbs().values().map(|entry| entry.encoded_size())) + .try_fold(0_u64, |total, bytes| total.checked_add(bytes)) + .ok_or_else(|| { + ReadError::internal("catalog payload byte count overflowed") + })?, )) } Self::Captured { view, .. } => { @@ -49,6 +62,18 @@ impl DependencyCatalog<'_> { crate::selection::verify_capsule_pointer_inspection(layout, inspection, cancel) .await } + Self::Checkpoint { checkpoint, .. } => { + let inspection = crab_metadata::capsule_protocol::inspect_pointer_catalog( + layout, + checkpoint.catalog(), + &[], + &[], + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await?; + crate::selection::verify_capsule_pointer_inspection(layout, inspection, cancel) + .await + } } } @@ -67,7 +92,7 @@ impl DependencyCatalog<'_> { .map(|entry| (hash.clone(), entry.clone())) }) .collect()), - Self::Captured { lookup, .. } => { + Self::Captured { lookup, .. } | Self::Checkpoint { lookup, .. } => { let selected = lookup.files(layout.store(), hashes).await?; Ok(selected.files().clone()) } @@ -212,6 +237,39 @@ pub async fn verify_installed_dependencies( .await } +/// Verify installed Git and external dependencies against one pinned checkpoint. +/// +/// Callers authenticate the checkpoint, install its verified packs, validate +/// the supplied ref visibility and retain its sources through completion. +/// The complete catalog is streamed; file bindings use that same checkpoint. +/// Token cancellation drains the Git worker before returning. +pub async fn verify_installed_checkpoint_dependencies( + layout: &StoreLayout, + git_dir: &Path, + refs: &BTreeMap, + checkpoint: &LayeredCheckpoint, + limits: crab_git::walk::PointerScanLimits, + cancellation: &CancellationToken, +) -> Result { + if cancellation.is_cancelled() { + return Err(ReadError::Cancelled); + } + let lookup = checkpoint + .pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; + verify_installed_dependencies_from( + layout, + git_dir, + refs, + DependencyCatalog::Checkpoint { + checkpoint, + lookup: &lookup, + }, + limits, + cancellation, + ) + .await +} + async fn verify_installed_dependencies_from( layout: &StoreLayout, git_dir: &Path, @@ -220,7 +278,7 @@ async fn verify_installed_dependencies_from( limits: crab_git::walk::PointerScanLimits, cancellation: &CancellationToken, ) -> Result { - let (counts, catalog_stats) = catalog.verify(layout, cancellation).await?; + let (counts, catalog_stats, payload_bytes) = catalog.verify(layout, cancellation).await?; let proof = verify_installed_pointer_bytes_from(layout, git_dir, refs, catalog, limits, cancellation) .await?; @@ -229,6 +287,7 @@ async fn verify_installed_dependencies_from( catalog_shards: counts.shards, catalog_xorbs: counts.xorbs, catalog_objects_read: catalog_stats.object_read_count, + catalog_payload_bytes: payload_bytes, reachable_crab_pointers: proof.crab, reachable_lfs_objects: proof.lfs_objects, reachable_lfs_bytes: proof.lfs_bytes, diff --git a/crates/crab-read/src/capsule_protocol/dependency_tests.rs b/crates/crab-read/src/capsule_protocol/dependency_tests.rs index 878ed7ae3..d244bdcc7 100644 --- a/crates/crab-read/src/capsule_protocol/dependency_tests.rs +++ b/crates/crab-read/src/capsule_protocol/dependency_tests.rs @@ -271,6 +271,16 @@ async fn installed_dependency_proof_accepts_reordered_repeated_file_chunks() { catalog_shards: 1, catalog_xorbs: 1, catalog_objects_read: 2, + catalog_payload_bytes: catalog + .shards() + .values() + .map(|entry| entry.encoded_size()) + .sum::() + + catalog + .xorbs() + .values() + .map(|entry| entry.encoded_size()) + .sum::(), reachable_crab_pointers: 1, reachable_lfs_objects: 0, reachable_lfs_bytes: 0 diff --git a/crates/crab-read/src/selection.rs b/crates/crab-read/src/selection.rs index c8ffae8ad..ade82b192 100644 --- a/crates/crab-read/src/selection.rs +++ b/crates/crab-read/src/selection.rs @@ -528,8 +528,9 @@ pub(crate) async fn verify_capsule_pointer_inspection( router: &StoreLayout, mut catalog: PointerCatalogInspection, cancel: &tokio_util::sync::CancellationToken, -) -> Result<(PointerCatalogCounts, ReadinessProbeStats)> { +) -> Result<(PointerCatalogCounts, ReadinessProbeStats, u64)> { let counts = catalog.counts(); + let payload_bytes = catalog.encoded_payload_bytes(); let mut stats = ReadinessProbeStats::default(); let result = tokio::select! { biased; @@ -543,7 +544,7 @@ pub(crate) async fn verify_capsule_pointer_inspection( reason, }); } - Ok((counts, stats)) + Ok((counts, stats, payload_bytes)) } => result, }; catalog.finish(result).await From 11d08837bc408383a2e57aef984eef432a6db58b Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 07:12:07 -0700 Subject: [PATCH 126/127] perf(metadata): reuse captured catalog proofs --- .../docs/architecture/indexed-remote-reads.md | 10 + crab/src/cmd/history_recovery_v2.rs | 12 +- crab/src/cmd/metadb.rs | 98 ++------- crab/src/cmd/metadb/capsule_lookup_tests.rs | 198 ++++++++++++++++++ crab/src/metadata/shard_sync.rs | 4 +- .../src/capsule_protocol/catalog_capture.rs | 30 ++- .../src/capsule_protocol/layered/catalog.rs | 30 +++ .../crab-metadata/src/capsule_protocol/mod.rs | 4 +- .../capsule_protocol/pointer/inspection.rs | 19 +- .../pointer/inspection/tests.rs | 2 +- .../pointer/object/reader/inspection.rs | 18 +- .../capsule_protocol/pointer/object/tests.rs | 6 +- crates/crab-read/src/capsule_protocol.rs | 5 + .../src/capsule_protocol/dependency.rs | 27 ++- .../src/capsule_protocol/dependency_tests.rs | 2 + crates/crab-read/src/selection.rs | 13 +- 16 files changed, 361 insertions(+), 117 deletions(-) create mode 100644 crab/src/cmd/metadb/capsule_lookup_tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index ad07873cf..8cdee2510 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2563,6 +2563,16 @@ phases and prevents an unrelated phase from reporting success before full proof. Ordinary GC and replica readiness do not perform shard grouping. Historical verification uses the same streamed complete catalog/body proof plus checkpoint- bound file lookups, preserving checked dependency counts and byte accounting. +The complete inspection also retains authenticated checkpoint catalog object/byte +counts. Historical accounting consumes those totals instead of repeating the +physical catalog walk. Empty durable catalog descriptors remain counted. +Recovery verifies selected file-index mappings in batches of 64 against one +captured lookup, including each selected dependency closure. Unrelated catalog +corruption is outside that selected check; complete inspection remains the +integrity boundary. Post-fetch shard discovery streams complete inspection from +the same metadata control-capture owner, retaining shard identities instead of +all file and xorb recipes. Per-ref head movement still disables generation-only +shard-list caching. Restore's catalog merge and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to protect those callers retaining recipe copies. diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index 9d40a7911..e08f7f58c 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -220,9 +220,8 @@ async fn verify_history( check_cancelled(cancel)?; let shard_count = dependencies.catalog_shards; let xorb_count = dependencies.catalog_xorbs; - let catalog_objects = checkpoint.catalog_objects(layout, u64::MAX).await?; let dependency_objects = 2_u64 - .checked_add(catalog_objects.len() as u64) + .checked_add(dependencies.checkpoint_catalog_objects) .and_then(|total| total.checked_add(source_count)) .and_then(|total| total.checked_add(shard_count)) .and_then(|total| total.checked_add(xorb_count)) @@ -230,11 +229,10 @@ async fn verify_history( .ok_or_else(|| CrabError::Internal("history dependency count overflowed".to_owned()))?; let segment_bytes = u64::try_from(segment.bytes().len()) .map_err(|_| CrabError::Internal("history segment size overflowed".to_owned()))?; - let checkpoint_bytes = catalog_objects - .iter() - .try_fold(checkpoint.pointer()?.size(), |total, (_, bytes)| { - total.checked_add(*bytes) - }) + let checkpoint_bytes = checkpoint + .pointer()? + .size() + .checked_add(dependencies.checkpoint_catalog_bytes) .ok_or_else(|| CrabError::Internal("history catalog byte count overflowed".to_owned()))?; let dependency_bytes = segment_bytes .checked_add(checkpoint_bytes) diff --git a/crab/src/cmd/metadb.rs b/crab/src/cmd/metadb.rs index 9aa9bf79d..05ae32c4f 100644 --- a/crab/src/cmd/metadb.rs +++ b/crab/src/cmd/metadb.rs @@ -2839,34 +2839,23 @@ async fn verify_capsule_file_index( root: crab_metadata::capsule_protocol::RootSnapshot, entries: &[(MerkleHash, MerkleHash)], ) -> Result> { - let view = crab_read::capsule_protocol::open_view_from_root( - layout, - root, - crab_read::capsule_protocol::CapsuleReadLimits { - max_capsule_bytes: CAPSULE_OWNER_MAX_CHECKPOINT_BYTES, - max_frontier_bytes: CAPSULE_OWNER_MAX_CHECKPOINT_BYTES, - }, - ) - .await?; - Ok(capsule_file_index_matches( - &view.pointer_catalog().await?, - entries, - )) -} - -fn capsule_file_index_matches( - catalog: &crab_metadata::capsule_protocol::PointerCatalog, - entries: &[(MerkleHash, MerkleHash)], -) -> Vec { - entries - .iter() - .map(|(file_hash, expected_shard)| { + // Recovery verifies selected mappings, not global payload health. Capture + // once so every batch sees the same file-binding precedence during pushes. + let lookup = + crab_metadata::capsule_protocol::load_pointer_catalog_lookup_from_root(layout, &root) + .await?; + let mut verified = Vec::with_capacity(entries.len()); + for batch in entries.chunks(64) { + let hashes = batch.iter().map(|(file, _)| file.hex()).collect::>(); + let catalog = lookup.files(layout.store(), &hashes).await?; + verified.extend(batch.iter().map(|(file, expected)| { catalog .files() - .get(&file_hash.hex()) - .is_some_and(|entry| entry.shard_hash() == expected_shard.hex()) - }) - .collect() + .get(&file.hex()) + .is_some_and(|entry| entry.shard_hash() == expected.hex()) + })); + } + Ok(verified) } async fn close_rebuild_guard(guard: MetaDbGuard, result: Result) -> Result { @@ -5071,59 +5060,6 @@ mod tests { assert!(legacy_objects.is_empty()); } - #[test] - fn capsule_file_index_verification_uses_authenticated_mapping() { - let file_hash = MerkleHash::from([1; 32]); - let expected_shard = MerkleHash::from([2; 32]); - let other_shard = MerkleHash::from([3; 32]); - let mut catalog = crab_metadata::capsule_protocol::PointerCatalog::new(); - catalog - .insert_file( - file_hash.hex(), - crab_metadata::capsule_protocol::FileCatalogEntry::new(42, expected_shard.hex()), - ) - .expect("insert file catalog entry"); - - assert_eq!( - capsule_file_index_matches( - &catalog, - &[ - (file_hash, expected_shard), - (file_hash, other_shard), - (MerkleHash::from([4; 32]), expected_shard), - ], - ), - vec![true, false, false] - ); - } - - #[tokio::test] - async fn capsule_file_index_verification_does_not_create_legacy_metadata() { - let inner: Arc = Arc::new(InMemory::new()); - let storage = crab_storage::Store::new(Arc::clone(&inner)); - let layout = crab_storage::StoreLayout::new(storage, "org/v2-recover".to_owned()); - let root = - crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") - .await - .expect("initialize capsule root"); - - let verified = verify_capsule_file_index( - &layout, - root, - &[(MerkleHash::from([1; 32]), MerkleHash::from([2; 32]))], - ) - .await - .expect("verify capsule catalog"); - - assert_eq!(verified, vec![false]); - let legacy_objects = inner - .list(Some(&ObjectPath::from("org/v2-recover/file_index_db"))) - .try_collect::>() - .await - .expect("list legacy metadata prefix"); - assert!(legacy_objects.is_empty()); - } - #[tokio::test] async fn continuous_owner_skips_completed_repository_identity() { let inner: Arc = Arc::new(InMemory::new()); @@ -5854,3 +5790,7 @@ mod tests { guard.close().await.expect("close"); } } + +#[cfg(test)] +#[path = "metadb/capsule_lookup_tests.rs"] +mod capsule_lookup_tests; diff --git a/crab/src/cmd/metadb/capsule_lookup_tests.rs b/crab/src/cmd/metadb/capsule_lookup_tests.rs new file mode 100644 index 000000000..4dce4bdd9 --- /dev/null +++ b/crab/src/cmd/metadb/capsule_lookup_tests.rs @@ -0,0 +1,198 @@ +#![expect(clippy::expect_used, clippy::unwrap_used, reason = "test assertions")] +use super::*; +use crab_metadata::capsule_protocol::{ + Capsule, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, CapsuleTransaction, + FileCatalogEntry, PointerCatalog, ShardCatalogEntry, +}; +use object_store::memory::InMemory; + +#[tokio::test] +async fn capsule_file_index_verification_does_not_create_legacy_metadata() { + let inner: Arc = Arc::new(InMemory::new()); + let storage = crab_storage::Store::new(Arc::clone(&inner)); + let layout = crab_storage::StoreLayout::new(storage, "org/v2-recover".to_owned()); + let root = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .expect("initialize capsule root"); + + let verified = verify_capsule_file_index( + &layout, + root, + &[(MerkleHash::from([1; 32]), MerkleHash::from([2; 32]))], + ) + .await + .expect("verify capsule catalog"); + + assert_eq!(verified, vec![false]); + let legacy_objects = inner + .list(Some(&ObjectPath::from("org/v2-recover/file_index_db"))) + .try_collect::>() + .await + .expect("list legacy metadata prefix"); + assert!(legacy_objects.is_empty()); +} + +async fn fixture( + missing_shard: bool, + unrelated: u32, + broken_unrelated: bool, +) -> ( + crab_storage::StoreLayout, + crab_metadata::capsule_protocol::RootSnapshot, +) { + let layout = crab_storage::StoreLayout::new( + crab_storage::Store::new(Arc::new(InMemory::new())), + "selected-recovery".to_owned(), + ); + let base = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let mut catalog = PointerCatalog::new(); + catalog + .insert_file( + MerkleHash::from([1; 32]).hex(), + FileCatalogEntry::new(42, MerkleHash::from([2; 32]).hex()), + ) + .unwrap(); + if !missing_shard { + catalog + .insert_shard( + MerkleHash::from([2; 32]).hex(), + ShardCatalogEntry::new(7, Vec::new()), + ) + .unwrap(); + } + for index in 0..unrelated { + catalog + .insert_file( + format!("{index:064x}"), + FileCatalogEntry::new(42, MerkleHash::from([2; 32]).hex()), + ) + .unwrap(); + } + if broken_unrelated { + catalog + .insert_file( + MerkleHash::from([5; 32]).hex(), + FileCatalogEntry::new(42, MerkleHash::from([6; 32]).hex()), + ) + .unwrap(); + } + let transaction = CapsuleTransaction::new( + base.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build( + &transaction, + Vec::new(), + vec![CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + catalog.encode_delta().unwrap(), + )], + ) + .unwrap(); + crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) + .await + .unwrap(); + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + (layout, root) +} + +#[tokio::test] +async fn selected_mapping_verification_preserves_order_and_duplicates_across_batches() { + let (layout, root) = fixture(false, 0, false).await; + let cases = [([1; 32], [2; 32]), ([1; 32], [3; 32]), ([4; 32], [2; 32])]; + let entries = (0..129) + .map(|index| { + let (file, shard) = cases[index % cases.len()]; + (MerkleHash::from(file), MerkleHash::from(shard)) + }) + .collect::>(); + assert_eq!( + verify_capsule_file_index(&layout, root, &entries) + .await + .unwrap(), + (0..129).map(|index| index % 3 == 0).collect::>() + ); +} + +#[tokio::test] +async fn selected_mapping_missing_dependency_is_an_error() { + let (layout, root) = fixture(true, 0, false).await; + assert!( + verify_capsule_file_index( + &layout, + root, + &[(MerkleHash::from([1; 32]), MerkleHash::from([2; 32]))] + ) + .await + .is_err() + ); +} + +#[tokio::test] +async fn selected_mapping_does_not_certify_unrelated_catalog_dependencies() { + let (layout, root) = fixture(false, 0, true).await; + assert_eq!( + verify_capsule_file_index( + &layout, + root.clone(), + &[(MerkleHash::from([1; 32]), MerkleHash::from([2; 32]))] + ) + .await + .unwrap(), + vec![true] + ); + assert!( + crab_metadata::capsule_protocol::inspect_pointer_catalog_from_root(&layout, &root) + .await + .is_err() + ); +} + +#[derive(Default)] +struct ReadBytes(std::sync::atomic::AtomicU64); +impl crab_storage::StorageObserver for ReadBytes { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, observation: crab_storage::StorageObservation) { + self.0 + .fetch_add(observation.bytes_read, std::sync::atomic::Ordering::Relaxed); + } +} + +#[tokio::test] +async fn fixed_recovery_query_reads_pages_as_unrelated_catalog_grows() { + for unrelated in [1, 10_000, 100_000] { + let (layout, root) = fixture(false, unrelated, false).await; + let reads = Arc::new(ReadBytes::default()); + let observed = crab_storage::StoreLayout::new( + layout.store().clone().with_storage_observer(reads.clone()), + layout.repo_prefix().to_owned(), + ); + assert_eq!( + verify_capsule_file_index( + &observed, + root, + &[(MerkleHash::from([1; 32]), MerkleHash::from([2; 32]))] + ) + .await + .unwrap(), + vec![true] + ); + let bytes = reads.0.load(std::sync::atomic::Ordering::Relaxed); + assert!( + bytes < 512 * 1024, + "{unrelated} unrelated mappings read {bytes} bytes" + ); + } +} diff --git a/crab/src/metadata/shard_sync.rs b/crab/src/metadata/shard_sync.rs index 8cc95f12d..adde21ac7 100644 --- a/crab/src/metadata/shard_sync.rs +++ b/crab/src/metadata/shard_sync.rs @@ -1172,14 +1172,14 @@ async fn load_published_shards(router: &StoreLayout) -> Result match crab_metadata::capsule_protocol::load_root(&capsule_layout).await { Ok(root) => { let generation = root.record().root().generation(); - let catalog = crab_metadata::capsule_protocol::load_pointer_catalog_from_root( + let inspection = crab_metadata::capsule_protocol::inspect_pointer_catalog_from_root( &capsule_layout, &root, ) .await?; Ok(PublishedShards { generation, - entries: catalog.shards().keys().cloned().collect(), + entries: inspection.into_shard_hashes().await?, // Per-ref heads can advance without changing the compacted root // generation, so this value cannot key the local generation cache. cacheable_generation: false, diff --git a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs index 6968b4c8c..584fdc0ab 100644 --- a/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs +++ b/crates/crab-metadata/src/capsule_protocol/catalog_capture.rs @@ -2,10 +2,10 @@ use super::pointer::paged::MAX_CATALOG_BYTES; use super::{ CapsuleControl, CapturedVisibility, LayeredCheckpoint, PointerCatalog, PointerCatalogAssembly, - PointerCatalogLookup, RootSnapshot, RunCatalogSource, VisibilityReconciliation, - apply_control_refs, capture_ref_heads, load_capsule_run_git_control, load_layered_checkpoint, - load_layered_checkpoint_control, materialize_visible_ref_heads, order_ref_controls, - select_ref_control_frontiers, select_ref_journal_controls, + PointerCatalogInspection, PointerCatalogLookup, RootSnapshot, RunCatalogSource, + VisibilityReconciliation, apply_control_refs, capture_ref_heads, load_capsule_run_git_control, + load_layered_checkpoint, load_layered_checkpoint_control, materialize_visible_ref_heads, + order_ref_controls, select_ref_control_frontiers, select_ref_journal_controls, }; use crate::error::{MetadataError, Result}; use crate::git_visibility::GitVisibilityIndex; @@ -25,6 +25,28 @@ pub async fn load_pointer_catalog_lookup_from_root( capture(layout, root).await?.lookup(layout) } +/// Inspect the complete catalog from one authenticated current-state control capture. +/// +/// The caller retains captured roots against GC and must finish the returned +/// spill cursor. Logical rows are normalized without retaining the recipe map. +pub async fn inspect_pointer_catalog_from_root( + layout: &StoreLayout, + root: &RootSnapshot, +) -> Result { + let captured = capture(layout, root).await?; + super::inspect_pointer_catalog( + layout, + captured + .checkpoint + .as_ref() + .and_then(LayeredCheckpoint::catalog), + &[], + &captured.controls, + MAX_CATALOG_BYTES, + ) + .await +} + pub(super) async fn materialize_from_root( layout: &StoreLayout, root: &RootSnapshot, diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs index 99663da35..7196d5298 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs @@ -331,6 +331,36 @@ mod tests { (layout, checkpoint, catalog) } + #[tokio::test] + async fn inspection_retains_authenticated_inventory_for_empty_and_nonempty_catalogs() { + for catalog in [PointerCatalog::new(), fixture().2] { + let (layout, _, _) = fixture(); + let (pages, objects) = Descriptor::build(&catalog).unwrap(); + let source = CheckpointCatalog::new(pages).unwrap(); + publish_objects(&layout, &objects).await.unwrap(); + let expected = ( + objects.len() as u64, + objects.values().map(|body| body.len() as u64).sum(), + ); + let mut inspection = crate::capsule_protocol::inspect_pointer_catalog( + &layout, + Some(&source), + &[], + &[], + crate::capsule_protocol::MAX_CATALOG_BYTES, + ) + .await + .unwrap(); + while inspection.next_object().await.unwrap().is_some() {} + let actual = inspection.checkpoint_catalog_footprint(); + inspection + .finish::<_, crate::error::MetadataError>(Ok(())) + .await + .unwrap(); + assert_eq!(actual, expected); + } + } + #[tokio::test] async fn cold_checkpoint_load_admits_catalog_before_reading_and_verifies_its_body() { let (layout, checkpoint, catalog) = fixture(); diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 51360bab1..0cdcff2a6 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -49,7 +49,9 @@ pub use captured_visibility::{ visibility_is_ready, }; #[cfg(feature = "storage")] -pub use catalog_capture::load_pointer_catalog_lookup_from_root; +pub use catalog_capture::{ + inspect_pointer_catalog_from_root, load_pointer_catalog_lookup_from_root, +}; pub use history::{ HistorySegment, HistorySegmentPointer, HistorySegmentState, MAX_HISTORY_CHAIN_BYTES, MAX_HISTORY_CHAIN_SEGMENTS, MAX_HISTORY_SEGMENT_BYTES, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs index 0de531ff5..f7b05f339 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection.rs @@ -291,6 +291,7 @@ impl Inspector { rows: Cursor::new(Run::default()), counts: PointerCatalogCounts::default(), payload_bytes: 0, + checkpoint_catalog: (0, 0), }); }; let result: Result<(Run, PointerCatalogCounts, u64)> = async { @@ -360,6 +361,7 @@ impl Inspector { rows: Cursor::new(run), counts, payload_bytes, + checkpoint_catalog: (0, 0), }), Err(error) => finish(Some(space), Err(error)).await, } @@ -397,6 +399,7 @@ pub struct PointerCatalogInspection { rows: Cursor, counts: PointerCatalogCounts, payload_bytes: u64, + checkpoint_catalog: (u64, u64), } impl PointerCatalogInspection { @@ -415,6 +418,15 @@ impl PointerCatalogInspection { self.payload_bytes } + /// Return distinct checkpoint catalog objects and their encoded bytes authenticated by this capture. + /// + /// Captures without a checkpoint catalog return `(0, 0)`. Embedded run + /// catalogs and external shard/xorb bodies are excluded. No I/O occurs. + #[must_use] + pub fn checkpoint_catalog_footprint(&self) -> (u64, u64) { + self.checkpoint_catalog + } + async fn next_entry(&mut self) -> Result> { match &self.space { Some(space) => self.rows.next(space).await, @@ -494,9 +506,10 @@ pub async fn inspect_pointer_catalog( ) -> Result { let maximum = maximum.min(MAX_CATALOG_BYTES); let mut inspector = Inspector::new(maximum); + let mut checkpoint_catalog = (0, 0); let checked = async { if let Some(checkpoint) = checkpoint { - checkpoint + checkpoint_catalog = checkpoint .reader(layout)? .inspect_into(layout.store(), maximum, &mut inspector) .await?; @@ -521,7 +534,9 @@ pub async fn inspect_pointer_catalog( Ok(()) } .await; - inspector.finish(checked).await + let mut inspection = inspector.finish(checked).await?; + inspection.checkpoint_catalog = checkpoint_catalog; + Ok(inspection) } mod grouped; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs index c568f2252..d6c348466 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/inspection/tests.rs @@ -247,7 +247,7 @@ async fn checkpoint_rows_stream_without_per_file_dependency_joins() { let checked = cold.inspect_into(&store, maximum, &mut inspector).await; assert_eq!( inspector - .finish(checked) + .finish(checked.map(|_| ())) .await .unwrap() .into_shard_hashes() diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs index aba3747e2..e3f15fdd2 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs @@ -31,10 +31,18 @@ impl CatalogPages { store: &Store, maximum: u64, output: &mut crate::capsule_protocol::pointer::inspection::Inspector, - ) -> Result<()> { - // The complete physical proof includes reverse placement and alias checks. - // Streaming logical rows must not hide corruption in an unselected column. - self.inspected_objects(store, maximum).await?; + ) -> Result<(u64, u64)> { + // Retain the authenticated inventory totals for accounting. Repeating the + // physical proof after consuming the logical rows doubles origin work. + let footprint = { + let objects = self.inspected_objects(store, maximum).await?; + let bytes = objects.iter().try_fold(0_u64, |total, (_, bytes)| { + total + .checked_add(*bytes) + .ok_or_else(|| corrupt("checkpoint catalog byte count overflows")) + })?; + (objects.len() as u64, bytes) + }; let mut work = PageWork::new(maximum, 0); for kind in [Kind::CatalogFiles, Kind::CatalogShards, Kind::CatalogXorbs] { let mut pending = self.root(kind)?.cloned().into_iter().collect(); @@ -82,7 +90,7 @@ impl CatalogPages { } } } - Ok(()) + Ok(footprint) } async fn inspect(&self, store: &Store, work: &mut PageWork, join: &mut Join) -> Result<()> { diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs index 6500905dc..22e7d59be 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/tests.rs @@ -334,7 +334,7 @@ async fn maintenance_streams_shared_recipes_without_materializing_the_catalog() let checked = cold.inspect_into(&store, maximum, &mut logical).await; assert!( logical - .finish(checked) + .finish(checked.map(|_| ())) .await .unwrap() .into_shard_hashes() @@ -529,7 +529,7 @@ async fn streamed_inspection_rejects_resigned_broken_dependency_edges() { let mut logical = crate::capsule_protocol::pointer::inspection::Inspector::new(1024 * 1024); let checked = forged.inspect_into(&store, 1024 * 1024, &mut logical).await; assert!( - logical.finish(checked).await.is_err(), + logical.finish(checked.map(|_| ())).await.is_err(), "logical: {scenario}" ); } @@ -610,7 +610,7 @@ async fn streamed_inspection_bounds_io_for_distinct_recipes_beyond_the_page_cach let checked = cold.inspect_into(&store, maximum, &mut logical).await; assert!( logical - .finish(checked) + .finish(checked.map(|_| ())) .await .unwrap() .into_shard_hashes() diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index b731abb70..dbc2336f4 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -76,6 +76,11 @@ pub struct CapsuleDependencyProof { pub catalog_objects_read: u64, /// Encoded bytes of all distinct, verified catalog shard and xorb bodies. pub catalog_payload_bytes: u64, + /// Distinct durable checkpoint catalog objects authenticated by this proof. + /// Assembled inputs and captures without such a source contribute zero. + pub checkpoint_catalog_objects: u64, + /// Encoded bytes of those objects, excluding embedded run catalogs and payload bodies. + pub checkpoint_catalog_bytes: u64, /// Reachable Crab pointer blobs whose complete file bytes were verified. pub reachable_crab_pointers: u64, /// Distinct reachable LFS bodies read and hash-verified from origin. diff --git a/crates/crab-read/src/capsule_protocol/dependency.rs b/crates/crab-read/src/capsule_protocol/dependency.rs index b78a7ba0a..d7316d699 100644 --- a/crates/crab-read/src/capsule_protocol/dependency.rs +++ b/crates/crab-read/src/capsule_protocol/dependency.rs @@ -31,7 +31,7 @@ impl DependencyCatalog<'_> { &self, layout: &StoreLayout, cancel: &CancellationToken, - ) -> Result<(PointerCatalogCounts, crate::ReadinessProbeStats, u64)> { + ) -> Result { match self { Self::Assembled(catalog) => { let stats = tokio::select! { @@ -39,14 +39,14 @@ impl DependencyCatalog<'_> { () = cancel.cancelled() => return Err(ReadError::Cancelled), result = crate::verify_capsule_pointer_catalog_objects(layout, catalog) => result?, }; - Ok(( - PointerCatalogCounts { + Ok(crate::selection::CatalogVerification { + counts: PointerCatalogCounts { files: catalog.files().len() as u64, shards: catalog.shards().len() as u64, xorbs: catalog.xorbs().len() as u64, }, stats, - catalog + payload_bytes: catalog .shards() .values() .map(|entry| entry.encoded_size()) @@ -55,7 +55,10 @@ impl DependencyCatalog<'_> { .ok_or_else(|| { ReadError::internal("catalog payload byte count overflowed") })?, - )) + // Assembled input authenticates no durable checkpoint catalog. + checkpoint_objects: 0, + checkpoint_bytes: 0, + }) } Self::Captured { view, .. } => { let inspection = view.pointer_catalog_inspection().await?; @@ -278,16 +281,18 @@ async fn verify_installed_dependencies_from( limits: crab_git::walk::PointerScanLimits, cancellation: &CancellationToken, ) -> Result { - let (counts, catalog_stats, payload_bytes) = catalog.verify(layout, cancellation).await?; + let verified = catalog.verify(layout, cancellation).await?; let proof = verify_installed_pointer_bytes_from(layout, git_dir, refs, catalog, limits, cancellation) .await?; Ok(CapsuleDependencyProof { - catalog_files: counts.files, - catalog_shards: counts.shards, - catalog_xorbs: counts.xorbs, - catalog_objects_read: catalog_stats.object_read_count, - catalog_payload_bytes: payload_bytes, + catalog_files: verified.counts.files, + catalog_shards: verified.counts.shards, + catalog_xorbs: verified.counts.xorbs, + catalog_objects_read: verified.stats.object_read_count, + catalog_payload_bytes: verified.payload_bytes, + checkpoint_catalog_objects: verified.checkpoint_objects, + checkpoint_catalog_bytes: verified.checkpoint_bytes, reachable_crab_pointers: proof.crab, reachable_lfs_objects: proof.lfs_objects, reachable_lfs_bytes: proof.lfs_bytes, diff --git a/crates/crab-read/src/capsule_protocol/dependency_tests.rs b/crates/crab-read/src/capsule_protocol/dependency_tests.rs index d244bdcc7..95e76f2ea 100644 --- a/crates/crab-read/src/capsule_protocol/dependency_tests.rs +++ b/crates/crab-read/src/capsule_protocol/dependency_tests.rs @@ -281,6 +281,8 @@ async fn installed_dependency_proof_accepts_reordered_repeated_file_chunks() { .values() .map(|entry| entry.encoded_size()) .sum::(), + checkpoint_catalog_objects: 0, + checkpoint_catalog_bytes: 0, reachable_crab_pointers: 1, reachable_lfs_objects: 0, reachable_lfs_bytes: 0 diff --git a/crates/crab-read/src/selection.rs b/crates/crab-read/src/selection.rs index ade82b192..fc75394a3 100644 --- a/crates/crab-read/src/selection.rs +++ b/crates/crab-read/src/selection.rs @@ -524,13 +524,22 @@ pub async fn verify_capsule_pointer_catalog_objects( Ok(stats) } +pub(crate) struct CatalogVerification { + pub counts: PointerCatalogCounts, + pub stats: ReadinessProbeStats, + pub payload_bytes: u64, + pub checkpoint_objects: u64, + pub checkpoint_bytes: u64, +} + pub(crate) async fn verify_capsule_pointer_inspection( router: &StoreLayout, mut catalog: PointerCatalogInspection, cancel: &tokio_util::sync::CancellationToken, -) -> Result<(PointerCatalogCounts, ReadinessProbeStats, u64)> { +) -> Result { let counts = catalog.counts(); let payload_bytes = catalog.encoded_payload_bytes(); + let (checkpoint_objects, checkpoint_bytes) = catalog.checkpoint_catalog_footprint(); let mut stats = ReadinessProbeStats::default(); let result = tokio::select! { biased; @@ -544,7 +553,7 @@ pub(crate) async fn verify_capsule_pointer_inspection( reason, }); } - Ok((counts, stats, payload_bytes)) + Ok(CatalogVerification { counts, stats, payload_bytes, checkpoint_objects, checkpoint_bytes }) } => result, }; catalog.finish(result).await From 3b79f76e3421cf14535f4a0b04040d0210e90898 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 16 Oct 2026 09:03:28 -0700 Subject: [PATCH 127/127] fix(history): verify paged restore unions Union durable catalog roots with shared admission and reusable recipe digests. Prove winning recipes before restoring refs; an old binding proof cannot certify a current replacement. Authenticate complete stored shards before stripping optional bloom trailers for the native Xet compaction loader. --- .../docs/architecture/indexed-remote-reads.md | 24 +- crab/src/cmd/compact.rs | 95 +++++- crab/src/cmd/history_recovery_v2.rs | 55 ++-- .../layered_catalog_tests.rs | 166 ++++++++++- .../cmd/history_recovery_v2/layered_tests.rs | 4 +- crates/crab-metadata/README.md | 6 + .../src/capsule_protocol/layered.rs | 4 +- .../src/capsule_protocol/layered/catalog.rs | 60 +++- .../layered/catalog/merge_tests.rs | 280 ++++++++++++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 2 +- .../capsule_protocol/pointer/object/reader.rs | 1 + .../pointer/object/reader/inspection.rs | 22 +- .../pointer/object/reader/merge.rs | 193 ++++++++++++ .../pointer/object/reader/merge/tests.rs | 164 ++++++++++ crates/crab-read/README.md | 6 + crates/crab-read/src/capsule_protocol.rs | 2 +- .../src/capsule_protocol/dependency.rs | 35 ++- crates/crab-remote/README.md | 7 + crates/crab-remote/src/checkpoint.rs | 67 ++++- 19 files changed, 1111 insertions(+), 82 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/layered/catalog/merge_tests.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge/tests.rs diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 8cdee2510..767090238 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2573,8 +2573,28 @@ integrity boundary. Post-fetch shard discovery streams complete inspection from the same metadata control-capture owner, retaining shard identities instead of all file and xorb recipes. Per-ref head movement still disables generation-only shard-list caching. -Restore's catalog merge and catalog-rewriting tools retain full logical catalogs. Materialization admission continues to -protect those callers retaining recipe copies. +Shard compaction authenticates the complete stored shard, including its optional +bloom trailer, before passing only the native Xet body to the private merge +loader. Xet locates its footer relative to EOF; feeding the bloom bytes to that +loader rejects valid native-push shards. The merge input's local content hash +may differ from the complete stored object's identity; source authentication +and retained-root ownership continue to use the complete object. +Restore unions durable catalogs by streaming their authenticated columns through +copy-on-write edits. Both input closures and every edit share one encoded-page +budget; unchanged subtrees keep their identities. Newer file bindings win only +when declared sizes agree; shard/xorb descriptor conflicts remain errors. Reverse +placements retain the lowest xorb and its first occurrence across both inputs. +Immutable nested recipes are compared by framed ordered-row digests, independent +of page partitioning. Digests are retained only for this merge and cached after +complete scans; each distinct root pays logical-work admission before I/O. +Before checkpoint or ref visibility changes, restore verifies historical Git +pointers against the final winning union bindings and hashes their complete +bytes at origin. Verifying only the old checkpoint would miss a broken newer +replacement recipe. The merge does not retain either full recipe map; complete +metadata proof and the largest selected recipe still consume resources. Current +uncheckpointed deltas retain their bounded materializer, and other catalog-rewrite +consumers remain follow-up work. Materialization admission continues to protect +callers retaining recipe copies. Release audit: all six upstream published tag objects were checked again before this change; none contains the capsule codec. Unshipped checkpoint formats diff --git a/crab/src/cmd/compact.rs b/crab/src/cmd/compact.rs index 79a406ad3..56fb0e2a1 100644 --- a/crab/src/cmd/compact.rs +++ b/crab/src/cmd/compact.rs @@ -596,8 +596,9 @@ async fn download_shards( }); } - // Write shard bytes to the temp directory via MDBShardFile. - let mut cursor = std::io::Cursor::new(data.as_ref()); + // Authenticate the complete stored object above. Xet seeks its footer + // from EOF, so private merge inputs must omit Crab's bloom trailer. + let mut cursor = std::io::Cursor::new(crab_xet::shard_parse::strip_bloom_trailer(&data)); MDBShardFile::write_out_from_reader(target_dir, &mut cursor, &shard_file_cache).map_err( |e| CrabError::Internal(format!("failed to write shard {hash_hex} to temp dir: {e}")), )?; @@ -964,7 +965,7 @@ mod tests { shard_body: Bytes, } - fn xet_file_shard(byte: u8) -> XetShardFixture { + fn xet_file_shard(byte: u8, bloom: bool) -> XetShardFixture { use crab_xet::shard::{ FileDataSequenceEntry, FileDataSequenceHeader, MDBFileInfo, MDBXorbInfo, ShardWriter, XorbChunkSequenceEntry, XorbChunkSequenceHeader, @@ -1001,7 +1002,13 @@ mod tests { metadata_ext: None, }) .unwrap(); - let (bytes, hash) = writer.finalize().unwrap(); + let (bytes, hash) = if bloom { + writer + .finalize_with_bloom(&[chunk.hash], &[chunk.hash]) + .unwrap() + } else { + writer.finalize().unwrap() + }; XetShardFixture { content_size: content.len() as u64, file_hash: chunk.hash, @@ -1014,6 +1021,78 @@ mod tests { } } + #[tokio::test] + async fn downloaded_shards_load_plain_and_bloom_objects_as_native_xet_bodies() { + for bloom in [false, true] { + let fixture = xet_file_shard(55, bloom); + let store = memory_store(); + let layout = crab_storage::StoreLayout::new( + store.as_storage().clone(), + "compact-format".to_owned(), + ); + layout + .store() + .put( + &layout.shard_path(&fixture.shard_hash), + fixture.shard_body.clone(), + ) + .await + .unwrap(); + let target = tempfile::tempdir().unwrap(); + download_shards( + &store, + &layout, + &[fixture.shard_hash.hex()], + target.path(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let shards = + MDBShardFile::load_all_valid(target.path(), &new_shard_file_cache()).unwrap(); + let mut actual = Vec::new(); + shards[0].read_into_buffer(&mut actual).unwrap(); + assert_eq!( + actual, + crab_xet::shard_parse::strip_bloom_trailer(&fixture.shard_body) + ); + } + } + + #[tokio::test] + async fn downloaded_bloom_shards_authenticate_trailer_before_creating_native_files() { + let fixture = xet_file_shard(55, true); + let store = memory_store(); + let layout = crab_storage::StoreLayout::new( + store.as_storage().clone(), + "compact-corrupt-bloom".to_owned(), + ); + let mut corrupt = fixture.shard_body.to_vec(); + let index = corrupt.len() - 13; + corrupt[index] ^= 1; + layout + .store() + .put( + &layout.shard_path(&fixture.shard_hash), + Bytes::from(corrupt), + ) + .await + .unwrap(); + let target = tempfile::tempdir().unwrap(); + let result = download_shards( + &store, + &layout, + &[fixture.shard_hash.hex()], + target.path(), + &CancellationToken::new(), + ) + .await; + assert!( + matches!(result, Err(CrabError::CorruptObject { .. })) + && std::fs::read_dir(target.path()).unwrap().next().is_none() + ); + } + #[test] fn parse_size_mib() { assert_eq!(parse_size_str("100MiB").unwrap(), 100 * 1024 * 1024); @@ -1277,8 +1356,8 @@ mod tests { crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") .await .unwrap(); - let fixture_a = xet_file_shard(11); - let fixture_b = xet_file_shard(12); + let fixture_a = xet_file_shard(11, true); + let fixture_b = xet_file_shard(12, false); for fixture in [&fixture_a, &fixture_b] { layout .store() @@ -1598,8 +1677,8 @@ mod tests { fn capsule_filter_keeps_only_authenticated_files_and_dependencies() { use crab_xet::shard::ShardWriter; - let retained = xet_file_shard(31); - let foreign = xet_file_shard(32); + let retained = xet_file_shard(31, false); + let foreign = xet_file_shard(32, false); let mut writer = ShardWriter::new(); for fixture in [&retained, &foreign] { let parsed = MDBMinimalShard::from_reader( diff --git a/crab/src/cmd/history_recovery_v2.rs b/crab/src/cmd/history_recovery_v2.rs index e08f7f58c..88cb21c68 100644 --- a/crab/src/cmd/history_recovery_v2.rs +++ b/crab/src/cmd/history_recovery_v2.rs @@ -29,7 +29,7 @@ struct VerifiedCapsuleHistory { segment: HistorySegment, checkpoint: LayeredCheckpoint, verification: HistoryVerificationPayload, - _workspace: tempfile::TempDir, + workspace: tempfile::TempDir, } pub(super) async fn run( @@ -208,11 +208,11 @@ async fn verify_history( validate_visibility(&segment, &checkpoint)?; let workspace = verify_git_checkpoint(layout, &segment, &checkpoint).await?; - let dependencies = crab_read::capsule_protocol::verify_installed_checkpoint_dependencies( + let dependencies = crab_read::capsule_protocol::verify_installed_catalog_dependencies( layout, &workspace.path().join("repository.git"), segment.refs(), - &checkpoint, + checkpoint.catalog(), crate::cmd::fsck_store::CAPSULE_GIT_SCAN_LIMITS, cancel, ) @@ -255,7 +255,7 @@ async fn verify_history( segment, checkpoint, verification, - _workspace: workspace, + workspace, }) } @@ -610,7 +610,7 @@ async fn restore_history( ) -> Result { let lease = crate::maintenance::GcSweepLease::acquire(store, router.repo_prefix(), cancel).await?; - let operation = async { + let operation = Box::pin(async { check_cancelled(cancel)?; let base = crab_write::capsule_protocol::open_root(layout).await?; let verified = verify_history( @@ -633,23 +633,42 @@ async fn restore_history( let previous_generation = view.root().root().generation(); let (refs_added, refs_updated, refs_deleted) = ref_change_counts(view.refs(), verified.segment.refs()); - let current_catalog = view.pointer_catalog().await?; - // Keep both catalogs: history may use a recipe no longer in the current - // view, while newer content must remain available after a rollback. - let mut restored_catalog = verified - .checkpoint - .pointer_catalog_from_store(layout, u64::MAX) - .await?; - restored_catalog.apply(¤t_catalog)?; - let outcome = crab_remote::checkpoint::publish_capsule_checkpoint_from_view( + let current_catalog = crab_remote::checkpoint::prepare_capsule_catalog_from_view( layout, &view, - 0, RESTORE_CHECKPOINT_BYTES, - cancel, ) .await .map_err(map_checkpoint_error)?; + // Retain newer-only content while older refs regain visibility. Current + // bindings still win, so the old byte proof alone cannot certify them. + let restored_catalog = crab_metadata::capsule_protocol::merge_checkpoint_catalogs( + layout, + verified.checkpoint.catalog(), + current_catalog.as_ref(), + RESTORE_CHECKPOINT_BYTES, + ) + .await?; + crab_read::capsule_protocol::verify_installed_catalog_dependencies( + layout, + &verified.workspace.path().join("repository.git"), + verified.segment.refs(), + restored_catalog.as_ref(), + crate::cmd::fsck_store::CAPSULE_GIT_SCAN_LIMITS, + cancel, + ) + .await?; + check_cancelled(cancel)?; + let outcome = + crab_remote::checkpoint::publish_capsule_checkpoint_with_catalog_root_from_view( + layout, + &view, + current_catalog, + RESTORE_CHECKPOINT_BYTES, + cancel, + ) + .await + .map_err(map_checkpoint_error)?; let checkpointed = crab_write::capsule_protocol::open_root(layout).await?; // The shared publisher reports CAS loss as a no-op. Restore cannot // fence an unrelated winner using the catalog and refs captured above. @@ -696,7 +715,7 @@ async fn restore_history( .await?; let restore = async { - let checkpoint = LayeredCheckpoint::build( + let checkpoint = LayeredCheckpoint::build_with_catalog_root( fenced.record().root().generation(), fenced.record().digest(), verified.checkpoint.sources().to_vec(), @@ -737,7 +756,7 @@ async fn restore_history( (Some(error), _) => Err(error), (None, Err(error)) => Err(error.into()), } - } + }) .await; let release = lease.release().await; match (operation, release) { diff --git a/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs b/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs index 2da05d627..9f3de1602 100644 --- a/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_catalog_tests.rs @@ -1,8 +1,12 @@ use super::*; -async fn xet_history_fixture() -> ( - CapsuleStoreLayout, - RootSnapshot, +async fn xet_catalog_fixture( + layout: &CapsuleStoreLayout, + parts: &[&[u8]], + file_hash: [u8; 32], +) -> ( + PointerCatalog, + crab_types::pointer::Pointer, Vec<(object_store::path::Path, Bytes)>, ) { use crab_metadata::capsule_protocol::{ @@ -16,13 +20,17 @@ async fn xet_history_fixture() -> ( format::Chunk, }, }; - let layout = empty_fixture().await; - let content = Bytes::from_static(b"historical xet payload"); - let chunk = Chunk::new(content.clone()); + let size = parts.iter().map(|part| part.len() as u64).sum(); + let chunks = parts + .iter() + .map(|part| Chunk::new(Bytes::copy_from_slice(part))) + .collect::>(); let mut builder = XorbBuilder::new(); - builder.push(&chunk, RunId(0)).unwrap(); + for chunk in &chunks { + builder.push(chunk, RunId(0)).unwrap(); + } let xorb = builder.finalize().unwrap().pop().unwrap(); - let file_hash = MerkleHash::from(*blake3::hash(&content).as_bytes()); + let file_hash = MerkleHash::from(file_hash); let placements = xorb .placements .iter() @@ -36,7 +44,14 @@ async fn xet_history_fixture() -> ( )) .unwrap(); writer - .add_file(file_info_from_placements(file_hash, &[chunk.hash], &placements).unwrap()) + .add_file( + file_info_from_placements( + file_hash, + &chunks.iter().map(|chunk| chunk.hash).collect::>(), + &placements, + ) + .unwrap(), + ) .unwrap(); let (shard, shard_hash) = writer.finalize().unwrap(); let bodies = vec![ @@ -50,7 +65,11 @@ async fn xet_history_fixture() -> ( XorbCatalogEntry::new( bodies[0].1.len() as u64, blake3::hash(&bodies[0].1).to_hex().to_string(), - vec![XorbChunkEntry::new(chunk.hash.hex(), content.len() as u32)], + chunks + .iter() + .zip(parts) + .map(|(chunk, part)| XorbChunkEntry::new(chunk.hash.hex(), part.len() as u32)) + .collect(), ), ) .unwrap(); @@ -63,7 +82,7 @@ async fn xet_history_fixture() -> ( catalog .insert_file( file_hash.hex(), - FileCatalogEntry::new(content.len() as u64, shard_hash.hex()), + FileCatalogEntry::new(size, shard_hash.hex()), ) .unwrap(); for (path, body) in &bodies { @@ -71,9 +90,21 @@ async fn xet_history_fixture() -> ( } let pointer = crab_types::pointer::Pointer { file_hash: file_hash.into(), - size: content.len() as u64, + size, shard_hint: None, }; + (catalog, pointer, bodies) +} + +async fn xet_history_fixture() -> ( + CapsuleStoreLayout, + RootSnapshot, + Vec<(object_store::path::Path, Bytes)>, +) { + let layout = empty_fixture().await; + let content = b"historical xet payload"; + let (catalog, pointer, bodies) = + xet_catalog_fixture(&layout, &[content], *blake3::hash(content).as_bytes()).await; publish_blob(&layout, "refs/tags/xet", &pointer.serialize(), catalog).await; assert!( crab_remote::checkpoint::publish_capsule_checkpoint( @@ -176,3 +207,114 @@ async fn historical_checkpoint_proof_rejects_corrupt_catalog_payload_bodies() { ); } } + +#[tokio::test] +async fn restore_keeps_newer_recipes_and_newer_only_content() { + let (layout, old_root, _) = xet_history_fixture().await; + let chain = history_chain(&layout, &old_root).await.unwrap(); + let historical = &chain[0]; + let content = b"historical xet payload"; + let (mut incoming, pointer, _) = xet_catalog_fixture( + &layout, + &[&content[..5], &content[5..]], + *blake3::hash(content).as_bytes(), + ) + .await; + let original_key = crab_xet::hash::MerkleHash::from(pointer.file_hash).hex(); + let replacement = incoming.files()[&original_key].clone(); + let other_content = b"content introduced after the restored checkpoint"; + let (other, other_pointer, _) = xet_catalog_fixture( + &layout, + &[other_content], + *blake3::hash(other_content).as_bytes(), + ) + .await; + let other_key = crab_xet::hash::MerkleHash::from(other_pointer.file_hash).hex(); + let other_entry = other.files()[&other_key].clone(); + incoming.apply(&other).unwrap(); + publish_blob(&layout, "refs/tags/newer", b"newer git object", incoming).await; + let store = Store::from_storage(layout.store().clone()); + let router = StoreLayout::new(store.clone(), layout.repo_prefix().to_owned()); + restore_history( + &store, + &router, + &layout, + &HistoryRestoreArgs { + generation: historical.checkpoint().covered_generation(), + digest: Some(historical.hash().to_owned()), + apply: true, + json: false, + }, + &CancellationToken::new(), + ) + .await + .unwrap(); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let checkpoint = crab_metadata::capsule_protocol::load_layered_checkpoint( + &layout, + root.record().root().checkpoint().unwrap(), + LIMIT, + ) + .await + .unwrap(); + let selected = checkpoint + .pointer_catalog_lookup(&layout, LIMIT) + .unwrap() + .files(layout.store(), &[original_key.clone(), other_key.clone()]) + .await + .unwrap(); + assert_eq!( + ( + root.record().root().refs(), + selected.files().get(&original_key), + selected.files().get(&other_key) + ), + (historical.refs(), Some(&replacement), Some(&other_entry)) + ); +} + +#[tokio::test] +async fn restore_rejects_current_winning_recipe_with_wrong_bytes_before_publication() { + let (layout, old_root, _) = xet_history_fixture().await; + let chain = history_chain(&layout, &old_root).await.unwrap(); + let historical = &chain[0]; + let claim = *blake3::hash(b"historical xet payload").as_bytes(); + let (incoming, _, _) = xet_catalog_fixture(&layout, &[b"historical Xet payload"], claim).await; + publish_blob(&layout, "refs/tags/newer", b"newer git object", incoming).await; + let before = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let store = Store::from_storage(layout.store().clone()); + let router = StoreLayout::new(store.clone(), layout.repo_prefix().to_owned()); + let cancel = CancellationToken::new(); + let result = restore_history( + &store, + &router, + &layout, + &HistoryRestoreArgs { + generation: historical.checkpoint().covered_generation(), + digest: Some(historical.hash().to_owned()), + apply: true, + json: false, + }, + &cancel, + ) + .await; + let error = result.unwrap_err(); + assert!( + matches!(&error, CrabError::HashMismatch { requested, actual } + if *requested == crab_xet::hash::MerkleHash::from(claim).hex() + && *actual == blake3::hash(b"historical Xet payload").to_hex().as_str()), + "unexpected error: {error}" + ); + let after = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + assert_eq!(after.record().digest(), before.record().digest()); + let lease = crate::maintenance::GcSweepLease::acquire(&store, router.repo_prefix(), &cancel) + .await + .unwrap(); + lease.release().await.unwrap(); +} diff --git a/crab/src/cmd/history_recovery_v2/layered_tests.rs b/crab/src/cmd/history_recovery_v2/layered_tests.rs index 014ad89f5..98a61edca 100644 --- a/crab/src/cmd/history_recovery_v2/layered_tests.rs +++ b/crab/src/cmd/history_recovery_v2/layered_tests.rs @@ -146,7 +146,7 @@ async fn layered_history_verification_preserves_historical_refs_and_bytes() { .unwrap(); assert_eq!(verified.verification.refs, 1); - let git_dir = verified._workspace.path().join("repository.git"); + let git_dir = verified.workspace.path().join("repository.git"); let refs = Command::new("git") .arg("--git-dir") .arg(&git_dir) @@ -250,7 +250,7 @@ async fn historical_thin_member_is_repaired_against_its_retained_base() { .unwrap(); let output = Command::new("git") .arg("--git-dir") - .arg(verified._workspace.path().join("repository.git")) + .arg(verified.workspace.path().join("repository.git")) .args(["cat-file", "blob", "refs/tags/target"]) .output() .unwrap(); diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 5fd57b510..d4b14a436 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -77,6 +77,12 @@ verified before checkpoint/history writes and root CAS. Ref-only publication and physical repacking reuse the existing commitment without catalog object I/O. A catalog delta resolves its selected dependency closure and rewrites affected paths, preserving ordered file bindings and immutable descriptor conflicts. +`merge_checkpoint_catalogs` unions durable roots without retaining logical +recipe maps. Complete input proofs and merge batches share one encoded-page +budget. It retains newer file bindings, checks immutable descriptors and file +sizes, chooses canonical reverse placements, and verifies immutable writes. +Callers hold GC admission through publication and separately prove external +payloads and final file bytes. Unchanged subtrees retain their identities. Readers share an aggregate page-byte budget across checkpoint and packed-run sources. Logical expansion has a separate allowance of four times the encoded ceiling: shared diff --git a/crates/crab-metadata/src/capsule_protocol/layered.rs b/crates/crab-metadata/src/capsule_protocol/layered.rs index 0bc6f2e46..d43c44cbe 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered.rs @@ -13,9 +13,9 @@ use crate::validation::{validate_content_hash, validate_sha1}; mod catalog; pub use catalog::CheckpointCatalog; -#[cfg(feature = "storage")] -pub use catalog::fold_checkpoint_catalog; use catalog::{CatalogInput, PreparedCatalog}; +#[cfg(feature = "storage")] +pub use catalog::{fold_checkpoint_catalog, merge_checkpoint_catalogs}; mod visibility_pages; #[cfg(feature = "storage")] diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs index 7196d5298..f907549fe 100644 --- a/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog.rs @@ -64,8 +64,12 @@ impl CheckpointCatalog { ) } + /// Capture selected lookups from this durable immutable catalog. + /// + /// The caller protects the root and its dependencies against GC. Each query + /// uses the supplied aggregate encoded-page budget and bounded parsed cache. #[cfg(feature = "storage")] - pub(super) fn lookup( + pub fn lookup( &self, layout: &crab_storage::StoreLayout, maximum: u64, @@ -293,6 +297,56 @@ pub async fn fold_checkpoint_catalog( Ok(Some(source)) } +/// Union two durable catalogs, retaining newer file bindings and immutable descriptors. +/// +/// Both complete physical closures and every merge batch share one byte budget. +/// Recipe maps are never materialized; unchanged subtrees retain their identities. +/// Missing/corrupt inputs, descriptor/size conflicts and exhausted encoded or +/// logical-work budgets return errors. The caller holds GC admission until the replacement checkpoint becomes visible. +/// External shard/xorb bodies and final file bytes require separate verification. +#[cfg(feature = "storage")] +pub async fn merge_checkpoint_catalogs( + layout: &crab_storage::StoreLayout, + older: Option<&CheckpointCatalog>, + newer: Option<&CheckpointCatalog>, + maximum_bytes: u64, +) -> Result> { + let mut work = crate::pages::PageWork::new(maximum_bytes, 0); + let older_reader = older.map(|source| source.reader(layout)).transpose()?; + let newer_reader = newer.map(|source| source.reader(layout)).transpose()?; + if let Some(reader) = &older_reader { + reader.verify(layout.store(), &mut work).await?; + } + if let Some(reader) = &newer_reader + && older != newer + { + reader.verify(layout.store(), &mut work).await?; + } + let (Some(older_reader), Some(newer_reader)) = (older_reader, newer_reader) else { + return Ok(newer.or(older).cloned()); + }; + if older == newer { + return Ok(newer.cloned()); + } + let descriptor = older_reader + .merge(layout.store(), &newer_reader, &mut work) + .await?; + if newer.is_some_and(|source| descriptor == source.pages) { + return Ok(newer.cloned()); + } + if older.is_some_and(|source| descriptor == source.pages) { + return Ok(older.cloned()); + } + let source = CheckpointCatalog::new(descriptor)?; + let body = source.pages.encode()?; + work.admit_object(*blake3::hash(&body).as_bytes(), body.len() as u64)?; + layout + .store() + .put_if_absent_verified(&layout.capsule_catalog_path(&source.hash), body) + .await?; + Ok(Some(source)) +} + #[cfg(all(test, feature = "storage"))] mod tests { use super::*; @@ -475,3 +529,7 @@ mod tests { ); } } + +#[cfg(all(test, feature = "storage"))] +#[path = "catalog/merge_tests.rs"] +mod merge_tests; diff --git a/crates/crab-metadata/src/capsule_protocol/layered/catalog/merge_tests.rs b/crates/crab-metadata/src/capsule_protocol/layered/catalog/merge_tests.rs new file mode 100644 index 000000000..40e7eee28 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/layered/catalog/merge_tests.rs @@ -0,0 +1,280 @@ +#![expect(clippy::unwrap_used, reason = "test assertions")] +use super::*; +use crate::capsule_protocol::{ + FileCatalogEntry, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, +}; +use crab_storage::{StorageObservation, StorageObserver, StorageOperation, Store, StoreLayout}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +#[derive(Default)] +struct Writes(AtomicU64); +impl StorageObserver for Writes { + fn started(&self, _: StorageOperation) {} + fn finished(&self, observation: StorageObservation) { + self.0 + .fetch_add(observation.bytes_written, Ordering::Relaxed); + } +} +fn layout() -> (StoreLayout, Arc) { + let writes = Arc::new(Writes::default()); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(writes.clone()); + (StoreLayout::new(store, "catalog-merge".to_owned()), writes) +} +async fn durable(layout: &StoreLayout, catalog: &PointerCatalog) -> CheckpointCatalog { + let (descriptor, objects) = Descriptor::build(catalog).unwrap(); + publish_objects(layout, &objects).await.unwrap(); + CheckpointCatalog::new(descriptor).unwrap() +} +fn catalog(files: std::ops::Range, shard: u32, xorb: u32) -> PointerCatalog { + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + format!("{xorb:064x}"), + XorbCatalogEntry::new( + 70, + "d".repeat(64), + vec![ + XorbChunkEntry::new("c".repeat(64), 7), + XorbChunkEntry::new("c".repeat(64), 7), + ], + ), + ) + .unwrap(); + catalog + .insert_shard( + format!("{shard:064x}"), + ShardCatalogEntry::new(40, vec![format!("{xorb:064x}")]), + ) + .unwrap(); + for file in files { + catalog + .insert_file( + format!("{file:064x}"), + FileCatalogEntry::new(14, format!("{shard:064x}")), + ) + .unwrap(); + } + catalog +} +async fn materialize(layout: &StoreLayout, source: &CheckpointCatalog) -> PointerCatalog { + source + .reader(layout) + .unwrap() + .full( + layout.store(), + &mut crate::pages::PageWork::new(32 * 1024 * 1024, 0), + ) + .await + .unwrap() +} + +#[tokio::test] +async fn union_matches_ordered_catalog_apply_including_reverse_placements() { + for (old_xorb, new_xorb) in [(1, 9), (9, 1)] { + let (layout, _) = layout(); + let old = catalog(0..10, 21, old_xorb); + let new = catalog(5..15, 22, new_xorb); + let older = durable(&layout, &old).await; + let newer = durable(&layout, &new).await; + let result = merge_checkpoint_catalogs(&layout, Some(&older), Some(&newer), 1024 * 1024) + .await + .unwrap() + .unwrap(); + let mut expected = old; + expected.apply(&new).unwrap(); + assert_eq!(materialize(&layout, &result).await, expected); + } +} + +#[tokio::test] +async fn union_rejects_conflicts_even_when_newer_bindings_win() { + for conflict in ["file-size", "shard", "shard-xorbs", "xorb", "xorb-chunks"] { + let (layout, writes) = layout(); + let old = catalog(0..2, 21, 1); + let mut new = PointerCatalog::new(); + for (identity, entry) in old.files() { + let size = entry.size() + u64::from(conflict == "file-size"); + new.insert_file( + identity.clone(), + FileCatalogEntry::new(size, entry.shard_hash()), + ) + .unwrap(); + } + for (identity, entry) in old.shards() { + new.insert_shard( + identity.clone(), + ShardCatalogEntry::new( + entry.encoded_size() + u64::from(conflict == "shard"), + if conflict == "shard-xorbs" { + vec![format!("{:064x}", 2)] + } else { + entry.xorb_hashes().to_vec() + }, + ), + ) + .unwrap(); + } + for (identity, entry) in old.xorbs() { + new.insert_xorb( + identity.clone(), + XorbCatalogEntry::new( + entry.encoded_size() + u64::from(conflict == "xorb"), + entry.body_digest(), + if conflict == "xorb-chunks" { + vec![ + XorbChunkEntry::new("e".repeat(64), 7), + XorbChunkEntry::new("e".repeat(64), 7), + ] + } else { + entry.chunks().to_vec() + }, + ), + ) + .unwrap(); + } + if conflict == "shard-xorbs" { + new.insert_xorb( + format!("{:064x}", 2), + old.xorbs().values().next().unwrap().clone(), + ) + .unwrap(); + } + let older = durable(&layout, &old).await; + let newer = durable(&layout, &new).await; + writes.0.store(0, Ordering::Relaxed); + assert!( + merge_checkpoint_catalogs(&layout, Some(&older), Some(&newer), 1024 * 1024) + .await + .is_err(), + "{conflict}" + ); + assert_eq!(writes.0.load(Ordering::Relaxed), 0); + } +} + +#[tokio::test] +async fn shared_catalog_reuses_newer_root_without_writes() { + let (layout, writes) = layout(); + let old = catalog(0..100_000, 21, 1); + let older = durable(&layout, &old).await; + let delta = catalog(99_999..100_001, 22, 2); + // The extra descriptor is immutable, while an existing file changes shard. + let newer = fold_checkpoint_catalog( + &layout, + Some(&older), + std::slice::from_ref(&delta), + 32 * 1024 * 1024, + ) + .await + .unwrap() + .unwrap(); + drop(old); + let mut objects = BTreeMap::new(); + for source in [&older, &newer] { + objects.extend( + source + .reader(&layout) + .unwrap() + .inspected_objects(layout.store(), 32 * 1024 * 1024) + .await + .unwrap(), + ); + } + let maximum = objects.values().sum(); + writes.0.store(0, Ordering::Relaxed); + let merged = merge_checkpoint_catalogs(&layout, Some(&older), Some(&newer), maximum) + .await + .unwrap(); + assert_eq!(merged, Some(newer)); + assert_eq!(writes.0.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn union_admission_covers_both_inputs_before_any_writes() { + let (layout, writes) = layout(); + let older = durable(&layout, &catalog(0..3000, 21, 1)).await; + let newer = durable(&layout, &catalog(5000..8000, 22, 2)).await; + let mut sizes = Vec::new(); + for source in [&older, &newer] { + sizes.push( + source + .reader(&layout) + .unwrap() + .inspected_objects(layout.store(), 1024 * 1024) + .await + .unwrap() + .iter() + .map(|(_, size)| size) + .sum::(), + ); + } + writes.0.store(0, Ordering::Relaxed); + assert!( + merge_checkpoint_catalogs( + &layout, + Some(&older), + Some(&newer), + *sizes.iter().max().unwrap() + ) + .await + .is_err() + ); + assert_eq!(writes.0.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn newer_bindings_cannot_hide_a_missing_older_descriptor() { + let (layout, writes) = layout(); + let older = durable(&layout, &catalog(0..2, 21, 1)).await; + let newer = durable(&layout, &catalog(0..3, 22, 2)).await; + layout + .store() + .delete(&layout.capsule_catalog_path(older.hash())) + .await + .unwrap(); + writes.0.store(0, Ordering::Relaxed); + assert!( + merge_checkpoint_catalogs(&layout, Some(&older), Some(&newer), 1024 * 1024) + .await + .is_err() + ); + assert_eq!(writes.0.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn union_retains_present_roots_and_empty_descriptors() { + let (layout, _) = layout(); + let empty = durable(&layout, &PointerCatalog::new()).await; + for (older, newer) in [ + (None, None), + (Some(&empty), None), + (None, Some(&empty)), + (Some(&empty), Some(&empty)), + ] { + assert_eq!( + merge_checkpoint_catalogs(&layout, older, newer, 4096) + .await + .unwrap(), + newer.or(older).cloned() + ); + } +} + +#[tokio::test] +async fn empty_newer_catalog_reuses_older_columns_without_writes() { + let (layout, writes) = layout(); + let older = durable(&layout, &catalog(0..3000, 21, 1)).await; + let newer = durable(&layout, &PointerCatalog::new()).await; + writes.0.store(0, Ordering::Relaxed); + assert_eq!( + merge_checkpoint_catalogs(&layout, Some(&older), Some(&newer), 1024 * 1024) + .await + .unwrap(), + Some(older) + ); + assert_eq!(writes.0.load(Ordering::Relaxed), 0); +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 0cdcff2a6..46aa11826 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -62,7 +62,7 @@ pub use layered::{ source_catalog_digest, visibility_object_set_digest, }; #[cfg(feature = "storage")] -pub use layered::{LayeredVisibilityReader, fold_checkpoint_catalog}; +pub use layered::{LayeredVisibilityReader, fold_checkpoint_catalog, merge_checkpoint_catalogs}; #[cfg(feature = "storage")] pub use plan::{ CapsulePlanReceipt, ensure_capsule_plan_unattempted, prepare_capsule_plan, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs index 85851daaa..e2ba0272f 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader.rs @@ -1,4 +1,5 @@ mod inspection; +mod merge; use super::*; use crate::pages::PageWork; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs index e3f15fdd2..caa9b4836 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/inspection.rs @@ -14,18 +14,28 @@ impl CatalogPages { maximum: u64, ) -> Result> { let mut work = PageWork::new(maximum, 0); - self.open(store, &mut work).await?; - let space = - crab_storage::scratch::ScratchSpace::temporary(maximum.saturating_mul(4)).await?; - let mut join = Join::new(space); - let checked = self.inspect(store, &mut work, &mut join).await; - join.finish(checked).await?; + self.verify(store, &mut work).await?; Ok(work .object_dependencies() .map(|(hash, bytes)| (self.pages.path(hash), bytes)) .collect()) } + // Merge inputs share admission before any page I/O; separate inspection + // budgets would let each input consume the whole operation allowance. + pub(in crate::capsule_protocol) async fn verify( + &self, + store: &Store, + work: &mut PageWork, + ) -> Result<()> { + self.open(store, work).await?; + let space = + crab_storage::scratch::ScratchSpace::temporary(work.maximum.saturating_mul(4)).await?; + let mut join = Join::new(space); + let checked = self.inspect(store, work, &mut join).await; + join.finish(checked).await + } + pub(in crate::capsule_protocol) async fn inspect_into( &self, store: &Store, diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge.rs new file mode 100644 index 000000000..c66823b50 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge.rs @@ -0,0 +1,193 @@ +//! Union authenticated checkpoint columns without retaining their recipes. +use super::*; +use crate::pages::object::Node; +use std::collections::BTreeMap; + +type RecipeDigests = BTreeMap<(u8, [u8; 32]), [u8; 32]>; + +impl CatalogPages { + // Both complete inputs must have passed `verify` under this same budget. + // Newer bindings win, but cannot hide immutable conflicts or size changes. + pub(in crate::capsule_protocol) async fn merge( + &self, + store: &Store, + newer: &Self, + work: &mut PageWork, + ) -> Result { + let mut result = newer.descriptor.clone(); + let mut recipes = RecipeDigests::new(); + for (kind, target) in [ + (Kind::CatalogFiles, &mut result.files), + (Kind::CatalogShards, &mut result.shards), + (Kind::CatalogXorbs, &mut result.xorbs), + (Kind::CatalogChunks, &mut result.chunks), + ] { + if self.root(kind)? == target.as_ref() { + continue; + } + if target.is_none() { + *target = self.root(kind)?.cloned(); + continue; + } + let mut pending = self.root(kind)?.cloned().into_iter().collect(); + while let Some((_, node)) = self + .pages + .next_page(store, &mut pending, kind, work) + .await? + { + let Node::Leaf(rows) = node.as_ref() else { + continue; + }; + let keys = rows.iter().map(|(key, _)| key.clone()).collect::>(); + let existing = newer + .pages + .lookup(store, target.as_ref(), kind, &keys, work) + .await?; + let mut changed = Vec::new(); + for ((key, incoming), existing) in rows.iter().zip(existing) { + let Some(prior) = existing else { + changed.push((key.clone(), incoming.clone())); + continue; + }; + if prior == *incoming { + continue; + } + match kind { + Kind::CatalogFiles => { + if file_from_bytes(&prior)?.size != file_from_bytes(incoming)?.size { + return Err(contract_error(format!( + "file {} has conflicting declared sizes", + hex(key)? + ))); + } + } + Kind::CatalogShards => { + let prior: ShardRow = from_json(&prior)?; + let incoming: ShardRow = from_json(incoming)?; + if prior.encoded_size != incoming.encoded_size + || !newer + .same_recipe( + store, + (prior.xorbs, incoming.xorbs), + Kind::CatalogShardXorbs, + &mut recipes, + work, + ) + .await? + { + return Err(contract_error(format!( + "shard {} has conflicting descriptors", + hex(key)? + ))); + } + } + Kind::CatalogXorbs => { + let prior: XorbRow = from_json(&prior)?; + let incoming: XorbRow = from_json(incoming)?; + if prior.encoded_size != incoming.encoded_size + || prior.body_digest != incoming.body_digest + || !newer + .same_recipe( + store, + (Some(prior.chunks), Some(incoming.chunks)), + Kind::CatalogXorbChunks, + &mut recipes, + work, + ) + .await? + { + return Err(contract_error(format!( + "xorb {} has conflicting descriptors", + hex(key)? + ))); + } + } + Kind::CatalogChunks => { + // Each input proved lowest-xorb/first-occurrence ownership. + // Their union must retain the lower of those two placements. + if prior[..36] > incoming[..36] { + changed.push((key.clone(), incoming.clone())); + } + } + _ => return Err(corrupt("catalog merge selected an invalid column")), + } + } + *target = newer + .pages + .edit(store, target.as_ref(), kind, changed, work) + .await?; + } + } + result.validate()?; + Ok(result) + } + async fn same_recipe( + &self, + store: &Store, + roots: (Option, Option), + kind: Kind, + recipes: &mut RecipeDigests, + work: &mut PageWork, + ) -> Result { + if roots.0 == roots.1 { + return Ok(true); + } + let (Some(left), Some(right)) = roots else { + return Ok(false); + }; + let left = self + .recipe_digest(store, &left, kind, recipes, work) + .await?; + let right = self + .recipe_digest(store, &right, kind, recipes, work) + .await?; + Ok(left == right) + } + + async fn recipe_digest( + &self, + store: &Store, + root: &Reference, + kind: Kind, + recipes: &mut RecipeDigests, + work: &mut PageWork, + ) -> Result<[u8; 32]> { + let identity = (kind as u8, root.hash); + if let Some(digest) = recipes.get(&identity) { + return Ok(*digest); + } + let value_bytes = match kind { + Kind::CatalogShardXorbs => 0, + Kind::CatalogXorbChunks => 36, + _ => return Err(corrupt("catalog comparison selected an invalid recipe")), + }; + // Cache only a complete authenticated scan. Charge logical work once per + // root, so shared or repaged recipes cannot authorize quadratic replay. + work.admit_materialized( + u64::from(root.count) * (kind.key_bytes() + value_bytes + 8) as u64, + )?; + let mut hasher = blake3::Hasher::new_derive_key("crab catalog recipe comparison"); + hasher.update(&[kind as u8]); + hasher.update(&root.count.to_le_bytes()); + let mut pending = vec![root.clone()]; + while let Some((_, node)) = self + .pages + .next_page(store, &mut pending, kind, work) + .await? + { + if let Node::Leaf(rows) = node.as_ref() { + for (key, value) in rows { + hasher.update(key); + hasher.update(&(value.len() as u64).to_le_bytes()); + hasher.update(value); + } + } + } + let digest = *hasher.finalize().as_bytes(); + recipes.insert(identity, digest); + Ok(digest) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge/tests.rs b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge/tests.rs new file mode 100644 index 000000000..db26e4fc2 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/pointer/object/reader/merge/tests.rs @@ -0,0 +1,164 @@ +use super::*; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +#[tokio::test] +async fn immutable_recipe_equality_ignores_valid_page_partitioning() { + let reads = Arc::new(AtomicUsize::new(0)); + let observed = reads.clone(); + let store = Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_read_request_observer(Arc::new(move |_| { + observed.fetch_add(1, Ordering::Relaxed); + })); + let prefix = Path::from("catalog"); + let mut catalog = PointerCatalog::new(); + let ids = (1..=2000) + .map(|id| format!("{id:064x}")) + .collect::>(); + for identity in &ids { + catalog + .insert_xorb( + identity.clone(), + XorbCatalogEntry::new( + 70, + "d".repeat(64), + vec![XorbChunkEntry::new("c".repeat(64), 7)], + ), + ) + .unwrap(); + } + catalog + .insert_shard("a".repeat(64), ShardCatalogEntry::new(40, ids.clone())) + .unwrap(); + catalog + .insert_file("b".repeat(64), FileCatalogEntry::new(7, "a".repeat(64))) + .unwrap(); + let (descriptor, objects) = Descriptor::build(&catalog).unwrap(); + let body = descriptor.encode().unwrap(); + let older = CatalogPages::new( + prefix.clone(), + descriptor.clone(), + *blake3::hash(&body).as_bytes(), + body.len() as u64, + ) + .unwrap(); + for (identity, body) in objects { + store + .put_if_absent_verified(&older.pages.path(&identity), body) + .await + .unwrap(); + } + let mut work = PageWork::new(16 * 1024 * 1024, 0); + let rows = |keys: &[String]| { + keys.iter() + .map(|key| (hash(key).unwrap().to_vec(), Vec::new())) + .collect::() + }; + // Insert the lower range last. The edited tree remains canonical but its + // partial leaf boundary differs from a single initial build of all rows. + let suffix = older + .pages + .edit( + &store, + None, + Kind::CatalogShardXorbs, + rows(&ids[1000..]), + &mut work, + ) + .await + .unwrap(); + let alternate = older + .pages + .edit( + &store, + suffix.as_ref(), + Kind::CatalogShardXorbs, + rows(&ids[..1000]), + &mut work, + ) + .await + .unwrap(); + let mut row: ShardRow = from_json( + older + .rows( + &store, + Kind::CatalogShards, + &[hash(&"a".repeat(64)).unwrap().to_vec()], + &mut work, + ) + .await + .unwrap()[0] + .as_ref() + .unwrap(), + ) + .unwrap(); + assert_ne!(row.xorbs, alternate); + let recipe_roots = (row.xorbs.clone(), alternate.clone()); + row.xorbs = alternate; + let mut descriptor = descriptor; + descriptor.shards = older + .pages + .edit( + &store, + descriptor.shards.as_ref(), + Kind::CatalogShards, + vec![(hash(&"a".repeat(64)).unwrap().to_vec(), json(&row).unwrap())], + &mut work, + ) + .await + .unwrap(); + let body = descriptor.encode().unwrap(); + let identity = *blake3::hash(&body).as_bytes(); + store + .put_if_absent_verified(&older.pages.path(&identity), body.clone()) + .await + .unwrap(); + let newer = CatalogPages::new(prefix, descriptor.clone(), identity, body.len() as u64).unwrap(); + let mut work = PageWork::new(16 * 1024 * 1024, 0); + older.verify(&store, &mut work).await.unwrap(); + newer.verify(&store, &mut work).await.unwrap(); + assert_eq!( + older.merge(&store, &newer, &mut work).await.unwrap(), + descriptor + ); + let mut recipes = RecipeDigests::new(); + assert!( + newer + .same_recipe( + &store, + recipe_roots.clone(), + Kind::CatalogShardXorbs, + &mut recipes, + &mut work + ) + .await + .unwrap() + ); + let body = descriptor.encode().unwrap(); + let cold = CatalogPages::new( + Path::from("catalog"), + descriptor, + *blake3::hash(&body).as_bytes(), + body.len() as u64, + ) + .unwrap(); + reads.store(0, Ordering::Relaxed); + assert!( + cold.same_recipe( + &store, + recipe_roots, + Kind::CatalogShardXorbs, + &mut recipes, + &mut work + ) + .await + .unwrap() + ); + assert_eq!( + reads.load(Ordering::Relaxed), + 0, + "shared recipe comparison repeated origin reads with a cold parsed-page cache" + ); +} diff --git a/crates/crab-read/README.md b/crates/crab-read/README.md index def45b954..44afc21a1 100644 --- a/crates/crab-read/README.md +++ b/crates/crab-read/README.md @@ -411,6 +411,12 @@ mandatory. Recovery currently reads complete sources for this strict proof and reads member ranges again for native installation; this is not a request-minimal history-verification claim. +`verify_installed_catalog_dependencies` binds an installed Git database and +its authenticated refs directly to one durable catalog root. Historical checks +and restore's final merged catalog use this owner without constructing another +checkpoint footer or materializing recipes. Callers still authenticate and retain +Git sources separately; an immutable catalog root does not authorize Git objects. + Current-view and historical integrity checks share the installed-database dependency verifier. It validates external catalog bodies, looks up Crab file identities in canonical Xet MerkleHash encoding, and verifies whole-file bytes diff --git a/crates/crab-read/src/capsule_protocol.rs b/crates/crab-read/src/capsule_protocol.rs index dbc2336f4..e6e17274f 100644 --- a/crates/crab-read/src/capsule_protocol.rs +++ b/crates/crab-read/src/capsule_protocol.rs @@ -25,7 +25,7 @@ use crate::{ReadError, Result}; mod dependency; pub use dependency::{ - verify_captured_installed_pointer_bytes, verify_installed_checkpoint_dependencies, + verify_captured_installed_pointer_bytes, verify_installed_catalog_dependencies, verify_installed_dependencies, verify_reachable_dependencies, }; diff --git a/crates/crab-read/src/capsule_protocol/dependency.rs b/crates/crab-read/src/capsule_protocol/dependency.rs index d7316d699..750e39e1e 100644 --- a/crates/crab-read/src/capsule_protocol/dependency.rs +++ b/crates/crab-read/src/capsule_protocol/dependency.rs @@ -5,7 +5,7 @@ use super::{ }; use crate::{ReadError, Result}; use crab_metadata::capsule_protocol::{ - FileCatalogEntry, PointerCatalog, PointerCatalogCounts, PointerCatalogLookup, + CheckpointCatalog, FileCatalogEntry, PointerCatalog, PointerCatalogCounts, PointerCatalogLookup, }; use crab_storage::{Store, StoreLayout}; use std::collections::{BTreeMap, BTreeSet}; @@ -20,8 +20,8 @@ enum DependencyCatalog<'a> { view: &'a CapsuleRepositoryView, lookup: &'a PointerCatalogLookup, }, - Checkpoint { - checkpoint: &'a LayeredCheckpoint, + Durable { + catalog: Option<&'a CheckpointCatalog>, lookup: &'a PointerCatalogLookup, }, } @@ -65,10 +65,10 @@ impl DependencyCatalog<'_> { crate::selection::verify_capsule_pointer_inspection(layout, inspection, cancel) .await } - Self::Checkpoint { checkpoint, .. } => { + Self::Durable { catalog, .. } => { let inspection = crab_metadata::capsule_protocol::inspect_pointer_catalog( layout, - checkpoint.catalog(), + *catalog, &[], &[], crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, @@ -95,7 +95,7 @@ impl DependencyCatalog<'_> { .map(|entry| (hash.clone(), entry.clone())) }) .collect()), - Self::Captured { lookup, .. } | Self::Checkpoint { lookup, .. } => { + Self::Captured { lookup, .. } | Self::Durable { lookup, .. } => { let selected = lookup.files(layout.store(), hashes).await?; Ok(selected.files().clone()) } @@ -240,31 +240,34 @@ pub async fn verify_installed_dependencies( .await } -/// Verify installed Git and external dependencies against one pinned checkpoint. +/// Verify installed Git and external dependencies against one pinned durable catalog. /// -/// Callers authenticate the checkpoint, install its verified packs, validate -/// the supplied ref visibility and retain its sources through completion. -/// The complete catalog is streamed; file bindings use that same checkpoint. +/// Callers authenticate the catalog root, install verified Git packs, validate +/// the supplied ref visibility and retain their sources through completion. +/// The complete catalog is streamed; file bindings use that same root. /// Token cancellation drains the Git worker before returning. -pub async fn verify_installed_checkpoint_dependencies( +pub async fn verify_installed_catalog_dependencies( layout: &StoreLayout, git_dir: &Path, refs: &BTreeMap, - checkpoint: &LayeredCheckpoint, + catalog: Option<&CheckpointCatalog>, limits: crab_git::walk::PointerScanLimits, cancellation: &CancellationToken, ) -> Result { if cancellation.is_cancelled() { return Err(ReadError::Cancelled); } - let lookup = checkpoint - .pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; + let maximum = crab_metadata::capsule_protocol::MAX_CATALOG_BYTES; + let lookup = catalog + .map(|catalog| catalog.lookup(layout, maximum)) + .transpose()? + .unwrap_or_else(|| PointerCatalogLookup::empty(maximum)); verify_installed_dependencies_from( layout, git_dir, refs, - DependencyCatalog::Checkpoint { - checkpoint, + DependencyCatalog::Durable { + catalog, lookup: &lookup, }, limits, diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 44f8c6dc7..609ee2ad6 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -20,6 +20,13 @@ outcome mapping, durable plan attribution, historical reconciliation, readiness, and protected-push request binding. CLI and HTTP callers use these mechanics without delegating their authorization decisions. +`checkpoint::prepare_capsule_catalog_from_view` prepares durable catalog pages +without making a checkpoint visible. Checkpoint publication and historical +restore use this same captured-delta folding owner under caller-held GC admission. +Ref-only preparation retains the existing root without catalog I/O. +`publish_capsule_checkpoint_with_catalog_root_from_view` consumes that prepared +root through the same publication owner, avoiding duplicate delta preparation. + `checkpoint` separates metadata-first logical publication from physical pack maintenance using the layered format only. Foreground admission checkpoints the authenticated source directory; diff --git a/crates/crab-remote/src/checkpoint.rs b/crates/crab-remote/src/checkpoint.rs index 25f49f7b2..6711ff06c 100644 --- a/crates/crab-remote/src/checkpoint.rs +++ b/crates/crab-remote/src/checkpoint.rs @@ -201,7 +201,11 @@ pub async fn maintain_capsule_repository_from_view( publish_checkpoint( layout, view, - catalog_for_checkpoint(layout, view, maximum_bytes).await?, + CatalogWrite::Durable( + prepare_capsule_catalog_from_view(layout, view, maximum_bytes) + .await? + .map(Box::new), + ), maximum_bytes, cancel, ) @@ -299,7 +303,11 @@ pub async fn repack_capsule_repository_from_view( view, consolidation.sources, member_oids, - catalog_for_checkpoint(layout, view, maximum_bytes).await?, + CatalogWrite::Durable( + prepare_capsule_catalog_from_view(layout, view, maximum_bytes) + .await? + .map(Box::new), + ), maximum_bytes, cancel, ) @@ -315,7 +323,11 @@ pub async fn repack_capsule_repository_from_view( publication = publish_checkpoint( layout, view, - catalog_for_checkpoint(layout, view, maximum_bytes).await?, + CatalogWrite::Durable( + prepare_capsule_catalog_from_view(layout, view, maximum_bytes) + .await? + .map(Box::new), + ), maximum_bytes, cancel, ) @@ -361,7 +373,11 @@ pub async fn publish_capsule_checkpoint_from_view( if view.capsule_count()? < u64::from(threshold) { return Ok(CheckpointOutcome::default()); } - let catalog = catalog_for_checkpoint(layout, view, maximum_bytes).await?; + let catalog = CatalogWrite::Durable( + prepare_capsule_catalog_from_view(layout, view, maximum_bytes) + .await? + .map(Box::new), + ); Ok( publish_checkpoint(layout, view, catalog, maximum_bytes, cancel) .await? @@ -393,16 +409,43 @@ pub async fn publish_capsule_checkpoint_with_catalog_from_view( .work) } +/// Publish one checkpoint with a durable catalog prepared from the same captured view. +/// +/// The caller verifies the catalog's complete dependency closure and retains GC +/// admission through visibility. Reusing prepared pages avoids folding captured +/// deltas a second time; publication still compares the exact root and ref heads. +pub async fn publish_capsule_checkpoint_with_catalog_root_from_view( + layout: &StoreLayout, + view: &crab_read::capsule_protocol::CapsuleRepositoryView, + catalog: Option, + maximum_bytes: u64, + cancel: &CancellationToken, +) -> Result { + Ok(publish_checkpoint( + layout, + view, + CatalogWrite::Durable(catalog.map(Box::new)), + maximum_bytes, + cancel, + ) + .await? + .work) +} + enum CatalogWrite { New(PointerCatalog), Durable(Option>), } -async fn catalog_for_checkpoint( +/// Prepare durable catalog pages for one captured maintenance view without publishing a checkpoint. +/// +/// The caller retains the view's sources under GC admission until publication. +/// New pages are verified before returning; a ref-only capture reuses its root. +pub async fn prepare_capsule_catalog_from_view( layout: &StoreLayout, view: &crab_read::capsule_protocol::CapsuleRepositoryView, maximum_bytes: u64, -) -> Result { +) -> Result, CheckpointError> { if view.has_catalog_deltas()? { let delta = view.visible_pointer_catalog_delta().await?; let existing = view @@ -419,16 +462,14 @@ async fn catalog_for_checkpoint( }, ) .await?; - return Ok(CatalogWrite::Durable(catalog.map(Box::new))); + return Ok(catalog); } // The checkpoint freezes prior replay precedence. Reuse only when every // newly selected control lacks a catalog delta, even if other refs moved. - Ok(CatalogWrite::Durable( - view.layered_checkpoint() - .and_then(LayeredCheckpoint::catalog) - .cloned() - .map(Box::new), - )) + Ok(view + .layered_checkpoint() + .and_then(LayeredCheckpoint::catalog) + .cloned()) } async fn publish_checkpoint(