diff --git a/Cargo.lock b/Cargo.lock index aa7fddfa3..dc6ffd938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2539,6 +2539,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.10.6", + "slatedb", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/crab/docs/architecture/git-protocol-v2.md b/crab/docs/architecture/git-protocol-v2.md index e188bda4c..24d6f8bcb 100644 --- a/crab/docs/architecture/git-protocol-v2.md +++ b/crab/docs/architecture/git-protocol-v2.md @@ -55,9 +55,8 @@ the invocation rather than allowing every protocol globally: git -c protocol.crab.allow=always clone --recurse-submodules crab://bucket/team/repo ``` -"All commands" is not a support claim. Date/ref-exclusion shallow selectors -remain unimplemented; signed push certificates, push options intended for -server-side hooks, and remote archive service need separate protocol contracts. +"All commands" is not a support claim. Signed push certificates, push options +intended for server-side hooks, and remote archive service need separate protocol contracts. Signed commits/tags are Git objects and are distinct from signed push requests. PRs, CI execution, and mandatory server-side hooks cannot execute inside a passive bucket. Clients with unrestricted bucket write credentials can bypass @@ -76,7 +75,7 @@ select a v2 authority. The session supports: - `ls-refs` with ref prefixes, symrefs, peeled tags, unborn HEAD, and hidden refs; - `fetch` with wants, haves, done, tags, sideband/progress, shallow/deepen, - and `deepen-relative`; + `deepen-relative`, `deepen-since`, and `deepen-not`; - the bounded filter matrix: `blob:none`; `blob:limit=[kmg]`; `tree:`; `object:type={tag,commit,tree,blob}`; `sparse:oid=`; and repeated filters or percent-encoded `combine:` intersections; @@ -85,6 +84,12 @@ select a v2 authority. The session supports: required or assumed; - standard local Git pack installation on the v2 path. +The line helper normalizes `--shallow-since` date expressions through Git's +date parser before passing epoch seconds to the shared planner. Date and +ref-exclusion selectors apply to requested tips as well as their parents. +If no commits qualify, fetching fails before pack installation or shallow-file +publication; it does not substitute a depth-one checkout. + An empty manifest is a valid ref snapshot without a catalog. Empty clones and `ls-remote` complete through v2; clients requesting `unborn` receive the configured HEAD target unless hidden. Object wants against that empty snapshot @@ -229,7 +234,7 @@ takeover. ## Explicitly unsupported The first profile rejects stateful `connect`, `packfile-uris`, `object-info`, -`ref-in-want`, date/ref-exclusion shallow selectors, and other capabilities not +`ref-in-want`, and other capabilities not listed above. A v2 session cannot fall back to the line protocol after the terminal handoff; it reports the protocol failure instead. @@ -249,6 +254,13 @@ filter reads must leave canonical object counts and bytes unchanged; generated response-pack cache and coordination objects are accounted for separately. This is not a claim that first-read repair works with read-only object-store credentials. +The add/commit/push RustFS smoke exercises the current physical fault boundary: +loss of a published capsule and corruption confined to its embedded Git pack +or index range. Each rejection probe uses an empty cache and requires no local +HEAD or checked-out payload. It compares the remote root and ref-head bytes +before and after the fault; discovery can itself reject a damaged capsule, +so successful `ls-remote` is not the mutation oracle. + Direct pushes and protected/service publication paths publish visibility before the corresponding ref or manifest commit. Protected receive extracts the closure from the already-verified materialization ODB before that workspace diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 1b57dd83b..866cc3b3d 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -69,7 +69,9 @@ counts contain no raw paths, OIDs, endpoints, content or credentials; separate oracle evidence contains the paths/OIDs needed to prove correctness. `/usr/bin/time` supplies process CPU and peak RSS. Those cover opening and both -queries together; they are not per-query RSS. Index construction uses an explicit +queries together; they are not per-query RSS. Catalog qualifier totals also +include the complete inspection oracle, so they cannot establish selected-query +memory bounds. Index construction uses an explicit `build-indexes` operation, outside measured reads, with its own requests, bytes, CPU and peak RSS. Read-only qualifications do not trigger maintenance. @@ -2188,11 +2190,212 @@ two path heads, two leases, 65 payload records and 4,194,304 retained segment bytes remain. These receipts establish the oracle and staging component; they do not qualify the new integrated binary or catalog format. -Combined-source CLI compilation and runtime qualification are still pending. -Remaining catalog consumers include shared/protected preparation, pinned -hydration/mount lookup and the current-state file lookup opener. Complete +The combined release binary compiled and the focused RustFS smoke passed +163 commands and 84 checks, including exact retirement, committed restaging +before first push, both cold versions, force/ref updates and invalid roots. +The integrated native-first entropy 10-GiB run passes 75 commands and 57 checks: +all four pushes, exact retirement, eight cold historical SHA-256 checks, Git +integrity and remote fsck with zero errors or repairs. +Those receipts bind the earlier integrated source, not subsequent reader changes. + +Pinned hydration/mount lookup uses captured catalog pages in PR #581. PR #584 +extends this to current-state file lookup and shares ref ordering and captured +visibility with Git readers. Metadata tests pass (458), as do all 47 read capsule-protocol tests after +preserving corruption classification at the read boundary. CLI, runtime and +retained-root lifecycle checks are required before readiness. +A reproduced independent-ref fixture also requires complete inspection to +validate dependencies after assembling the visible set, while publication-time +delta validation remains strict. + +`qualify_remote ... catalog-current` measures the actual +current-state lookup session. The argument uses Xet `MerkleHash::hex()` encoding, +not the pointer wire format’s raw-byte hex. Hydration converts parsed pointer +bytes with `MerkleHash::from(pointer.file_hash)` before lookup; the qualifier +accepts that catalog identity directly. Opening, cold queries and warm queries are separate; the full +inspection oracle runs after measured queries and explicit session close. This +keeps unrelated catalog decoding out of cold-query preparation. Use a quiescent, +dedicated fixture so the post-query full oracle has the same visible authority. +Compare these +receipts across fixed small edits and increasing distinct historical chunks, +ref counts and uncompacted history; checkpoint row bounds alone are insufficient. + +Shared/protected preparation remains a separate integration boundary. Complete inspection remains appropriate for fsck, compaction and GC; foreground lookups -must not inherit those scans. Carry forward applicable CLI range/cancellation -contracts before final qualification. The complete integrated 10-GiB matrix, -history scaling, lifecycle/migration, fuller blame and representative S3 gates -remain open. +must not inherit those scans. CLI snapshot reads still install all Git packs; +carry forward the range/cancellation and cleanup contracts when removing that +cost. The final combined 10-GiB matrix, history scaling, lifecycle/migration, +fuller blame and representative S3 gates remain open. + +The GC regression passes with both existing cleanup tests (three tests total). +It captures a 20,000-file checkpoint catalog without a file query, replaces it, +and sweeps past the grace period. Actual orphan deletion is required; the old +reader then fetches cold pages and resolves duplicated file requests, while a +new session observes the replacement. Both checkpoints have a valid physical +source descriptor/body. This is an in-memory metadata lifecycle proof, separate +from payload hydration and concurrent real-store qualification. + +Current-state qualification against the four-version, two-file 10-GiB entropy +history passes with the updated reader. Opening uses 28 backend operations and +149,860 bytes. The duplicate-file cold query uses 497 range reads and 7,434,846 +bytes; the warm query repeats all 497 reads. Both results match full inspection, +operations drain and no writes occur. These are one local debug-profile sample, +not cloud latency or independent history-scaling proof. Repeated recipe metadata +reads are a measured remaining performance gap, not a successful warm-cache gate. +The first attempted receipt remains failed: it supplied pointer-wire hex where +the qualifier requires the distinct catalog `MerkleHash::hex()` representation. + +A differential run of the same qualifier binary against a real 4-MiB file passes: +five cold ranges / 3,755 bytes and zero warm backend operations. Combined with +the unchanged captured lookup and the existing 256-page / 8-MiB FIFO page cache, +this supports recipe-page eviction as the cause of repeated large-file warm +reads. A bounded cache of already-verified file bindings is the next candidate; +it must preserve the initial complete dependency proof, immutable capture, +ordered duplicates/misses, error retry and shared close/cancellation contracts. + +The current reader's minimal CLI/NFS check and debug build pass. The force-push +policy/visibility regression also passes after awaiting the now-async visibility +API in its test caller; all six policy cases retain their assertions. Eleven +Python fixture tests pass. The previously qualified staging-version assertion +from the combined branch is carried forward with exact post-push retirement. + +The complete current-source RustFS CLI smoke remains failed (152 commands, +61 checks). Its shallow-since case supplies an ISO date; the helper treats the +remote-helper option as an integer wire timestamp and rejects it before clone. +The harness still expects the earlier unsupported-option diagnostic. Git's +remote-helper boundary passes date text through, while wire fetch uses a +normalized timestamp. Fixing that adapter and qualifying dated boundaries +against native Git are required; changing the expected message alone would +hide the missing behavior. No full CLI E2E pass is claimed from this receipt. + + +## Capsule chunk authority and exact staged reuse in progress + +PR #593 publishes rebuildable global chunk candidates after capsule ref +publication. Each candidate names a source shard in the source repository's +current authenticated catalog and a canonical origin receipt in the caller's +trusted global scope. A physical xorb alone is insufficient: its catalog shard +must retain the payload through the existing GC contract. Direct CLI publication +and protected receive share the metadata encoder and writer. Protected receive's +80 native-Git/in-memory tests pass; this is not real-S3 qualification. + +The follow-up stores each placement under its immutable digest as well as the +replaceable chunk head. Push re-reads the exact add-time placement and origin +IDs, validates source membership, and requires current provider identity even +with warm payload caches. Its read-only proof session closes before publication +opens the global writer. Verified source descriptors and original receipts stay +caller-owned, with one retained-metadata byte allowance across source repositories. +Conflicting descriptors or physical assertions fail explicitly. + +A regression measured one redundant full xorb GET after otherwise successful +source/origin proof. Adopting the authenticated descriptor removes that GET: +the first before/after pair fails at one versus zero body reads, then passes the +unchanged zero-read bound. Final-source validation passes all 10 focused CLI/Xet tests, including partial +add-time hints and mixed stale/fresh origin assertions. The cold source fixture +uses native Git and real staging with an in-memory object store. +Released v1.2.4 manifest receipts lack these capsule descriptors and still require +full-body verification when imported into capsule publication. + +The dedicated cross-repository smoke oracle now requires one complete capsule +source-proof event and its matching adoption event. It checks exact planned, +indexed, matched, source-verified and payload-verified counts, with zero stale, +mismatched or globally reclassified chunks. Five oracle tests pass; the original +substring oracle rejected valid capsule proofs and accepted counters assembled +from different events. This representation correction is not product E2E proof. +Payload-only run tests pass (28), candidate/key-codec/writer tests pass (19), +existing origin/range siblings pass (12), and strict minimal and remote-index +metadata Clippy pass. The production CLI binary build also passes. + +The current-source RustFS cross-repository fixture passes 34 commands and 32 +checks at 4 MiB (`capsule-exact-proof-r2-c6a202ae`, code head `098859c02f1`). +The source worktree is clean, and its recorded source hashes remain unchanged +throughout the run. Consumer add retains neither segment nor prepared payload; +push revalidates all 58 exact placements with zero stale, mismatched or globally +reclassified chunks and uploads no new xorb. Add and push each read two xorb +ranges totaling 2,658 bytes (footer and chunk metadata), with no full xorb GET. +A separate empty clone cache then performs a real remote payload GET during +hydration and produces the original SHA-256. This is one local debug-profile +correctness fixture, not a latency or history-scaling qualification. + +The binary SHA-256 is +`5002c1c59f8e2d6d3fa82abed1c5686c67d46feb93f975c0c1d76c1930f3dc70`; +the report SHA-256 is +`36d6ec1b744a2f5545a9a998710271bc2fa092ceadf756dd56c95acf75eeff2f`. + +The shared `crab-remote` preparation consumer, final 10-GiB qualification, +history scaling, provider latency, lifecycle and full blame semantics remain +open gates. + + +## Shared capsule preparation in progress + +The shared `Prepared::upload_capsule` caller in HTTP receive now uses captured +file, shard and xorb lookups instead of materializing unrelated historical +catalogs. An ordinary Git upload with no new Crab pointer performs zero catalog +reads. Selected pointer and prepared-content closures preserve size, membership +and descriptor conflict checks. Capsule-specific implementation lives in +`crates/crab-remote/src/prepare/capsule.rs`; its public API remains under +`crab_remote::prepare`. + +Four regressions failed before the change and pass afterward: one redundant +historical-catalog read for a Git-only upload; Xet-word-order formatting of a raw +body digest; acceptance of rewritten xorb/shard scratch bodies after inspection; +and no shared chunk candidate index after a successful native-content commit. +The initial Git-only fixture sent an all-history pack containing an old pointer; +its corrected incremental pack excludes the advertised tip before measuring. +No weakened assertion or production failure is inferred from that fixture issue. + +Six public-API integration tests and strict all-target Clippy pass. The tests use +native Git packs, real Xet artifacts, explicit publication leases, exact origin +and source-anchor checks, complete file reconstruction, and a noncanonical index +fault that must leave committed refs successful. The SlateDB test dependency uses +the same pinned 0.15.0 release already used by metadata; the lockfile adds only +that test dependency edge. Production dependency versions do not change. + +The dedicated RustFS public-API test passes for both newly created and occupied +xorb keys. It uses native Git preparation, real S3 publication, cold source and +origin-anchor checks, and complete file-hash reconstruction. The historical +pointer integration also accepts the correct declared size and rejects a +conflicting one. The final source passes strict all-target Clippy and the +no-default-feature check. + +Eight HTTP receive/fault tests pass. Three dedicated real-RustFS cells also pass: +the public native-content API, native Git push/fetch and browser editing over the +HTTP listener, and the six receive-fault/restart scenarios. Source hashes remain +unchanged throughout these runs. The API test-binary SHA-256 is +`fe8a75004fd4ef7ec0946b999d32a829b83b28f94b51317a9c34e9990e9ebf5d`; the HTTP test-binary SHA-256 is +`92318648278160dd57f96f35dbcffc01f44799a75a43b84442251965c07fef51`. These are small correctness fixtures, separate from the required +large-file, growth and latency cells. + +The final combined 10-GiB matrix, unrelated-history/ref/frontier scaling, broader +lifecycle/provider proof and fuller blame semantics still prevent completion. + + +## Combined catalog/staging qualification and HTTP fault boundaries + +The combined revision carries selected publication, verified catalog bindings, +source/browse retention, retained-source fsck, exact capsule chunk reuse and +shared native-content preparation. Focused checks passed: 558 metadata tests, +81 protected-receive tests, six shared-preparation tests, 242 staging tests, +26 CLI integration tests, 57 CLI siblings, 20 origin/lease tests and eight HTTP +receive/fault tests. Explicit live tests remain separately qualified; these +counts do not establish the full release gate. + +The first combined RustFS native HTTP fixture returned 202 during its +corrupt-descriptor case, where the corruption oracle required 503. Readiness +can become visible before the publishing repair worker finishes. A diagnostic +assertion with a one-second delayed generation-owner release response proved +that the fixture could inject its next fault while that worker was active. +The shared native HTTP fixture now awaits maintenance completion before each +fault, reloads the current descriptor and checks its snapshot identity. All +original status, error-code, recovery and Git assertions remain intact. The +controlled probe then passed across ten delayed lease releases, and all eight +HTTP unit/fault checks passed. This changes test synchronization, not the valid +production distinction between incomplete readiness and corrupt metadata. + +Raw before/after evidence remains outside the checkout in +`final-integration-http-metadata-probe-r4.json`, +`final-integration-http-metadata-probe-r5.json` and +`http-fault-boundary-after-r1.log`. Combined-source 10 GiB qualification, +representative S3 latency, additional history/ref scaling and fuller Git blame +semantics remain open gates. Earlier component receipts retain their original +source identities and are not final combined-source proof. diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index eda232407..16fbe9e6e 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -203,6 +203,42 @@ SlateDB contains only committed, origin-bound receipts and is updated after a successful push CAS; it never stores add claims, local paths, or pending uploads. +Capsule pushes publish chunk hints into the same shared database after ref +visibility. Their head/source keys are separate from the released manifest +receipt keys. A source assertion carries the whole-xorb catalog digest and an +origin receipt captured from the verified PUT or GET; the receipt's payload +digest remains distinct. Readers batch requested chunks, authenticate the +source repository's current paged catalog, check each exact chunk position and +size, then verify canonical origin identity. An absent source or changed +physical descriptor cannot turn an index hint into authority. + +Each capsule placement is also stored under its immutable placement digest. +Staged recipes retain that digest and the origin proof ID, so replacing a shared +chunk head does not erase an add's chosen assertion. Capsule push re-reads those +exact records, revalidates the source shard, and requires the recorded physical +origin identity before adopting remote-only chunks. Warm payload/index caches +cannot substitute for that identity check. The proof session is read-only and +closed before publication opens the shared index writer. + +The revalidator returns caller-owned source descriptors and original receipts +with the accepted candidates. Capsule preparation adopts those descriptors +without a full xorb GET or decompression. Retained metadata shares one catalog +byte allowance across source repositories; conflicting physical assertions fail +explicitly. Released manifest receipts still use full-body verification when +constructing a capsule descriptor that their older record does not contain. + +Canonical origin receipts belong to the trusted `global_prefix` of the same +authorization scope as the repository. Managed and ACL-scoped views use their +own supplied global root. Local staging and advisory caches only name candidates; +they cannot issue canonical receipts or certify an origin identity. + +Direct publication and protected receive use the same bounded candidate +encoder/writer. Protected promotion carries the canonical operation identity; +client staging-prefix writes do not confer canonical origin proof. Every index +writer is explicitly closed, including batch failures. A post-commit index +failure leaves a successful ref publication visible and reports repair work; +local packing remains available for future lookup misses. + Concurrent add lookups own their receipt snapshot and validation result across remote awaits. They share keyed advisory caches, not the push pipeline's mutable per-push proof accumulator; otherwise one file can erase another's @@ -224,6 +260,11 @@ repacking; push still revalidates every positive placement and origin proof. Cache failures, stale entries, and evictions are advisory misses; they cannot change the bytes selected for a push. +An uninitialized global chunk index is remembered only for the current add +session. It does not establish per-chunk absence, so it does not populate the +persistent negative cache. Existing index snapshots keep their normal cache +behavior; unavailable remote authority still leaves payload ownership local. + Push reads remote candidates in bounded pages under a shared deadline. It retains completed pages and local cache hits if the deadline expires or a later remote page fails; every retained placement still crosses normal proof diff --git a/crab/docs/guides/fsck.md b/crab/docs/guides/fsck.md index e07f692c3..0312f9136 100644 --- a/crab/docs/guides/fsck.md +++ b/crab/docs/guides/fsck.md @@ -13,6 +13,12 @@ crab fsck [OPTIONS] `crab fsck` cross-checks Crab manifests, pack/index presence, the shard/xorb data chain, and coordination state against the object store. +For protocol-v2 repositories, the check pins an authenticated root, verifies +Git connectivity, and follows every reachable Crab or Git LFS pointer to its +origin payload. Crab files must reconstruct to the pointer's exact Blake3 hash +and size; LFS objects must match their SHA-256 identity and size. Full checking +reads the payload bytes, even when a local cache already contains them. + With `--repair`, it can automatically fix certain categories of issues. ## Options @@ -104,7 +110,7 @@ crab fsck: 1 error(s), 0 info, 1 repaired, 0 repair failure(s) ## Verification Boundaries -- The Git-object check currently confirms that ref-journal records are +- For legacy layouts, the Git-object check confirms that ref-journal records are readable; it does not reconstruct the current repository and run full Git reachability for missing commits, trees, or blobs. Use `crab recover history verify ` for strict Git fsck of a selected historical root, and diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index b79cbe9b9..fd9cf3c17 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -10,7 +10,7 @@ clone, hydration, and byte identity. A separate ordinary-Git matrix proves missing/corrupt metadata-root handling, update/fetch, deletion, force and non-fast-forward behavior, atomic rejection, tag following, shallow history, -immutable-pack failures, and concurrent push serialization. +capsule loss, embedded pack/index corruption, and concurrent push serialization. With ``--source``, run only the two managed-file workflows on disposable clones of a read-only real repository. Use a run root on the workspace volume @@ -102,6 +102,44 @@ class SmokeReport: updated_at: str = "" +def capsule_staged_proof_complete(stderr: str, count: int) -> bool: + """Require complete direct reuse evidence for the capsule duplicate fixture.""" + if type(count) is not int or count <= 0: + return False + proof_message = "revalidated staged remote placement proofs" + adoption_message = "capsule staged remote proof adoption complete" + events = [] + for line in stderr.splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + fields = record.get("fields") if isinstance(record, dict) else None + if isinstance(fields, dict) and fields.get("message") in ( + proof_message, adoption_message + ): + events.append(fields) + # This fixture makes one push with one direct-proof batch. Counts from + # different batches cannot jointly prove that every planned chunk survived. + if len(events) != 2: + return False + proof, adoption = events + if proof.get("message") != proof_message or adoption.get("message") != adoption_message: + return False + expected = { + "planned": count, "indexed": count, "matched": count, + "capsule_source_verified": count, "generation_verified": 0, + "payload_verified": count, "placement_mismatches": 0, + "origin_proof_mismatches": 0, + } + adopted = {"planned_existing": count, "stale_existing": 0, "global_existing": 0} + return all( + type(event.get(key)) is int and event[key] == value + for event, counters in [(proof, expected), (adoption, adopted)] + for key, value in counters.items() + ) + + def utc_now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -205,6 +243,31 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def capsule_member_range(body: bytes, kind: str) -> tuple[int, int]: + """Locate one pack/index fault inside the synthetic single-member run.""" + if len(body) < 48 or body[-8:] != b"CRBRUN01" or kind not in ("pack", "index"): + raise SmokeError("fault fixture is not a capsule run with a pack/index target") + footer_length = int.from_bytes(body[-48:-40], "big") + footer_start = len(body) - 48 - footer_length + if not 0 < footer_length <= 8 * 1024 * 1024 or footer_start < 0: + raise SmokeError("fault fixture footer is outside its container") + try: + footer = json.loads(body[footer_start:-48]) + members = footer["git_packs"] + if footer["version"] != 4 or len(members) != 1: + raise SmokeError("fault fixture requires one current-format pack member") + offset, length = members[0][kind]["offset"], members[0][kind]["length"] + except (KeyError, TypeError, ValueError) as exc: + raise SmokeError("fault fixture has no usable pack member") from exc + if (type(offset) is not int or type(length) is not int + or offset < 0 or length < 4 or offset + length > footer_start): + raise SmokeError("fault fixture range overlaps control bytes or escapes its container") + magic = b"PACK" if kind == "pack" else b"\xfftOc" + if body[offset:offset + 4] != magic: + raise SmokeError("fault fixture member does not contain the selected Git format") + return offset, length + + def parse_pointer(blob: str) -> PointerRecord: raw = blob.encode("utf-8") if len(raw) > 256: @@ -1097,53 +1160,81 @@ def run_atomic_rejection_case(self) -> None: def run_shallow_case(self) -> None: case_name = "shallow-history" repo, remote_url, _repo_prefix = self.prepare_git_repo(case_name) - first = self.commit_text(repo, "history.txt", "one\n", "one") - self.commit_text(repo, "history.txt", "two\n", "two") - self.commit_text(repo, "history.txt", "three\n", "three") - self.run_git(repo, ["push", "-u", "origin", "main"]) - - selector_clone = self.run_root / case_name / "shallow-since-clone" - selector_result = self.clone_git( - remote_url, - selector_clone, - name=f"{case_name} unsupported shallow-since clone", - extra_args=["--shallow-since", "2100-01-01T00:00:00Z"], - check=False, - ) - selector_stderr = Path(selector_result.stderr_log).read_text( - encoding="utf-8", errors="replace" - ) - self.check( - f"{case_name}-shallow-since-fails-explicitly", - selector_result.exit_code != 0 - and "deepen-since is not supported" in selector_stderr - and not (selector_clone / "history.txt").exists(), - { - "exit_code": selector_result.exit_code, - "stderr_log": selector_result.stderr_log, - }, - ) - exclude_clone = self.run_root / case_name / "shallow-exclude-clone" - exclude_result = self.clone_git( - remote_url, - exclude_clone, - name=f"{case_name} unsupported shallow-exclude clone", - extra_args=["--shallow-exclude", "main"], - check=False, - ) - exclude_stderr = Path(exclude_result.stderr_log).read_text( - encoding="utf-8", errors="replace" - ) - self.check( - f"{case_name}-shallow-exclude-fails-explicitly", - exclude_result.exit_code != 0 - and "deepen-not is not supported" in exclude_stderr - and not (exclude_clone / "history.txt").exists(), - { - "exit_code": exclude_result.exit_code, - "stderr_log": exclude_result.stderr_log, + commits = [] + for index, text in enumerate(("one", "two", "three"), start=1): + (repo / "history.txt").write_text(text + "\n", encoding="utf-8") + self.run_git(repo, ["add", "history.txt"]) + date = f"2023-11-{14 + index:02d}T22:13:20Z" + self.run_cmd( + f"{case_name} dated commit {text}", + ["git", "commit", "-m", text], + repo, + extra_env={"GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}, + ) + commits.append(self.rev_parse(repo, "HEAD")) + first = commits[0] + # Exclude the first commit through a side tip, keeping that ancestor + # unadvertised for the raw-SHA policy check below. + archive = self.read_stdout(self.run_cmd( + f"{case_name} archive side tip", + ["git", "commit-tree", f"{first}^{{tree}}", "-p", first, "-m", "archive"], + repo, + extra_env={ + "GIT_AUTHOR_DATE": "2023-11-15T23:13:20Z", + "GIT_COMMITTER_DATE": "2023-11-15T23:13:20Z", }, - ) + )) + self.run_git(repo, ["tag", "archive", archive]) + self.run_git(repo, ["push", "-u", "origin", "main", "--tags"]) + + # Match native upload-pack before asserting Crab's selector behavior. + # Fixed dates separate date parsing from commit-boundary selection. + selectors = ( + ("since", ["--shallow-since", "2023-11-16T22:13:20Z"]), + ("exclude", ["--shallow-exclude", "archive"]), + ("future", ["--shallow-since", "2030-01-01T00:00:00Z"]), + ("exclude-all", ["--shallow-exclude", "main"]), + ) + for selector, arguments in selectors: + expected_success = selector in ("since", "exclude") + transports = ( + ("native", repo.resolve().as_uri(), []), + ("crab-v2", remote_url, ["--config", "protocol.version=2"]), + ("crab-classic", remote_url, ["--config", "protocol.version=0"]), + ) + for transport, url, protocol in transports: + clone = self.run_root / case_name / f"{transport}-{selector}-clone" + result = self.clone_git( + url, clone, + name=f"{case_name} {transport} {selector} clone", + extra_args=[*protocol, *arguments], check=False, + ) + detail = {"exit_code": result.exit_code, "stderr_log": result.stderr_log} + if not expected_success: + diagnostic = Path(result.stderr_log).read_text( + encoding="utf-8", errors="replace" + ) + self.check( + f"{case_name}-{transport}-{selector}-rejects-empty-selection", + result.exit_code != 0 + and "no commits selected for shallow requests" in diagnostic + and not (clone / "history.txt").exists(), + detail, + ) + continue + self.check( + f"{case_name}-{transport}-{selector}-clone-succeeds", + result.exit_code == 0, detail, + ) + selected = self.read_stdout(self.run_git(clone, ["rev-list", "HEAD"])) + boundary = (clone / ".git" / "shallow").read_text().splitlines() + self.check( + f"{case_name}-{transport}-{selector}-exact-history-and-boundary", + selected.splitlines() == list(reversed(commits[1:])) + and boundary == [commits[1]] + and (clone / "history.txt").read_text() == "three\n", + {"commits": selected.splitlines(), "boundary": boundary}, + ) raw_fetch = self.run_root / case_name / "raw-fetch" raw_fetch.mkdir() @@ -1286,122 +1377,74 @@ def run_corrupt_root_case(self) -> None: {"stderr_log": clone.stderr_log}, ) - def run_immutable_object_failure_case(self, object_kind: str) -> None: - case_name = f"missing-{object_kind}" + def run_capsule_failure_case(self, fault: str) -> None: + case_name = "missing-capsule" if fault == "missing" else f"corrupt-{fault}" repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) remote_tip = self.commit_text(repo, "payload.txt", "immutable\n", "immutable") self.run_git(repo, ["push", "-u", "origin", "main"]) - keys = self.list_keys(repo_prefix) - suffix = f".{object_kind}" - candidates = sorted(key for key in keys if key.endswith(suffix)) - self.check( - f"{case_name}-fixture-object-found", - bool(candidates), - {"suffix": suffix, "keys": sorted(keys)}, - ) - target_key = candidates[0] - status, _, body = self.signed_s3_request("DELETE", target_key) self.check( - f"{case_name}-fixture-object-deleted", - status in (200, 204), - { - "status": status, - "key": target_key, - "body": body.decode("utf-8", errors="replace")[:200], - }, - ) - - clone_dir = self.run_root / case_name / "clone" - clone = self.clone_git( - remote_url, clone_dir, name=f"{case_name} git clone", check=False - ) - head = self.run_git( - self.run_root, - ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], - name=f"{case_name} verify no local ref update", - check=False, - ) - self.check( - f"{case_name}-clone-fails-before-checkout", - clone.exit_code != 0 - and head.exit_code != 0 - and not (clone_dir / "payload.txt").exists(), - { - "clone_exit_code": clone.exit_code, - "head_exit_code": head.exit_code, - "target_key": target_key, - }, - ) - refs = self.ls_remote(remote_url, name=f"{case_name} refs remain coherent") - self.check( - f"{case_name}-remote-ref-remains-original", - refs.get("refs/heads/main") == remote_tip, - {"expected": remote_tip, "actual": refs.get("refs/heads/main")}, - ) - - def run_corrupt_index_failure_case(self) -> None: - case_name = "corrupt-idx" - repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) - remote_tip = self.commit_text(repo, "payload.txt", "immutable\n", "immutable") - self.run_git(repo, ["push", "-u", "origin", "main"]) - candidates = sorted( - key for key in self.list_keys(repo_prefix) if key.endswith(".idx") - ) - self.check( - f"{case_name}-fixture-object-found", - bool(candidates), - {"suffix": ".idx", "keys": candidates}, + f"{case_name}-healthy-tip-before-fault", + self.ls_remote(remote_url, name=f"{case_name} healthy refs").get("refs/heads/main") == remote_tip, ) + keys = self.list_keys(repo_prefix) + candidates = sorted(key for key in keys if key.startswith(f"{repo_prefix}/v2/capsules/")) + self.check(f"{case_name}-fixture-object-found", len(candidates) == 1, {"keys": candidates}) target_key = candidates[0] - status, _, body = self.signed_s3_request( - "PUT", target_key, body=b"not a valid git pack index\n" - ) - self.check( - f"{case_name}-fixture-object-corrupted", - status in (200, 201), - { - "status": status, - "key": target_key, - "body": body.decode("utf-8", errors="replace")[:200], - }, - ) - + authority = {} + for key in sorted(key for key in keys if key == f"{repo_prefix}/v2/root" or key.startswith(f"{repo_prefix}/v2/refs/")): + status, _, value = self.signed_s3_request("GET", key) + self.check(f"{case_name}-authority-readable-before-fault", status == 200, {"key": key}) + authority[key] = value + status, headers, original = self.signed_s3_request("GET", target_key) + self.check(f"{case_name}-fixture-body-readable", status == 200 and len(original) < 1024 * 1024) + if fault == "missing": + status, _, _ = self.signed_s3_request("DELETE", target_key) + absent, _, _ = self.signed_s3_request("GET", target_key) + self.check(f"{case_name}-fixture-object-deleted", status in (200, 204) and absent == 404) + else: + offset, length = capsule_member_range(original, fault) + damaged = bytearray(original) + damaged[offset] ^= 1 + status, _, _ = self.signed_s3_request( + "PUT", target_key, body=bytes(damaged), extra_headers={"if-match": headers["etag"]}, + ) + read_status, _, stored = self.signed_s3_request("GET", target_key) + self.check( + f"{case_name}-fixture-member-corrupted", + status in (200, 201) and read_status == 200 and stored == bytes(damaged), + {"key": target_key, "member_offset": offset, "member_length": length}, + ) clone_dir = self.run_root / case_name / "clone" - clone = self.clone_git( - remote_url, clone_dir, name=f"{case_name} git clone", check=False + clone_cache = self.run_root / case_name / "cold-cache" + self.check(f"{case_name}-clone-cache-starts-empty", not clone_cache.exists()) + clone = self.run_cmd( + f"{case_name} git clone", ["git", "clone", remote_url, str(clone_dir)], self.run_root, + check=False, extra_env={"CRAB_CACHE_DIR": str(clone_cache)}, ) head = self.run_git( - self.run_root, - ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], - name=f"{case_name} verify no local ref update", - check=False, - ) - clone_stderr = Path(clone.stderr_log).read_text( - encoding="utf-8", errors="replace" + self.run_root, ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], + name=f"{case_name} verify no local ref update", check=False, ) self.check( f"{case_name}-clone-fails-before-checkout", - clone.exit_code != 0 - and head.exit_code != 0 - and not (clone_dir / "payload.txt").exists(), - { - "clone_exit_code": clone.exit_code, - "head_exit_code": head.exit_code, - "target_key": target_key, - }, - ) - self.check( - f"{case_name}-clone-surfaces-integrity-error", - "CRAB-E0020" in clone_stderr - or "corrupt object" in clone_stderr.lower(), - {"stderr_log": clone.stderr_log}, - ) - refs = self.ls_remote(remote_url, name=f"{case_name} refs remain coherent") - self.check( - f"{case_name}-remote-ref-remains-original", - refs.get("refs/heads/main") == remote_tip, - {"expected": remote_tip, "actual": refs.get("refs/heads/main")}, + clone.exit_code != 0 and head.exit_code != 0 and not (clone_dir / "payload.txt").exists(), + {"clone_exit_code": clone.exit_code, "head_exit_code": head.exit_code, "target_key": target_key}, ) + if fault != "missing": + diagnostic = Path(clone.stderr_log).read_text(encoding="utf-8", errors="replace") + self.check( + f"{case_name}-clone-surfaces-integrity-error", + "CRAB-E0020" in diagnostic or "corrupt object" in diagnostic.lower(), + {"stderr_log": clone.stderr_log}, + ) + # Payloads share their physical object with discovery controls. Compare + # authority bytes directly; failed discovery cannot prove unchanged refs. + after = {} + for key in authority: + status, _, value = self.signed_s3_request("GET", key) + self.check(f"{case_name}-authority-readable-after-fault", status == 200, {"key": key}) + after[key] = value + self.check(f"{case_name}-remote-authority-unchanged", after == authority) def run_concurrent_push_case(self) -> None: case_name = "concurrent-cas" @@ -2026,13 +2069,7 @@ def run_cross_repository_remote_duplicate_case(self) -> None: remote_chunk_count = inventory["recipe_remote_chunks"] self.check( f"{case_name}-push-revalidates-staged-proof-directly", - "revalidated staged remote placement proofs" in push_stderr - and f'"planned":{remote_chunk_count}' in push_stderr - and f'"matched":{remote_chunk_count}' in push_stderr - and f'"generation_verified":{remote_chunk_count}' in push_stderr - and f'"payload_verified":{remote_chunk_count}' in push_stderr - and '"stale_existing":0' in push_stderr - and '"global_existing":0' in push_stderr, + capsule_staged_proof_complete(push_stderr, remote_chunk_count), {"remote_chunks": remote_chunk_count}, ) self.head_repository_state(consumer_prefix) @@ -2047,12 +2084,19 @@ def run_cross_repository_remote_duplicate_case(self) -> None: ) clone_dir = self.run_root / case_name / "consumer-clone" + clone_cache = self.run_root / case_name / "consumer-clone-cache" + self.check(f"{case_name}-clone-cache-starts-empty", not clone_cache.exists()) + clone_env = {"CRAB_CACHE_DIR": str(clone_cache)} self.run_cmd( f"{case_name} consumer clone", [self.crab_bin, "clone", consumer_url, str(clone_dir), "--jsonl"], self.run_root, + extra_env=clone_env, + ) + self.run_crab( + clone_dir, ["hydrate", "--all"], name=f"{case_name} hydrate", + extra_env=clone_env, ) - self.run_crab(clone_dir, ["hydrate", "--all"], name=f"{case_name} hydrate") self.check( f"{case_name}-fresh-hydrate-is-byte-identical", sha256_file(clone_dir / "model.bin") == expected_sha256, @@ -2100,11 +2144,18 @@ def run_committed_restage_before_first_push_case(self) -> None: ["staging", "stats", "--json"], name=f"{case_name} staging ownership before push", ) - before_push = json.loads(self.read_stdout(before_push_record))["data"]["lifecycle"] + before_push_data = json.loads(self.read_stdout(before_push_record))["data"] + before_push = before_push_data["lifecycle"] + expected_versions = {first_pointer.file_hash, second_pointer.file_hash} + retained_versions = {entry["file_hash"] for entry in before_push_data["files"]} + # Native preparation and Git-clean retain independent handoff leases. + # Version coverage, followed by exact retirement, proves their lifetime; + # a fixed pre-push lease count cannot identify the preserved versions. self.check( f"{case_name}-retains-current-and-committed-history-owners", - before_push["path_heads"] == 2 - and before_push["path_leases"] == 2 + retained_versions == expected_versions + and before_push["recipes"] == len(expected_versions) + and before_push["path_leases"] >= len(expected_versions) and before_push["open_batches_without_publication"] == 0 and before_push["reclaimable_superseded_leases"] == 0 and before_push["reclaimable_files"] == 0, @@ -2277,9 +2328,8 @@ def run(self) -> None: self.run_ref_update_delete_force_and_tag_case() self.run_atomic_rejection_case() self.run_shallow_case() - self.run_immutable_object_failure_case("pack") - self.run_immutable_object_failure_case("idx") - self.run_corrupt_index_failure_case() + for fault in ("missing", "pack", "index"): + self.run_capsule_failure_case(fault) self.run_concurrent_push_case() self.run_committed_restage_before_first_push_case() self.run_partial_overlap_case() diff --git a/crab/scripts/e2e/test_add_commit_push_proof.py b/crab/scripts/e2e/test_add_commit_push_proof.py new file mode 100644 index 000000000..e9ced8bb2 --- /dev/null +++ b/crab/scripts/e2e/test_add_commit_push_proof.py @@ -0,0 +1,65 @@ +"""Regression tests for the capsule smoke's direct-proof oracle.""" + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from run_add_commit_push_rustfs_smoke import capsule_staged_proof_complete + + +class CapsuleProofTest(unittest.TestCase): + def proof(self): + return { + "message": "revalidated staged remote placement proofs", + "planned": 7, "indexed": 7, "matched": 7, + "generation_verified": 0, "capsule_source_verified": 7, + "payload_verified": 7, "placement_mismatches": 0, + "origin_proof_mismatches": 0, + } + + def adoption(self): + return { + "message": "capsule staged remote proof adoption complete", + "planned_existing": 7, "stale_existing": 0, "global_existing": 0, + } + + def log(self, *events): + return "\n".join(json.dumps({"fields": e}, separators=(",", ":")) for e in events) + + def test_accepts_complete_capsule_evidence(self): + self.assertTrue(capsule_staged_proof_complete(self.log(self.proof(), self.adoption()), 7)) + + def test_rejects_incomplete_or_invalid_counters(self): + for key in self.proof(): + if key == "message": + continue + for value in [None, -1, True, "7", 8]: + with self.subTest(key=key, value=value): + event = {**self.proof(), key: value} + self.assertFalse(capsule_staged_proof_complete(self.log(event, self.adoption()), 7)) + for key in ["planned_existing", "stale_existing", "global_existing"]: + with self.subTest(key=key): + event = {**self.adoption(), key: 1} + self.assertFalse(capsule_staged_proof_complete(self.log(self.proof(), event), 7)) + + def test_rejects_counts_spread_across_events(self): + first = {**self.proof(), "generation_verified": 7, "matched": 0} + second = {**first, "matched": 7, "payload_verified": 0} + self.assertFalse(capsule_staged_proof_complete(self.log(first, second, self.adoption()), 7)) + + def test_rejects_missing_reordered_or_ambiguous_events(self): + for events in [[], [self.proof()], [self.adoption()], + [self.adoption(), self.proof()], + [self.proof(), self.proof(), self.adoption()]]: + with self.subTest(events=events): + self.assertFalse(capsule_staged_proof_complete(self.log(*events), 7)) + + def test_rejects_non_positive_expected_count(self): + for count in [0, -1, True]: + self.assertFalse(capsule_staged_proof_complete(self.log(self.proof(), self.adoption()), count)) + + +if __name__ == "__main__": + unittest.main() diff --git a/crab/scripts/e2e/test_add_commit_push_source.py b/crab/scripts/e2e/test_add_commit_push_source.py index 5b9ef5052..4f2c42e1a 100644 --- a/crab/scripts/e2e/test_add_commit_push_source.py +++ b/crab/scripts/e2e/test_add_commit_push_source.py @@ -214,5 +214,36 @@ def failed_run(smoke: SMOKE.AddCommitPushSmoke) -> None: self.assertNotIn(secret, report_text + stderr.getvalue()) +class CapsuleFaultRangeTests(unittest.TestCase): + @staticmethod + def fixture(*, offset: int = 8, length: int = 9, version: int = 4) -> bytes: + body = b"PACKbody\xfftOcindex" + footer = json.dumps({ + "version": version, + "git_packs": [{ + "pack": {"offset": 0, "length": 8}, + "index": {"offset": offset, "length": length}, + }], + }).encode() + return body + footer + len(footer).to_bytes(8, "big") + bytes(32) + b"CRBRUN01" + + def test_targets_the_absolute_pack_and_index_ranges(self) -> None: + body = self.fixture() + for kind, expected in (("pack", (0, 8)), ("index", (8, 9))): + with self.subTest(kind=kind): + self.assertEqual(SMOKE.capsule_member_range(body, kind), expected) + + def test_refuses_ranges_that_could_damage_unselected_controls(self) -> None: + for arguments in ({"offset": -1}, {"offset": 8, "length": 10}, {"offset": True}, {"length": 3}): + with self.subTest(arguments=arguments), self.assertRaises(SMOKE.SmokeError): + SMOKE.capsule_member_range(self.fixture(**arguments), "index") + + def test_refuses_other_container_formats_and_malformed_trailers(self) -> None: + body = self.fixture() + for malformed in (body[:-1], body[:20], self.fixture(version=3), body[:-48] + bytes(48)): + with self.subTest(size=len(malformed)), self.assertRaises(SMOKE.SmokeError): + SMOKE.capsule_member_range(malformed, "pack") + + if __name__ == "__main__": unittest.main() diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 17c2cabd0..a3845ee03 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -91,10 +91,13 @@ pub struct StoreChecker { capsule: Option, } -#[derive(Clone)] struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, + source_lifetime: Arc, + browse_lifetime: + Arc>, + git_workspace: tokio::sync::OnceCell, } impl StoreChecker { @@ -134,13 +137,19 @@ impl StoreChecker { ) .await?; let catalog = view.pointer_catalog().await?; - verify_capsule_history(&layout, view.root().root()).await?; + let source_lifetime = verify_capsule_history(&layout, view.root().root()).await?; Ok(Self { store, prefix, router, multipart_journal: None, - capsule: Some(CapsuleFsckState { view, catalog }), + capsule: Some(CapsuleFsckState { + view, + catalog, + source_lifetime: Arc::new(source_lifetime), + browse_lifetime: Arc::new(tokio::sync::OnceCell::new()), + git_workspace: tokio::sync::OnceCell::new(), + }), }) } @@ -151,6 +160,22 @@ impl StoreChecker { } async fn check_capsule_data_chain(&self, state: &CapsuleFsckState) -> Result> { + let layout = crab_storage::StoreLayout::with_global_prefix( + self.store.as_storage().clone(), + self.router.repo_prefix().to_owned(), + self.router.global_prefix().to_owned(), + ); + // Current catalog health cannot prove cold captures still have their + // original sources. Full fsck checks lifetime roots; mirror recipe probes stay narrow. + state + .browse_lifetime + .get_or_try_init(|| { + crab_metadata::capsule_protocol::verify_retained_browse_indexes( + &layout, + MAX_FSCK_FRONTIER_BYTES, + ) + }) + .await?; let mut issues = Vec::new(); for (hash, entry) in state.catalog.xorbs() { @@ -221,6 +246,26 @@ impl StoreChecker { } } + if issues.is_empty() { + // Keep the authenticated Git database across phases. Catalog bodies + // alone cannot prove reachable pointer coverage or whole-file/LFS bytes. + let workspace = state + .git_workspace + .get_or_try_init(|| { + check_capsule_git_connectivity(&self.store, &self.router, state) + }) + .await?; + crab_read::capsule_protocol::verify_installed_dependencies( + &layout, + workspace.path(), + state.view.refs(), + &state.catalog, + CAPSULE_GIT_SCAN_LIMITS, + &CancellationToken::new(), + ) + .await?; + } + self.check_capsule_root_stability(state).await?; Ok(issues) } @@ -246,6 +291,16 @@ impl StoreChecker { .to_owned(), )); } + require_current_source_lifetime(&layout, &state.source_lifetime).await?; + // Full fsck keeps browse authority through payload and final checks. + // Mirror recipe probes never initialize this expensive maintenance proof. + if let Some(capture) = state.browse_lifetime.get() + && !capture.is_current(&layout).await? + { + return Err(CrabError::Protocol( + "browse retention changed during fsck; retry against one stable capture".to_owned(), + )); + } Ok(()) } @@ -876,7 +931,7 @@ impl StoreChecker { async fn verify_capsule_history( layout: &crab_storage::StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, -) -> Result<()> { +) -> Result { let segments = match root.history() { Some(history) => { crab_metadata::capsule_protocol::load_history_chain( @@ -969,7 +1024,22 @@ async fn verify_capsule_history( .buffer_unordered(16) .try_collect::>() .await?; - Ok(()) + // Per-ref retirement and pruning can move without changing visible refs. + // Keep the capture for the later data/Git checks instead of rescanning history. + require_current_source_lifetime(layout, &retained).await?; + Ok(retained) +} + +async fn require_current_source_lifetime( + layout: &crab_storage::StoreLayout, + capture: &crab_metadata::capsule_protocol::RetainedCapsuleSources, +) -> Result<()> { + if capture.is_current(layout).await? { + return Ok(()); + } + Err(CrabError::Protocol( + "source lifetime changed during fsck; retry against one stable capture".to_owned(), + )) } fn insert_history_pointer( @@ -1164,10 +1234,10 @@ async fn install_capsule_git_packs( } async fn check_capsule_git_connectivity( - store: Store, - router: StoreLayout, - state: CapsuleFsckState, -) -> Result<()> { + store: &Store, + router: &StoreLayout, + state: &CapsuleFsckState, +) -> Result { let workspace = tempfile::tempdir()?; crab_git::initialize_bare_git_dir(workspace.path())?; let storage = store.as_storage().clone(); @@ -1183,7 +1253,7 @@ async fn check_capsule_git_connectivity( .map(|(name, oid)| (name.clone(), oid.clone())) .collect::>(); install_capsule_git_packs( - state.view, + state.view.clone(), layout, workspace.path().to_owned(), MAX_FSCK_FRONTIER_BYTES, @@ -1197,7 +1267,8 @@ async fn check_capsule_git_connectivity( .await .map_err(|error| CrabError::Internal(format!("Git connectivity scan failed: {error}")))??; crab_git::batch::verify_git_dir_blobs(workspace.path(), &scan.unchecked_blobs, &|| false) - .map_err(CrabError::Io) + .map_err(CrabError::Io)?; + Ok(workspace) } impl FsckChecker for StoreChecker { @@ -1207,13 +1278,15 @@ impl FsckChecker for StoreChecker { let store = self.store.clone(); let router = self.router.clone(); let prefix = self.prefix.clone(); - let capsule = self.capsule.clone(); Box::pin(async move { - if let Some(state) = capsule { + if let Some(state) = &self.capsule { // Pack installation validates every advertised pack. The reachable // walker then proves each ref's commit/tree/blob closure instead of // treating an authenticated ref OID as sufficient evidence. - check_capsule_git_connectivity(store, router, state).await?; + state + .git_workspace + .get_or_try_init(|| check_capsule_git_connectivity(&store, &router, state)) + .await?; return Ok(Vec::new()); } // Git-object connectivity requires a local git repo and gix-fsck. @@ -1707,6 +1780,9 @@ mod verification_tests; #[cfg(test)] mod capsule_history_tests; +#[cfg(test)] +mod pointer_data_tests; + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -1869,7 +1945,7 @@ mod tests { create_manifest(store, &router, &manifest).await.unwrap(); } - async fn capsule_checker_fixture() -> ( + pub(super) async fn capsule_checker_fixture() -> ( Store, String, crab_metadata::capsule_protocol::RootSnapshot, @@ -1897,7 +1973,8 @@ mod tests { let parser = XorbParser::parse(xorb_bytes.clone()).unwrap(); assert_eq!(parser.num_chunks(), 1); let chunk = parser.chunk_meta(0).unwrap(); - let file_hash = chunk.hash; + // File identity uses ordinary Blake3; Xet's chunk hash is keyed. + let file_hash = MerkleHash::from(*blake3::hash(&content).as_bytes()); let placement = crab_xet::xorb::format::ChunkPlacement { chunk_hash: chunk.hash, xorb_hash, @@ -2060,6 +2137,23 @@ mod tests { assert!(shard_issues.is_empty()); } + #[tokio::test] + async fn capsule_data_check_rejects_corrupt_browse_lifetime() { + let (store, prefix, root, _, _) = capsule_checker_fixture().await; + let checker = StoreChecker::for_capsule_repository(store.clone(), prefix, root) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + store + .put( + &checker.router.repo_path("v2/browse-retention"), + bytes::Bytes::from_static(b"corrupt"), + ) + .await + .unwrap(); + assert!(checker.check_data_chain().await.is_err()); + } + #[tokio::test] async fn capsule_checker_validates_repository_without_legacy_layout() { let (store, prefix, root, _, _) = capsule_checker_fixture().await; diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index c8b0cb691..bf3fa7de8 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -5,8 +5,91 @@ use crab_metadata::capsule_protocol::{ LayeredCheckpoint, PackLayer, PackSourceDescriptor, PointerCatalog, RootSnapshot, }; use crab_storage::{StorageObservation, StorageObserver, StorageOperation}; -use object_store::{ObjectStoreExt, memory::InMemory}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, memory::InMemory, + path::Path as ObjectPath, +}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CorruptLifetimeDuringRead { + inner: Arc, + source: ObjectPath, + head: ObjectPath, + fired: AtomicBool, +} + +impl std::fmt::Display for CorruptLifetimeDuringRead { + fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + output.write_str("source lifetime corruption fixture") + } +} + +#[async_trait::async_trait] +impl ObjectStore for CorruptLifetimeDuringRead { + async fn put_opts( + &self, + path: &ObjectPath, + payload: PutPayload, + options: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + + async fn put_multipart_opts( + &self, + path: &ObjectPath, + options: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + + async fn get_opts( + &self, + path: &ObjectPath, + options: GetOptions, + ) -> object_store::Result { + let full_body = !options.head && options.range.is_none(); + let response = self.inner.get_opts(path, options).await?; + if full_body && *path == self.source && !self.fired.swap(true, Ordering::SeqCst) { + self.inner + .put(&self.head, Bytes::from_static(b"corrupt").into()) + .await?; + } + Ok(response) + } + + fn delete_stream( + &self, + paths: futures_util::stream::BoxStream<'static, object_store::Result>, + ) -> futures_util::stream::BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + + fn list( + &self, + prefix: Option<&ObjectPath>, + ) -> futures_util::stream::BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&ObjectPath>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &ObjectPath, + to: &ObjectPath, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} async fn history_fixture() -> ( crab_storage::StoreLayout, @@ -104,6 +187,183 @@ struct SourceReads { count: AtomicUsize, } +#[tokio::test] +async fn fsck_rejects_lifetime_corruption_during_original_source_verification() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + let head = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await + .unwrap() + .remove(0) + .location; + let backend = Arc::new(CorruptLifetimeDuringRead { + inner: Arc::clone(layout.store().inner()), + source: layout.capsule_path(run.hash()), + head, + fired: AtomicBool::new(false), + }); + let observed = crab_storage::StoreLayout::new( + crab_storage::Store::new(backend.clone()), + layout.repo_prefix().to_owned(), + ); + let result = verify_capsule_history(&observed, root.record().root()).await; + assert!(backend.fired.load(Ordering::SeqCst)); + assert!( + result.is_err(), + "fsck accepted corrupt lifetime metadata after its original source read: {result:?}" + ); +} + +#[tokio::test] +async fn data_checks_reject_browse_corruption_during_xorb_verification() { + for readiness in [false, true] { + let (store, prefix, root, xorb_hash, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let backend = Arc::new(CorruptLifetimeDuringRead { + inner: Arc::clone(store.inner()), + source: router.xorb_path(&xorb_hash), + head: if readiness { + layout.capsule_browse_indexes_path() + } else { + layout.browse_retained_head_path() + }, + fired: AtomicBool::new(false), + }); + let store = crab_storage::Store::new(backend.clone()); + let checker = StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(store), + prefix, + root, + ) + .await + .unwrap(); + let result = checker.check_data_chain().await; + assert!(backend.fired.load(Ordering::SeqCst)); + assert!( + result.is_err(), + "fsck accepted corrupt browse metadata after its xorb read: readiness={readiness}, {result:?}" + ); + } +} + +#[tokio::test] +async fn final_snapshot_check_rejects_browse_movement_after_payload_verification() { + for readiness in [false, true] { + let (store, prefix, root, _, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let checker = StoreChecker::for_capsule_repository(store.clone(), prefix, root) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + let indexes = crab_metadata::capsule_protocol::BrowseIndexes::new( + "a".repeat(64), + "b".repeat(64), + None, + ) + .unwrap(); + if readiness { + store + .put( + &layout.capsule_browse_indexes_path(), + indexes.encode().unwrap(), + ) + .await + .unwrap(); + } else { + crab_metadata::capsule_protocol::retain_browse_indexes(&layout, &indexes, 0, None, &[]) + .await + .unwrap(); + } + assert!(matches!( + checker.check_orphan_file_index().await, + Err(CrabError::Protocol(reason)) if reason.contains("browse retention changed") + )); + } +} + +#[tokio::test] +async fn capsule_recipe_probe_does_not_require_browse_maintenance() { + let (store, prefix, root, _, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + store + .put( + &router.repo_path("v2/browse-retention"), + Bytes::from_static(b"corrupt"), + ) + .await + .unwrap(); + let checker = StoreChecker::for_capsule_repository(store, prefix, root) + .await + .unwrap(); + let (hash, file) = checker + .capsule + .as_ref() + .unwrap() + .catalog + .files() + .first_key_value() + .unwrap(); + let pointer = Pointer { + file_hash: MerkleHash::from_hex(hash).unwrap().into(), + size: file.size(), + shard_hint: None, + }; + let proof = checker + .verify_capsule_pointer_data(&[pointer], &CancellationToken::new()) + .await + .unwrap(); + assert!(proof.verified == 1 && proof.issues.is_empty(), "{proof:?}"); +} + +#[tokio::test] +async fn data_checks_reject_lifetime_movement_after_checker_construction() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + let checker = StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(layout.store().clone()), + layout.repo_prefix().to_owned(), + root, + ) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + crab_metadata::capsule_protocol::prune_retained_capsule_sources(&layout, 0) + .await + .unwrap(); + assert!(matches!( + checker.check_data_chain().await, + Err(CrabError::Protocol(reason)) if reason.contains("source lifetime changed") + )); +} + #[tokio::test] async fn retired_sources_are_checked_without_checkpoint_history() { let (layout, root, run) = history_fixture().await; diff --git a/crab/src/cmd/fsck_store/pointer_data_tests.rs b/crab/src/cmd/fsck_store/pointer_data_tests.rs new file mode 100644 index 000000000..051bc6562 --- /dev/null +++ b/crab/src/cmd/fsck_store/pointer_data_tests.rs @@ -0,0 +1,226 @@ +#![expect(clippy::unwrap_used, reason = "test assertions")] + +use super::*; +use bytes::Bytes; +use crab_metadata::capsule_protocol::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, + CapsuleTransaction, CapsuleVisibilityDelta, FileCatalogEntry, PointerCatalog, + ShardCatalogEntry, +}; +use crab_metadata::git_visibility::GitVisibilityEdit; +use sha2::{Digest, Sha256}; + +async fn publish_pointer( + layout: &crab_storage::StoreLayout, + root: crab_metadata::capsule_protocol::RootSnapshot, + body: &[u8], + delta: &PointerCatalog, +) -> crab_metadata::capsule_protocol::RootSnapshot { + let kind = gix_object::Kind::Blob; + let oid = crab_remote::objects::object_id(kind, body) + .unwrap() + .to_string(); + let mut bytes = Vec::new(); + crab_git::pack_writer::write_pack( + &mut bytes, + std::iter::once(Ok((kind, body.len() as u64, body))), + 1024 * 1024, + || false, + ) + .unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let path = scratch.path().join("source.pack"); + std::fs::write(&path, &bytes).unwrap(); + let indexed = crab_git::pack::install_pack_file_from_path( + &scratch.path().join("indexed"), + &path, + blake3::hash(&bytes).to_hex().as_ref(), + 1024 * 1024, + true, + ) + .unwrap(); + let checksum = gix_hash::ObjectId::from_hex(indexed.git_sha1.as_bytes()).unwrap(); + let kinds = crab_git::pack_locator::encode_pack_kind_metadata(checksum, &[kind]).unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from(bytes), + Bytes::from(std::fs::read(indexed.idx_path).unwrap()), + Bytes::from(std::fs::read(indexed.rev_path).unwrap()), + Bytes::from(kinds), + indexed.git_sha1, + 1, + ) + .unwrap(); + let name = "refs/tags/payload"; + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new(name, None, Some(oid.clone()), None)], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + name.to_owned(), + GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid]), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![pack], + vec![ + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + ), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + delta.encode_delta().unwrap(), + ), + ], + ) + .unwrap(); + crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + .await + .unwrap() +} + +async fn pointer_checker(case: &str) -> StoreChecker { + let (store, prefix, root, xorb_hash, shard_hash) = + super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let mut pointer = Pointer { + file_hash: blake3::hash(&[0x61; 1024]).into(), + size: 1024, + shard_hint: None, + }; + let mut delta = PointerCatalog::new(); + match case { + "missing" => pointer.file_hash = [42; 32], + "size" => pointer.size += 1, + "hash" => { + let (body, _) = store + .get_with_etag_bounded(&router.shard_path(&shard_hash), MAX_FSCK_SHARD_BYTES) + .await + .unwrap(); + let reader = ShardReader::from_bytes(body, shard_hash); + let mut recipe = reader + .get_file_info(&MerkleHash::from(pointer.file_hash)) + .unwrap() + .unwrap(); + pointer.file_hash = [42; 32]; + recipe.metadata.file_hash = MerkleHash::from(pointer.file_hash); + let mut shard = crab_xet::shard::ShardWriter::new(); + shard + .add_xorb(Arc::new(reader.get_xorb_info(&xorb_hash).unwrap().unwrap())) + .unwrap(); + shard.add_file(recipe).unwrap(); + let (body, hash) = shard.finalize().unwrap(); + delta + .insert_shard( + hash.hex(), + ShardCatalogEntry::new(body.len() as u64, vec![xorb_hash.hex()]), + ) + .unwrap(); + delta + .insert_file( + MerkleHash::from(pointer.file_hash).hex(), + FileCatalogEntry::new(pointer.size, hash.hex()), + ) + .unwrap(); + store + .put(&router.shard_path(&hash), Bytes::from(body)) + .await + .unwrap(); + } + _ => {} + } + let body = if case.starts_with("lfs-") { + let lfs = crab_git::lfs_pointer::LfsPointer { + oid: Sha256::digest(b"hello").into(), + size: 5, + extensions: Vec::new(), + }; + if case != "lfs-missing" { + let path = crab_lfs::LfsObjectStore::new(layout.store().clone(), layout.repo_prefix()) + .object_path_for(&lfs.oid); + let body = if case == "lfs-corrupt" { + b"wrong" + } else { + b"hello" + }; + layout + .store() + .put(&path, Bytes::copy_from_slice(body)) + .await + .unwrap(); + } + lfs.serialize() + } else { + pointer.serialize() + }; + let root = publish_pointer(&layout, root, &body, &delta).await; + StoreChecker::for_capsule_repository(store, prefix, root) + .await + .unwrap() +} + +#[tokio::test] +async fn full_fsck_proves_reachable_crab_and_lfs_payloads() { + let mut outcomes = Vec::new(); + for case in [ + "valid", + "missing", + "size", + "hash", + "lfs-valid", + "lfs-missing", + "lfs-corrupt", + ] { + let checker = Box::pin(pointer_checker(case)).await; + let (_, outcome) = crate::cmd::fsck::run_fsck( + &crate::cmd::fsck::FsckArgs::default(), + &checker, + &crate::cmd::fsck::NullRepairer, + &CancellationToken::new(), + Duration::from_hours(1), + None, + ) + .await + .unwrap(); + outcomes.push((case, outcome.errors == 0)); + } + assert_eq!( + outcomes, + vec![ + ("valid", true), + ("missing", false), + ("size", false), + ("hash", false), + ("lfs-valid", true), + ("lfs-missing", false), + ("lfs-corrupt", false) + ] + ); +} + +#[tokio::test] +async fn fsck_retains_one_git_database_through_data_checks_then_releases_it() { + let checker = Box::pin(pointer_checker("valid")).await; + checker.check_git_objects().await.unwrap(); + let state = checker.capsule.as_ref().unwrap(); + let original = state.git_workspace.get().unwrap().path().to_owned(); + let clean = checker.check_data_chain().await.unwrap().is_empty(); + let reused = state.git_workspace.get().unwrap().path() == original; + drop(checker); + assert!(clean && reused && !original.exists()); +} + +#[tokio::test] +async fn data_checks_prove_pointer_coverage_without_a_prior_git_phase() { + let checker = Box::pin(pointer_checker("missing")).await; + assert!(matches!(checker.check_data_chain().await, + Err(CrabError::CorruptObject { reason, .. }) if reason.contains("absent from the catalog") + )); +} diff --git a/crab/src/cmd/gc/capsule_cleanup_tests.rs b/crab/src/cmd/gc/capsule_cleanup_tests.rs index a501550aa..d1e0311e1 100644 --- a/crab/src/cmd/gc/capsule_cleanup_tests.rs +++ b/crab/src/cmd/gc/capsule_cleanup_tests.rs @@ -19,6 +19,199 @@ use crate::core::error::CrabError; use crate::storage::StoreLayout; use crate::storage::store::Store; +#[tokio::test] +async fn captured_catalog_pages_survive_checkpoint_replacement_and_gc() { + use crab_metadata::capsule_protocol::{ + CapsuleGitPack, FileCatalogEntry, LayeredCheckpoint, PackLayer, PointerCatalog, + ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + }; + use crab_metadata::file_index_lookup::FileIndexLookupSession; + use crab_xet::hash::MerkleHash; + + let store = Store::new(Arc::new(InMemory::new())); + let router = StoreLayout::new(store.clone(), "org/catalog-gc".to_owned()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let base = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let mut catalog = PointerCatalog::new(); + let shard = "a".repeat(64); + let xorb = "b".repeat(64); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 80, + "c".repeat(64), + vec![XorbChunkEntry::new("d".repeat(64), 16)], + ), + ) + .unwrap(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + for index in 0..20_000 { + catalog + .insert_file( + format!("{index:064x}"), + FileCatalogEntry::new(16, shard.clone()), + ) + .unwrap(); + } + let transaction = CapsuleTransaction::new( + base.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap(); + crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) + .await + .unwrap(); + let view = crab_read::capsule_protocol::open_view( + &layout, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 8 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let layer = PackLayer::build( + &CapsuleGitPack::new( + Bytes::from_static(b"PACK"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "3".repeat(40), + 1, + ) + .unwrap(), + ) + .unwrap(); + let source = layer.source_descriptor().unwrap(); + store + .put( + &layout.capsule_pack_layer_path(layer.hash()), + layer.bytes().clone(), + ) + .await + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + view.root().root().generation(), + view.root().digest(), + vec![source.clone()], + catalog, + None, + ) + .unwrap(); + let first = crab_write::capsule_protocol::publish_ref_layered_checkpoint( + &layout, + view.root_snapshot().clone(), + &checkpoint, + view.refs().clone(), + view.peeled_refs().clone(), + view.visible_ref_transactions().clone(), + view.capsule_run_pointers().to_vec(), + ) + .await + .unwrap(); + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed_reads = reads.clone(); + let observed = layout + .store() + .clone() + .with_read_request_observer(Arc::new(move |_| { + observed_reads.fetch_add(1, Ordering::SeqCst); + })); + let captured = FileIndexLookupSession::open_for_storage(&observed, layout.repo_prefix()) + .await + .unwrap(); + // No file query has run: the old reader must fetch its catalog leaf after + // compaction and a real sweep, rather than succeed from a warmed page cache. + let next = LayeredCheckpoint::build( + first.record().root().generation(), + first.record().digest(), + vec![source], + PointerCatalog::new(), + None, + ) + .unwrap(); + let root = crab_write::capsule_protocol::publish_ref_layered_checkpoint( + &layout, + first.clone(), + &next, + first.record().root().refs().clone(), + first.record().root().peeled_refs().clone(), + first.record().root().compacted_ref_transactions().clone(), + Vec::new(), + ) + .await + .unwrap(); + let file = MerkleHash::from_hex(&format!("{:064x}", 19_999)).unwrap(); + let latest = FileIndexLookupSession::open_for_storage(layout.store(), layout.repo_prefix()) + .await + .unwrap(); + let latest_result = latest.lookup(&file).await; + latest.close().await.unwrap(); + assert_eq!(latest_result.unwrap(), None); + + let orphan = layout.capsule_checkpoint_path(&"e".repeat(64)); + store + .put(&orphan, Bytes::from_static(b"orphan")) + .await + .unwrap(); + let current = crab_read::capsule_protocol::open_view_from_root_with_control( + &layout, + root.clone(), + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 8 * 1024 * 1024, + max_frontier_bytes: 8 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let snapshot = current.git_snapshot().unwrap(); + let outcome = sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + root.record().root(), + current.capsule_run_pointers(), + &snapshot.manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now() + Duration::from_secs(2 * 3600), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + assert_eq!(outcome.packs_deleted, 1); + assert!(matches!( + store.head(&orphan).await, + Err(CrabError::NotFound { .. }) + )); + reads.store(0, Ordering::SeqCst); + let selected = captured.lookup_batch(&[file, file]).await; + captured.close().await.unwrap(); + assert!( + reads.load(Ordering::SeqCst) > 0, + "captured lookup must fetch cold pages after GC" + ); + let expected = MerkleHash::from_hex(&shard).unwrap(); + assert_eq!(selected.unwrap(), vec![Some(expected), Some(expected)]); +} + #[tokio::test] async fn failed_capsule_view_releases_root_and_sweep_fences() { let store = Store::new(Arc::new(InMemory::new())); diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index 5d6c9e128..fc58d03c3 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -396,6 +396,7 @@ async fn run_inner( .collect::>() }; let mut xet_stats = super::xet_publication::XetPublicationStats::default(); + let mut pointer_delta = crab_metadata::capsule_protocol::PointerCatalog::new(); let publication: Result>> = async { // LFS bytes share the ref visibility boundary with Git and Xet data. @@ -409,19 +410,22 @@ async fn run_inner( cancel, ) .await?; - let (pointer_delta, prepared_xet_stats) = super::xet_publication::prepare_delta( - &layout, - &base, - &prepared.pointers, - staging, - caching_store, - metrics, - config.upload_concurrency, - config.protected_push.is_none(), - cancel, - ) - .await?; + let (prepared_pointer_delta, prepared_xet_stats) = + super::xet_publication::prepare_delta( + &layout, + &base, + &prepared.pointers, + staging, + caching_store, + metrics, + &config.metadb, + config.upload_concurrency, + config.protected_push.is_none(), + cancel, + ) + .await?; xet_stats = prepared_xet_stats; + pointer_delta = prepared_pointer_delta; if config.protected_push.is_none() && let Some(replication) = config.active_active_replication.as_ref() { @@ -477,6 +481,7 @@ async fn run_inner( } let result = if changes_namespace { let commit_layout = layout.clone(); + let commit_pointer_delta = &pointer_delta; let commit_push_ref_head_bases = push_ref_head_bases.clone(); crab_write::with_ref_namespaces_wait( layout.store(), @@ -501,7 +506,7 @@ async fn run_inner( base, &transaction, &capsule, - &pointer_delta, + commit_pointer_delta, &lfs_objects, &commit_push_ref_head_bases, ) @@ -545,6 +550,58 @@ async fn run_inner( } }; if let Some(active_active_commit) = committed { + if layout.store().staging_write_prefix().is_none() { + let mut index_config = + crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + if let Some(path) = config.metadb.chunk_index.db.path.as_ref() { + index_config.chunk_index_path.clone_from(path); + } + // The ref is already committed. Index failures only remove acceleration; + // reporting an uncommitted push here would invite a duplicate publication. + match crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &index_config, + layout.repo_prefix(), + &pointer_delta, + &xet_stats.origins, + ) + .await + { + Ok(()) => { + let cache_path = crate::cache::add_remote_candidate_cache_path( + &crate::cache::default_cache_root(), + &store.bucket_identity(), + layout.global_prefix(), + ); + if cache_path.is_file() { + let hashes = pointer_delta + .xorbs() + .values() + .flat_map(|entry| entry.chunks()) + .filter_map(|chunk| MerkleHash::from_hex(chunk.hash()).ok()) + .collect::>(); + let invalidation = tokio::task::spawn_blocking(move || { + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( + &cache_path, + ) + .and_then(|cache| cache.forget_negatives(&hashes)) + }) + .await; + if !matches!(invalidation, Ok(Ok(()))) { + tracing::warn!( + "capsule committed; candidate cache invalidation failed" + ); + } + } + } + Err(error) => { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair") + } + } + } if let Some(snapshot) = staging_snapshot && let Err(error) = snapshot.commit() { diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index d9e7b97fe..5e46f30aa 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -4494,17 +4494,23 @@ impl crab_staging::push_plan::ExistingChunkLookup for AddRemoteChunkClassifier { "remote add classifier failed: {error}" )), })?; + // Keep index absence in the session's OnceCell: persisting one miss + // per chunk makes cache writes scale with the entire new file. + let cache = self + .candidate_cache + .as_ref() + .filter(|_| matches!(self.remote_chunk_index.get(), Some(Some(_)))); let mut updates = Vec::with_capacity(misses.len()); for chunk_hash in misses { let candidate = fetched.get(&chunk_hash).copied(); - if self.candidate_cache.is_some() { + if cache.is_some() { updates.push((chunk_hash, candidate)); } if let Some(candidate) = candidate { candidates.insert(chunk_hash, candidate); } } - if let Some(cache) = &self.candidate_cache { + if let Some(cache) = cache { if let Err(error) = cache.memory_insert_batch(&updates) { warn!(error = %error, "add remote candidate memory cache update failed"); } @@ -9964,7 +9970,7 @@ impl PushPipeline { &self, refs: &HashMap, ) -> Result> { - self.verify_xorb_refs_with_committed_receipts(refs, &HashMap::new()) + self.verify_xorb_refs_with_committed_receipts(refs, &HashMap::new(), false) .await } @@ -9972,6 +9978,7 @@ impl PushPipeline { &self, refs: &HashMap, committed_receipts: &HashMap, + require_committed_receipt: bool, ) -> Result> { use futures_util::StreamExt; @@ -10001,7 +10008,7 @@ impl PushPipeline { let cache = self.remote_full_xorb_ref_cache.lock().await; refs.values() .map(|xorb_ref| xorb_ref.xorb_hash) - .filter(|xorb_hash| cache.contains(xorb_hash)) + .filter(|xorb_hash| !require_committed_receipt && cache.contains(xorb_hash)) .collect::>() }; { @@ -10012,7 +10019,8 @@ impl PushPipeline { .entry(xorb_ref.xorb_hash) .or_default() .push((*chunk_hash, *xorb_ref)); - } else if chunk_ref_cache.contains(&RemoteChunkRefKey::new(*chunk_hash, *xorb_ref)) + } else if !require_committed_receipt + && chunk_ref_cache.contains(&RemoteChunkRefKey::new(*chunk_hash, *xorb_ref)) { verified.insert(*chunk_hash, *xorb_ref); cached_xorb_hashes.insert(xorb_ref.xorb_hash); @@ -10051,7 +10059,7 @@ impl PushPipeline { { let index_cache = self.remote_xorb_index_cache.lock().await; for xorb_hash in refs_by_xorb.keys().copied() { - if let Some(index) = index_cache.get(&xorb_hash) { + if !require_committed_receipt && let Some(index) = index_cache.get(&xorb_hash) { verified_xorbs.insert(xorb_hash, index.clone()); } else { planned_hashes.push(xorb_hash); @@ -10179,6 +10187,14 @@ impl PushPipeline { return None; } Some(async move { + // Capsule assertions bind the exact origin selected at add. + // Cached payload validity cannot certify a replaced object. + if require_committed_receipt && !committed_receipt.as_ref().zip(object_meta.as_ref()) + .is_some_and(|(receipt, meta)| committed_origin_receipt_matches_remote( + receipt, &xorb_hash, &path, &index, meta, + )) { + return Ok((Vec::new(), None, None, Vec::new())); + } if is_full_cached { let xorb_info = build_xorb_info_from_remote_index(&index)?; return Ok((valid_refs, Some((xorb_hash, xorb_info)), None, Vec::new())); @@ -10355,19 +10371,19 @@ impl PushPipeline { async fn revalidate_add_plan_existing_candidates( &self, planned: &HashMap, - ) -> Result> { + ) -> Result { if planned.is_empty() { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); } let chunk_store = { let guard = self.metadb.lock().await; let Some(guard) = guard.as_ref() else { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); }; match guard.chunk_index().await { Ok(store) => store, Err(error) if error.is_metadb_read_only_uninitialized() => { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); } Err(error) => return Err(error), } @@ -10376,8 +10392,47 @@ impl PushPipeline { .iter() .map(|(chunk_hash, candidate)| (*chunk_hash, candidate.placement_id)) .collect::>(); - let remote_candidates = tokio::time::timeout( - GLOBAL_CHUNK_LOOKUP_BUDGET, + let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; + let (capsule, capsule_indexed, capsule_matched) = + tokio::time::timeout_at(deadline, async { + let mut candidates = chunk_store + .capsule_candidates_by_id(&exact_candidates) + .await?; + let indexed = candidates.placements.len(); + candidates.placements.retain(|hash, placement| { + planned + .get(&MerkleHash::from(*hash)) + .is_some_and(|candidate| { + candidate.placement_id == placement.placement_id() + && candidate.origin_proof_id == placement.origin_proof_id + && candidate.xorb_ref + == XorbRef { + xorb_hash: MerkleHash::from(placement.xorb_hash), + chunk_index: placement.chunk_index, + uncompressed_size: placement.uncompressed_size, + } + }) + }); + let matched = candidates.placements.len(); + let verified = self.verify_capsule_chunk_candidates(candidates).await?; + Ok::<_, CrabError>((verified, indexed, matched)) + }) + .await + .map_err(|_| { + CrabError::NetworkTransient(object_store::Error::Generic { + store: "chunk_index_db", + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "staged capsule-proof revalidation budget exhausted", + )), + }) + })??; + let exact_candidates = exact_candidates + .into_iter() + .filter(|(hash, _)| !capsule.candidates.contains_key(hash)) + .collect::>(); + let remote_candidates = tokio::time::timeout_at( + deadline, chunk_store.get_committed_candidates_by_id_batch(&exact_candidates), ) .await @@ -10426,26 +10481,30 @@ impl PushPipeline { let committed_receipts = self.validate_committed_chunk_receipts(&refs).await; let validated = self.verified_committed_chunk_receipts.lock().await.clone(); let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &committed_receipts) + .verify_xorb_refs_with_committed_receipts(&refs, &committed_receipts, false) .await?; debug!( planned = planned.len(), - indexed = indexed_candidates, - matched = refs.len(), + indexed = indexed_candidates + capsule_indexed, + matched = refs.len() + capsule_matched, generation_verified = validated.len(), - payload_verified = verified.len(), + capsule_source_verified = capsule.candidates.len(), + payload_verified = verified.len() + capsule.candidates.len(), placement_mismatches, origin_proof_mismatches, "revalidated staged remote placement proofs" ); - Ok(verified - .into_iter() - .filter(|(chunk_hash, _)| { - placement_ids - .get(chunk_hash) - .is_some_and(|placement_id| validated.get(chunk_hash) == Some(placement_id)) - }) - .collect()) + let mut output = capsule; + output + .candidates + .extend(verified.into_iter().filter_map(|(hash, _)| { + let placement = placement_ids.get(&hash)?; + (validated.get(&hash) == Some(placement)) + .then(|| planned.get(&hash).copied()) + .flatten() + .map(|candidate| (hash, candidate)) + })); + Ok(output) } async fn prove_all_origin_xorbs_for_publish(&self) -> Result<()> { @@ -10954,7 +11013,7 @@ impl PushPipeline { // executor stack. This path owns several repository-sized maps; // heap-pinning the future avoids inflating the default test stack. match Box::pin(self.revalidate_add_plan_existing_candidates(&candidate_refs)).await { - Ok(refs) => refs, + Ok(verified) => verified.into_refs(), Err(CrabError::Cancelled) => return Err(CrabError::Cancelled), Err(error) => return Err(error), } @@ -15009,6 +15068,173 @@ impl PushPipeline { Ok(Some((hits, skipped_remote))) } + async fn lookup_capsule_chunks_for_add( + &self, + hashes: &[MerkleHash], + chunk_store: &crate::metadata::ChunkIndexStore, + ) -> Result> { + let candidates = chunk_store.capsule_candidates(hashes).await?; + self.verify_capsule_chunk_candidates(candidates) + .await + .map(|verified| verified.candidates) + } + + async fn verify_capsule_chunk_candidates( + &self, + candidates: crab_metadata::capsule_protocol::CapsuleChunkCandidates, + ) -> Result { + let Some(store) = &self.store else { + return Ok(VerifiedRemoteCandidates::default()); + }; + #[derive(Default)] + struct SourceCandidates<'a> { + shards: HashSet<&'a str>, + placements: Vec<&'a crab_metadata::receipts::CommittedChunkPlacement>, + } + let mut by_repo: BTreeMap<&str, SourceCandidates<'_>> = BTreeMap::new(); + for placement in candidates.placements.values() { + if let Some(anchor) = candidates.anchors.get(&placement.source_anchor_id) { + let source = by_repo.entry(&anchor.source_repo_prefix).or_default(); + source.shards.insert(&anchor.source_shard_hash); + source.placements.push(placement); + } + } + let mut refs = HashMap::new(); + let mut origins = HashMap::new(); + let mut proofs = HashMap::new(); + let mut bindings: HashMap< + MerkleHash, + ( + crab_metadata::capsule_protocol::XorbCatalogEntry, + crab_metadata::receipts::OriginReceipt, + ), + > = HashMap::new(); + let mut retained_bytes = 0_u64; + for (repo, source) in by_repo { + check_cancelled(&self.cancel)?; + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + repo.to_owned(), + self.router.global_prefix().to_owned(), + ); + let root = match crab_metadata::capsule_protocol::load_root(&layout).await { + Ok(root) => root, + Err(crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::NotFound { .. }, + }) => continue, + Err(error) => return Err(error.into()), + }; + let lookup = crab_metadata::capsule_protocol::load_pointer_catalog_lookup_from_root( + &layout, &root, + ) + .await?; + let hashes = source + .shards + .into_iter() + .map(str::to_owned) + .collect::>(); + // Validate and release one source closure before opening the next; + // otherwise each source could retain its entire format byte budget. + let catalog = lookup.shards(layout.store(), &hashes).await?; + for placement in source.placements { + let Some(anchor) = candidates.anchors.get(&placement.source_anchor_id) else { + continue; + }; + let xorb_hash = MerkleHash::from(placement.xorb_hash); + let path = self.router.xorb_path(&xorb_hash); + if anchor + .validate_placement(placement, &catalog, path.as_ref()) + .is_err() + { + continue; + } + let entry = catalog.xorbs().get(&xorb_hash.hex()).ok_or_else(|| { + CrabError::Internal( + "validated capsule placement lost its descriptor".to_owned(), + ) + })?; + if let Some((prior, origin)) = bindings.get(&xorb_hash) { + // Observation time may differ; physical assertions for the + // same immutable xorb cannot disagree or certify one another. + if prior != entry + || origin.etag != anchor.origin.etag + || origin.object_version != anchor.origin.object_version + || origin.payload_digest != anchor.origin.payload_digest + || origin.size != anchor.origin.size + { + return Err(CrabError::CorruptObject { + path: path.to_string(), + reason: "conflicting canonical xorb assertions".to_owned(), + }); + } + } else { + let metadata_bytes = std::mem::size_of_val(entry.chunks()) as u64 + + entry + .chunks() + .iter() + .map(|chunk| chunk.hash().len() as u64) + .sum::() + + entry.body_digest().len() as u64 + + serde_json::to_vec(&anchor.origin) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))? + .len() as u64; + retained_bytes = + retained_bytes.checked_add(metadata_bytes).ok_or_else(|| { + CrabError::Internal("selected xorb metadata size overflows".to_owned()) + })?; + if retained_bytes > crab_metadata::capsule_protocol::MAX_CATALOG_BYTES { + return Err(CrabError::CorruptObject { + path: "capsule chunk candidates".to_owned(), + reason: "selected xorb metadata exceeds the catalog byte limit" + .to_owned(), + }); + } + bindings.insert(xorb_hash, (entry.clone(), anchor.origin.clone())); + } + let hash = MerkleHash::from(placement.chunk_hash); + refs.insert( + hash, + XorbRef { + xorb_hash, + chunk_index: placement.chunk_index, + uncompressed_size: placement.uncompressed_size, + }, + ); + origins.insert(xorb_hash, anchor.origin.clone()); + proofs.insert(hash, (placement.placement_id(), placement.origin_proof_id)); + } + } + // Source catalogs establish commitment. Receipts belong to the trusted + // scope's canonical global index; fresh provider identity binds their + // verified bytes, while cache or staged IDs alone cannot certify reuse. + let verified = self + .verify_xorb_refs_with_committed_receipts(&refs, &origins, true) + .await?; + let accepted_xorbs = verified + .values() + .map(|reference| reference.xorb_hash) + .collect::>(); + bindings.retain(|hash, _| accepted_xorbs.contains(hash)); + let candidates = verified + .into_iter() + .filter_map(|(hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proofs.get(&hash).copied()?; + Some(( + hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + }) + .collect(); + Ok(VerifiedRemoteCandidates { + candidates, + xorbs: bindings, + }) + } + async fn lookup_proven_remote_chunks_for_add( &self, chunk_hashes: &[MerkleHash], @@ -15018,12 +15244,8 @@ impl PushPipeline { return Ok(HashMap::new()); } check_cancelled(&self.cancel)?; - let remote_candidates = tokio::time::timeout( - GLOBAL_CHUNK_LOOKUP_BUDGET, - chunk_store.get_committed_candidates_batch(chunk_hashes), - ) - .await - .map_err(|_| { + let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; + let timeout_error = || { CrabError::NetworkTransient(object_store::Error::Generic { store: "chunk_index_db", source: Box::new(std::io::Error::new( @@ -15031,7 +15253,27 @@ impl PushPipeline { "add remote classification proof budget exhausted", )), }) - })??; + }; + let mut capsule = tokio::time::timeout_at( + deadline, + self.lookup_capsule_chunks_for_add(chunk_hashes, &chunk_store), + ) + .await + .map_err(|_| timeout_error())??; + let remaining = chunk_hashes + .iter() + .copied() + .filter(|hash| !capsule.contains_key(hash)) + .collect::>(); + if remaining.is_empty() { + return Ok(capsule); + } + let remote_candidates = tokio::time::timeout_at( + deadline, + chunk_store.get_committed_candidates_batch(&remaining), + ) + .await + .map_err(|_| timeout_error())??; let mut refs = HashMap::with_capacity(remote_candidates.placements.len()); let mut proof_ids = HashMap::with_capacity(remote_candidates.placements.len()); @@ -15058,22 +15300,20 @@ impl PushPipeline { .validate_committed_chunk_candidates(&refs, candidates) .await; let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins) + .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins, false) .await?; - Ok(verified - .into_iter() - .filter_map(|(chunk_hash, xorb_ref)| { - let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; - (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( - chunk_hash, - crab_staging::push_plan::ExistingChunkCandidate { - xorb_ref, - placement_id, - origin_proof_id, - }, - )) - }) - .collect()) + capsule.extend(verified.into_iter().filter_map(|(chunk_hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; + (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( + chunk_hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + })); + Ok(capsule) } async fn lookup_cache_service_chunk_refs( @@ -15168,7 +15408,7 @@ impl PushPipeline { let committed_receipts = Box::pin(self.validate_committed_chunk_receipts(&hits)).await; let refs = self - .verify_xorb_refs_with_committed_receipts(&hits, &committed_receipts) + .verify_xorb_refs_with_committed_receipts(&hits, &committed_receipts, false) .await?; let stale_hashes = hits .keys() @@ -16940,6 +17180,77 @@ impl PushPipeline { }) } } +#[derive(Debug, Default)] +pub(crate) struct VerifiedRemoteCandidates { + pub candidates: HashMap, + pub xorbs: HashMap< + MerkleHash, + ( + crab_metadata::capsule_protocol::XorbCatalogEntry, + crab_metadata::receipts::OriginReceipt, + ), + >, +} + +impl VerifiedRemoteCandidates { + fn into_refs(self) -> HashMap { + self.candidates + .into_iter() + .map(|(hash, candidate)| (hash, candidate.xorb_ref)) + .collect() + } +} + +// Capsule preparation owns this read-only session through revalidation and close. +// Reusing the planner's verifier keeps released manifest and capsule proofs on +// the same origin checks without opening a writer during payload preparation. +pub(crate) async fn revalidate_staged_remote_chunks( + metadb: &crate::core::config::MetaDbTomlConfig, + layout: &crab_storage::StoreLayout, + planned: &HashMap, + cancel: &CancellationToken, +) -> Result { + if planned.is_empty() { + return Ok(VerifiedRemoteCandidates::default()); + } + check_cancelled(cancel)?; + let store = Store::from_storage(layout.store().clone()); + let router = StoreLayout::with_global_prefix( + store.clone(), + layout.repo_prefix().to_owned(), + layout.global_prefix().to_owned(), + ); + let config = PushConfig { + metadb: metadb.clone(), + ..Default::default() + }; + let pipeline = PushPipeline::new( + config, + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + cancel.clone(), + None, + ); + pipeline.install_metadb(build_push_metadb_guard_with_object_store( + &store, None, &router, None, metadb, true, + )); + let result = Box::pin(pipeline.revalidate_add_plan_existing_candidates(planned)).await; + let guard = { pipeline.metadb.lock().await.take() }; + let closed = match guard { + Some(guard) => guard.close().await, + None => Ok(()), + }; + let verified = result?; + closed?; + check_cancelled(cancel)?; + Ok(verified) +} + /// Run the push pipeline for a batch of push specs. /// /// This is the main entry point called by the remote helper when Git @@ -20588,6 +20899,7 @@ mod tests { struct RangeCountingStore { inner: object_store::memory::InMemory, range_gets: Arc, + full_xorb_gets: AtomicUsize, head_e_tag: Option>>, } @@ -20596,6 +20908,7 @@ mod tests { Self { inner: object_store::memory::InMemory::new(), range_gets, + full_xorb_gets: AtomicUsize::new(0), head_e_tag: None, } } @@ -20607,6 +20920,7 @@ mod tests { Self { inner: object_store::memory::InMemory::new(), range_gets, + full_xorb_gets: AtomicUsize::new(0), head_e_tag: Some(head_e_tag), } } @@ -20643,6 +20957,9 @@ mod tests { options: GetOptions, ) -> object_store::Result { let is_head = options.head; + if !is_head && options.range.is_none() && location.as_ref().contains("/xorbs/") { + self.full_xorb_gets.fetch_add(1, Ordering::Relaxed); + } if options.range.is_some() { self.range_gets.fetch_add(1, Ordering::Relaxed); } @@ -22045,16 +22362,24 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn add_classifier_does_not_repeat_an_uninitialized_index_probe() { - let inner = Arc::new(object_store::memory::InMemory::new()); - let reads = Arc::new(Mutex::new(Vec::new())); - let recording_store: Arc = Arc::new(RecordingReadStore { - inner, - reads: Arc::clone(&reads), - }); - let store = Store::new(recording_store); - let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); + mod capsule_chunk_tests; + + fn add_classifier_with_private_cache( + store: &Store, + router: &StoreLayout, + cache_root: &std::path::Path, + ) -> ( + AddRemoteChunkClassifier, + Arc, + ) { + let cache_root = cache_root.join("cache"); + crab_cache::ensure_private_cache_directory(&cache_root).expect("private cache root"); + let candidate_cache = Arc::new( + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( + &cache_root.join("candidates.sqlite"), + ) + .expect("open candidate cache"), + ); let pipeline = PushPipeline::new( PushConfig::default(), Vec::new(), @@ -22067,20 +22392,32 @@ mod tests { CancellationToken::new(), None, ); - pipeline.install_metadb(build_push_metadb_guard( - &store, - &router, - None, - &crate::core::config::MetaDbTomlConfig::default(), - true, - )); + let mut config = crate::core::config::MetaDbTomlConfig::default(); + config.chunk_index.local_path = Some(cache_root.join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard(store, router, None, &config, true)); let classifier = AddRemoteChunkClassifier { pipeline, remote_chunk_index: tokio::sync::OnceCell::new(), - candidate_cache: None, + candidate_cache: Some(Arc::clone(&candidate_cache)), candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), }; + (classifier, candidate_cache) + } + + #[tokio::test(flavor = "multi_thread")] + async fn add_classifier_keeps_uninitialized_index_absence_session_scoped() { + let cache_root = tempfile::tempdir().expect("private candidate cache"); + let inner = Arc::new(object_store::memory::InMemory::new()); + let reads = Arc::new(Mutex::new(Vec::new())); + let recording_store: Arc = Arc::new(RecordingReadStore { + inner, + reads: Arc::clone(&reads), + }); + let store = Store::new(recording_store); + let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); + let (classifier, candidate_cache) = + add_classifier_with_private_cache(&store, &router, cache_root.path()); let first = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( &classifier, &[(MerkleHash::from([1; 32]), 4)], @@ -22118,6 +22455,45 @@ mod tests { assert!(matches!(cancelled, crab_staging::StagingError::Cancelled)); assert_eq!(manifest_probes(), initial_probes); classifier.close().await; + assert!( + candidate_cache + .load_persistent(&[MerkleHash::from([1; 32]), MerkleHash::from([2; 32])]) + .expect("inspect persisted candidates") + .is_empty(), + "an absent index must not write a per-chunk negative cache" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn add_classifier_persists_misses_from_an_initialized_index() { + let cache_root = tempfile::tempdir().expect("private candidate cache"); + let (store, router) = test_store_router("repo-add-initialized-index"); + let seed = build_push_metadb_guard( + &store, + &router, + None, + &crate::core::config::MetaDbTomlConfig::default(), + false, + ); + seed.chunk_index_system_keys().await.expect("seed manifest"); + seed.close().await.expect("close seed writer"); + let (classifier, candidate_cache) = + add_classifier_with_private_cache(&store, &router, cache_root.path()); + let hash = MerkleHash::from([4; 32]); + let lookup = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( + &classifier, + &[(hash, 4)], + ) + .await; + classifier.close().await; + lookup.expect("initialized-index lookup"); + assert_eq!( + candidate_cache + .load_persistent(&[hash]) + .expect("inspect persisted miss") + .get(&hash), + Some(&None) + ); } #[tokio::test(flavor = "multi_thread")] @@ -30527,7 +30903,11 @@ mod tests { range_gets.store(0, Ordering::Relaxed); let verified = pipeline - .verify_xorb_refs_with_committed_receipts(&refs, &HashMap::from([(xorb_hash, receipt)])) + .verify_xorb_refs_with_committed_receipts( + &refs, + &HashMap::from([(xorb_hash, receipt)]), + false, + ) .await .expect("verify refs from committed origin receipt"); diff --git a/crab/src/git/push/tests/capsule_chunk_tests.rs b/crab/src/git/push/tests/capsule_chunk_tests.rs new file mode 100644 index 000000000..0f60b530e --- /dev/null +++ b/crab/src/git/push/tests/capsule_chunk_tests.rs @@ -0,0 +1,480 @@ +use super::*; + +struct FirstCandidateOnly(Arc); + +#[async_trait::async_trait] +impl crab_staging::push_plan::ExistingChunkLookup for FirstCandidateOnly { + async fn lookup_existing_candidates( + &self, + chunks: &[(MerkleHash, u64)], + ) -> crab_staging::Result>> { + let candidates = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( + self.0.as_ref(), + chunks, + ) + .await?; + Ok(candidates + .into_iter() + .enumerate() + .map( + |(index, candidate)| { + if index == 0 { candidate } else { None } + }, + ) + .collect()) + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { + use crab_staging::push_plan::ExistingChunkLookup; + use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; + + let _git_env = CleanGitEnvGuard::new(); + let content = test_deterministic_bytes(0x4cab, 2 * 1024 * 1024); + let source = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .args(args) + .current_dir(source.path()) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + git(&["init", "-b", "main"]); + let staging_root = source.path().join("staging"); + let staging = crab_staging::StagingArea::open(staging_root.clone()) + .await + .unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let chunks = staging + .recipe_page(&staged.recipe, 0) + .unwrap() + .chunks + .into_iter() + .map(|chunk| (chunk.chunk_hash, chunk.len)) + .collect::>(); + staging + .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + let pointer = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); + git(&["add", "model.bin"]); + git(&["commit", "-m", "shared payload"]); + staging.close().await.unwrap(); + let staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(staging_root) + .await + .unwrap(), + ); + let reads = Arc::new(RangeCountingStore::new(Arc::new(AtomicUsize::new(0)))); + let store = Store::new(reads.clone()); + let source_router = StoreLayout::new(store.clone(), "repos/source".to_owned()); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..Default::default() + }; + let (result, _) = crate::git::capsule_push::run( + &config, + &[crate::git::remote_helper::PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &source_router, + None, + &[], + Some(&staging), + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(result.all_ok(), "source publication failed: {result:?}"); + drop(staging); + + let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); + let pipeline = PushPipeline::new( + PushConfig::default(), + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + CancellationToken::new(), + None, + ); + let cache_root = tempfile::tempdir().unwrap(); + let mut metadata = crate::core::config::MetaDbTomlConfig::default(); + metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard( + &store, &router, None, &metadata, true, + )); + // No advisory caches or source-repository state can manufacture a hit. + // This exercises the real publisher-to-classifier authority boundary. + let classifier = Arc::new(AddRemoteChunkClassifier { + pipeline, + remote_chunk_index: tokio::sync::OnceCell::new(), + candidate_cache: None, + candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), + candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), + }); + let candidates = classifier.lookup_existing_candidates(&chunks).await; + let consumer_staging_root = cache_root.path().join("consumer-staging"); + let consumer_staging = crab_staging::StagingArea::open(consumer_staging_root.clone()) + .await + .unwrap(); + let remote_staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &consumer_staging, + StreamStageProgress { + existing_lookup: Some(classifier.clone()), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await; + let partial_root = cache_root.path().join("partial-staging"); + let partial_staging = crab_staging::StagingArea::open(partial_root.clone()) + .await + .unwrap(); + let partial_staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &partial_staging, + StreamStageProgress { + existing_lookup: Some(Arc::new(FirstCandidateOnly(classifier.clone()))), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await; + classifier.close().await; + let partial_staged = partial_staged.unwrap(); + partial_staging + .delegate_publication(&[(&partial_staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + partial_staging.close().await.unwrap(); + let remote_staged = remote_staged.unwrap(); + consumer_staging + .delegate_publication(&[(&remote_staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + consumer_staging.close().await.unwrap(); + let candidates = candidates.unwrap(); + assert!( + !candidates.is_empty() && candidates.iter().all(Option::is_some), + "committed source chunks must be reusable from a cold consumer: {candidates:?}" + ); + + let planned = chunks + .iter() + .zip(&candidates) + .filter_map(|((hash, _), candidate)| candidate.map(|candidate| (*hash, candidate))) + .collect::>(); + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + let catalog = crab_metadata::capsule_protocol::load_pointer_catalog_from_root(&layout, &root) + .await + .unwrap(); + let mut origins = HashMap::new(); + for candidate in planned.values() { + let path = layout.xorb_path(&candidate.xorb_ref.xorb_hash); + let (bytes, identity) = layout.store().get_with_etag(&path).await.unwrap(); + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).unwrap(); + let mut origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + candidate.xorb_ref.xorb_hash.into(), + parser.payload_digest(), + bytes.len() as u64, + identity.e_tag, + identity.version, + ); + // Change the head's proof identity while retaining the actual committed source. + origin.proven_at_unix_secs += 10; + origins.insert(candidate.xorb_ref.xorb_hash, origin); + } + crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), + "repos/source", + &catalog, + &origins, + ) + .await + .unwrap(); + let consumer_layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/consumer".to_owned()); + let verified = revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + verified.candidates.len(), + planned.len(), + "old exact proof survives a newer shared head" + ); + + for field in ["placement", "origin", "position"] { + let mut forged = planned.clone(); + for candidate in forged.values_mut() { + match field { + "placement" => candidate.placement_id = [9; 32], + "origin" => candidate.origin_proof_id = [9; 32], + "position" => candidate.xorb_ref.chunk_index += 1, + _ => unreachable!(), + } + } + assert!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &forged, + &CancellationToken::new() + ) + .await + .unwrap() + .candidates + .is_empty(), + "accepted forged {field}" + ); + } + + let consumer_base = crab_write::capsule_protocol::initialize( + &consumer_layout, + &"2".repeat(64), + "refs/heads/main", + ) + .await + .unwrap(); + let consumer_staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(consumer_staging_root) + .await + .unwrap(), + ); + reads.full_xorb_gets.store(0, Ordering::Relaxed); + let (delta, stats) = crate::git::xet_publication::prepare_delta( + &consumer_layout, + &consumer_base, + &[crab_types::pointer::Pointer { + file_hash: remote_staged.file_hash, + size: remote_staged.size, + shard_hint: None, + }], + Some(&consumer_staging), + None, + None, + &metadata, + 1, + true, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + (delta.files().len(), stats.xorbs_uploaded), + (1, 0), + "remote-only staging must publish without repacking" + ); + assert_eq!( + reads.full_xorb_gets.load(Ordering::Relaxed), + 0, + "an exact, current source/origin proof must not download the reused xorb body" + ); + drop(consumer_staging); + let partial_staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(partial_root) + .await + .unwrap(), + ); + reads.full_xorb_gets.store(0, Ordering::Relaxed); + let (_, partial_stats) = crate::git::xet_publication::prepare_delta( + &consumer_layout, + &consumer_base, + &[crab_types::pointer::Pointer { + file_hash: partial_staged.file_hash, + size: partial_staged.size, + shard_hint: None, + }], + Some(&partial_staging), + None, + None, + &metadata, + 1, + true, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + ( + partial_stats.xorbs_uploaded, + reads.full_xorb_gets.load(Ordering::Relaxed) + ), + (0, 0), + "partial add-time hints must reuse the authenticated requested closure without packing or downloading" + ); + drop(partial_staging); + + let source_key = layout.capsule_root_path(); + let (source_bytes, _) = layout.store().get_with_etag(&source_key).await.unwrap(); + layout.store().delete(&source_key).await.unwrap(); + let unrooted = revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new(), + ) + .await + .unwrap(); + layout.store().put(&source_key, source_bytes).await.unwrap(); + assert!( + unrooted.candidates.is_empty(), + "physical bytes alone cannot replace a source root" + ); + + for candidate in planned.values() { + let key = layout.xorb_path(&candidate.xorb_ref.xorb_hash); + let (bytes, previous) = layout.store().get_with_etag(&key).await.unwrap(); + // Bypass the immutable adapter only to inject a provider-side replacement. + let replaced = layout + .store() + .inner() + .put(&key, bytes.into()) + .await + .unwrap(); + assert_ne!( + (previous.e_tag, previous.version), + (replaced.e_tag, replaced.version), + "the fault must actually change the provider identity" + ); + } + assert!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new() + ) + .await + .unwrap() + .candidates + .is_empty(), + "a replacement identity cannot reuse the original proof" + ); + let verified_refs = verified.into_refs(); + let cached = classifier + .pipeline + .verify_xorb_refs_with_committed_receipts(&verified_refs, &origins, true) + .await + .unwrap(); + assert!( + cached.is_empty(), + "warm proofs cannot certify the replacement identity" + ); + + // A fresh attestation for one occurrence must not certify another + // occurrence's stale provider identity in the same immutable xorb. + let mut fresh_origins = HashMap::new(); + for hash in planned + .values() + .map(|candidate| candidate.xorb_ref.xorb_hash) + .collect::>() + { + let key = layout.xorb_path(&hash); + let (bytes, identity) = layout.store().get_with_etag(&key).await.unwrap(); + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).unwrap(); + parser.verify_payload_digest().unwrap(); + parser.verify_all_chunks().unwrap(); + fresh_origins.insert( + hash, + crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + key.to_string(), + hash.into(), + parser.payload_digest(), + bytes.len() as u64, + identity.e_tag, + identity.version, + ), + ); + } + crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), + "repos/source", + &catalog, + &fresh_origins, + ) + .await + .unwrap(); + let guard = build_push_metadb_guard(&store, &router, None, &metadata, true); + let chunk_store = guard.chunk_index().await.unwrap(); + let hashes = planned.keys().copied().collect::>(); + let fresh = classifier + .pipeline + .lookup_capsule_chunks_for_add(&hashes, &chunk_store) + .await; + guard.close().await.unwrap(); + let fresh = fresh.unwrap(); + assert_eq!( + fresh.len(), + planned.len(), + "replacement proof fixture must be independently valid" + ); + assert!( + fresh.len() > 1, + "mixed-origin coverage needs distinct chunks in one xorb" + ); + let mut mixed = planned.clone(); + let (hash, candidate) = fresh.into_iter().next().unwrap(); + mixed.insert(hash, candidate); + assert!( + matches!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &mixed, + &CancellationToken::new(), + ) + .await, + Err(CrabError::CorruptObject { .. }) + ), + "conflicting immutable-xorb identities must not certify one another" + ); +} diff --git a/crab/src/git/remote_helper.rs b/crab/src/git/remote_helper.rs index b95344ac3..107c4967a 100644 --- a/crab/src/git/remote_helper.rs +++ b/crab/src/git/remote_helper.rs @@ -1105,7 +1105,11 @@ where match cmd { HelperCommand::Option { key, value } => { - handle_option(&key, &value, options, writer).await?; + tokio::select! { + biased; + () = cancellation.cancelled() => return Err(CrabError::Cancelled), + result = handle_option(&key, &value, options, writer) => result?, + } } HelperCommand::Capabilities => { return Ok(Some(Batch::Capabilities)); @@ -1223,22 +1227,39 @@ async fn handle_option( } }, "deepen-since" => { - let timestamp = value - .parse::() - .map_err(|_| CrabError::Protocol(format!("invalid deepen-since value: {value}")))?; - if options - .fetch_options - .deepen_since - .replace(timestamp) - .is_some() - { + if options.fetch_options.deepen_since.is_some() { return Err(CrabError::Protocol( "duplicate deepen-since option".to_owned(), )); } + // Helpers receive Git date expressions; only upload-pack wire requests + // contain epoch seconds. Use Git's approxidate parser for the same grammar. + let date = decode_option_value(value)?; + let output = tokio::process::Command::new("git") + .arg("rev-parse") + .arg(format!("--since={date}")) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .output() + .await?; + if !output.status.success() { + return Err(CrabError::Protocol(format!( + "cannot parse deepen-since date: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + let timestamp = std::str::from_utf8(&output.stdout) + .ok() + .and_then(|line| line.trim().strip_prefix("--max-age=")) + .and_then(|seconds| seconds.parse::().ok()) + .ok_or_else(|| { + CrabError::Protocol("invalid Git deepen-since timestamp".to_owned()) + })?; + options.fetch_options.deepen_since = Some(timestamp); writer.write_all(b"ok\n").await?; } "deepen-not" => { + let value = decode_option_value(value)?; if value.is_empty() || value.bytes().any(|byte| byte.is_ascii_whitespace()) { return Err(CrabError::Protocol( "deepen-not requires one non-empty reference".to_owned(), @@ -1248,9 +1269,9 @@ async fn handle_option( .fetch_options .deepen_not .iter() - .any(|name| name == value) + .any(|name| name == &value) { - options.fetch_options.deepen_not.push(value.to_owned()); + options.fetch_options.deepen_not.push(value); } writer.write_all(b"ok\n").await?; } @@ -1356,19 +1377,25 @@ async fn handle_option( Ok(()) } -fn parse_ref_lease(value: &str) -> Result<(String, Option)> { +fn decode_option_value(value: &str) -> Result { if value.starts_with('"') && !value.ends_with('"') { return Err(CrabError::Protocol( - "unterminated quoted ref lease".to_owned(), + "unterminated quoted helper option".to_owned(), )); } let (decoded, consumed) = gix_quote::ansi_c::undo(value.as_bytes().into()) - .map_err(|error| CrabError::Protocol(format!("invalid quoted ref lease: {error}")))?; + .map_err(|error| CrabError::Protocol(format!("invalid quoted helper option: {error}")))?; if consumed != value.len() { - return Err(CrabError::Protocol("trailing data in ref lease".to_owned())); + return Err(CrabError::Protocol( + "trailing data in helper option".to_owned(), + )); } - let decoded = std::str::from_utf8(decoded.as_ref()) - .map_err(|error| CrabError::Protocol(format!("ref lease is not UTF-8: {error}")))?; + String::from_utf8(decoded.to_vec()) + .map_err(|error| CrabError::Protocol(format!("helper option is not UTF-8: {error}"))) +} + +fn parse_ref_lease(value: &str) -> Result<(String, Option)> { + let decoded = decode_option_value(value)?; let (name, oid) = decoded .rsplit_once(':') .ok_or_else(|| CrabError::Protocol("ref lease requires ref:oid".to_owned()))?; @@ -5732,7 +5759,7 @@ mod tests { let (source, store, router, commits, _tag) = capsule_history_fixture().await; let cutoff = String::from_utf8(run_git( source.path(), - &["show", "-s", "--format=%ct", &commits[1]], + &["show", "-s", "--format=%cI", &commits[1]], )) .expect("commit timestamp is utf8") .trim() @@ -5764,6 +5791,40 @@ mod tests { assert!(!git_object_exists(target.path(), &commits[0])); } + #[tokio::test] + async fn classic_capsule_empty_shallow_selectors_install_nothing() { + let (_source, store, router, commits, _tag) = capsule_history_fixture().await; + for selector in ["deepen-since 1893456000", "deepen-not main"] { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let git_dir = target.path().join(".git"); + let _guard = GitEnvCwdGuard::set(target.path(), &git_dir, target.path()); + let tip = commits.last().expect("history tip"); + let input = format!("option {selector}\nfetch {tip} refs/heads/main\n\n"); + let context = test_context( + store.clone(), + router.repo_prefix(), + target.path().join("push-state"), + ); + let (_, result) = + run_with_context(&input, context, tokio_util::sync::CancellationToken::new()).await; + let error = result.expect_err("empty shallow selection must fail"); + assert!( + error + .to_string() + .contains("no commits selected for shallow requests") + ); + assert_eq!( + ( + git_object_exists(target.path(), tip), + git_dir.join("shallow").exists() + ), + (false, false), + "{selector} must fail before pack installation or shallow publication" + ); + } + } + #[tokio::test] async fn capsule_promisor_fetch_installs_the_authorized_object() { let (store, router, blob) = capsule_promisor_fixture().await; @@ -7386,6 +7447,9 @@ mod tests { #[tokio::test] async fn shallow_selectors_are_accepted_without_degrading_to_full_fetch() { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let _guard = GitEnvCwdGuard::set(target.path(), &target.path().join(".git"), target.path()); let mut options = HelperOptions::default(); let mut output = Vec::new(); handle_option("deepen-since", "1700000000", &mut options, &mut output) @@ -7404,6 +7468,30 @@ mod tests { assert_eq!(String::from_utf8(output).unwrap(), "ok\nok\n"); } + #[tokio::test] + async fn option_shallow_since_normalizes_git_dates_and_quoting() { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let _guard = GitEnvCwdGuard::set(target.path(), &target.path().join(".git"), target.path()); + for date in [ + "1700000000", + "2023-11-14T22:13:20Z", + "2023-11-14T23:13:20+01:00", + "\"2023-11-14 22:13:20 +0000\"", + ] { + let mut options = HelperOptions::default(); + let mut output = Vec::new(); + handle_option("deepen-since", date, &mut options, &mut output) + .await + .unwrap(); + assert_eq!( + (options.fetch_options.deepen_since, output), + (Some(1_700_000_000), b"ok\n".to_vec()), + "{date}" + ); + } + } + #[tokio::test] async fn option_update_shallow_accepts_booleans_and_rejects_other_values() { assert_eq!(run("option update-shallow true\n").await, "ok\n"); diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 3e13c2392..266fa1d6f 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -23,11 +23,12 @@ use crate::core::metrics::Metrics; // existing payload bound even when a caller requests more network concurrency. const MAX_IN_FLIGHT_XORBS: usize = 4; -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub(crate) struct XetPublicationStats { pub xorbs_uploaded: u64, pub shards_uploaded: u64, pub xorb_bytes_uploaded: u64, + pub origins: HashMap, } #[expect( @@ -41,6 +42,7 @@ pub(crate) async fn prepare_delta( staging: Option<&Arc>, caching_store: Option<&crab_cache_store::CachingStore>, metrics: Option<&Metrics>, + metadb: &crate::core::config::MetaDbTomlConfig, upload_concurrency: usize, publish_gc_roots: bool, cancel: &CancellationToken, @@ -65,7 +67,8 @@ pub(crate) async fn prepare_delta( }, ) .await?; - let lookup = view.pointer_catalog_lookup(layout, 2 * 1024 * 1024 * 1024)?; + let lookup = + view.pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; drop(view); let mut pointer_sizes = BTreeMap::::new(); @@ -133,11 +136,12 @@ pub(crate) async fn prepare_delta( let mut shard_session = PushShardSession::new(); let mut pending_files = Vec::with_capacity(unresolved.len()); let mut uploaded_xorbs = HashSet::new(); + let mut origins = HashMap::new(); let mut xorb_bytes_uploaded = 0_u64; let mut prepared_files = Vec::with_capacity(unresolved.len()); let mut prepared_candidates = Vec::::new(); let mut candidate_indices = HashMap::::new(); - let mut external_candidates = HashMap::>::new(); + let mut staged_candidates = HashMap::new(); for (file_hash, file_size) in unresolved { check_cancelled(cancel)?; @@ -206,11 +210,8 @@ pub(crate) async fn prepare_delta( expected_size ))); } - let refs = external_candidates - .entry(candidate.xorb_ref.xorb_hash) - .or_default(); - match refs.insert(chunk_hash, candidate.xorb_ref) { - Some(existing) if existing != candidate.xorb_ref => { + match staged_candidates.insert(chunk_hash, candidate) { + Some(existing) if existing != candidate => { return Err(CrabError::StagingCorrupt(format!( "remote xorb authority disagrees for chunk {}", chunk_hash.hex() @@ -263,33 +264,97 @@ pub(crate) async fn prepare_delta( drop(base_chunks); drop(needed_chunks); - let mut external_candidates = external_candidates - .into_iter() - .filter(|(hash, refs)| { - !xorb_entries.contains_key(hash) - && refs - .keys() - .any(|chunk_hash| !placements.contains_key(chunk_hash)) - }) - .collect::>(); + // The recipe retains the exact add-time IDs. Re-read their immutable + // records and current source roots before trusting any remote-only bytes. + staged_candidates.retain(|hash, _| !placements.contains_key(hash)); + let planned_existing = staged_candidates.len(); + let verified_staged = + super::push::revalidate_staged_remote_chunks(metadb, layout, &staged_candidates, cancel) + .await?; + tracing::debug!( + planned_existing, + stale_existing = planned_existing.saturating_sub(verified_staged.candidates.len()), + global_existing = 0, + "capsule staged remote proof adoption complete", + ); + let mut reused_xorbs = verified_staged.xorbs.len(); + let requested = requested_chunks + .iter() + .map(String::as_str) + .collect::>(); + for (hash, (entry, origin)) in verified_staged.xorbs { + if xorb_entries + .get(&hash) + .is_some_and(|existing| existing != &entry) + { + return Err(CrabError::CorruptObject { + path: layout.xorb_path(&hash).to_string(), + reason: "verified xorb differs from the captured destination catalog".to_owned(), + }); + } + // The authenticated closure also proves requested chunks whose rebuildable + // heads were absent during add. Expand only outgoing occurrences, so an + // incomplete global index cannot force repacking the same verified bytes. + for (index, chunk) in entry.chunks().iter().enumerate() { + if requested.contains(chunk.hash()) { + let chunk_hash = parse_merkle_hash(chunk.hash(), "verified catalog chunk")?; + placements.entry(chunk_hash).or_insert(ChunkPlacement { + chunk_hash, + xorb_hash: hash, + chunk_index: u32::try_from(index).map_err(|_| { + CrabError::Internal("verified xorb has too many chunks".to_owned()) + })?, + uncompressed_size: chunk.uncompressed_size(), + }); + } + } + xorb_entries.insert(hash, entry); + origins.insert(hash, origin); + } + drop(requested); + let mut external_candidates = HashMap::>::new(); + for (hash, candidate) in verified_staged.candidates { + let xorb_ref = candidate.xorb_ref; + if xorb_entries.contains_key(&xorb_ref.xorb_hash) { + // Exact source membership and current provider identity already + // certify this descriptor. Re-reading payload would download a + // whole reused file despite the committed origin receipt. + placements.entry(hash).or_insert(ChunkPlacement { + chunk_hash: hash, + xorb_hash: xorb_ref.xorb_hash, + chunk_index: xorb_ref.chunk_index, + uncompressed_size: xorb_ref.uncompressed_size, + }); + } else { + // Manifest receipts shipped in v1.2.4 lack capsule body descriptors. + // Full-body verification supplies that value when importing them + // into capsule publication. + external_candidates + .entry(xorb_ref.xorb_hash) + .or_default() + .insert(hash, xorb_ref); + } + } + let mut external_candidates = external_candidates.into_iter().collect::>(); external_candidates.sort_by_key(|(hash, _)| hash.hex()); let external_reads = futures_util::stream::iter(external_candidates.into_iter().enumerate().map( |(ordinal, (expected_hash, expected_refs))| async move { check_cancelled(cancel)?; let path = layout.xorb_path(&expected_hash); - let bytes = match layout + let (bytes, identity) = match layout .store() .get_with_etag_bounded(&path, MAX_XORB_SIZE as u64) .await { - Ok((bytes, _)) => bytes, + Ok(result) => result, Err(crab_storage::StorageError::NotFound { .. }) => { return Ok::<_, CrabError>((ordinal, None)); } Err(error) => return Err(error.into()), }; - let verified = verify_external_xorb(&path, expected_hash, &expected_refs, bytes)?; + let verified = + verify_external_xorb(&path, expected_hash, &expected_refs, bytes, identity)?; Ok((ordinal, verified)) }, )) @@ -300,12 +365,12 @@ pub(crate) async fn prepare_delta( verified_external.push(result?); } verified_external.sort_by_key(|(ordinal, _)| *ordinal); - let mut reused_xorbs = 0_usize; for (_, verified) in verified_external { - let Some((xorb_hash, entry, xorb_placements)) = verified else { + let Some((xorb_hash, entry, xorb_placements, origin)) = verified else { continue; }; reused_xorbs += 1; + origins.insert(xorb_hash, origin); for placement in xorb_placements { placements.entry(placement.chunk_hash).or_insert(placement); } @@ -375,7 +440,7 @@ pub(crate) async fn prepare_delta( let bytes = Bytes::from(bytes); let (entry, xorb_placements) = verify_prepared_xorb(&path, planned_hash, &planned, bytes.clone())?; - let (entry, xorb_placements, created) = publish_xorb_candidate( + let (entry, xorb_placements, created, origin) = publish_xorb_candidate( layout, caching_store, planned_hash, @@ -391,6 +456,7 @@ pub(crate) async fn prepare_delta( entry, xorb_placements, uploaded_bytes, + origin, )) }, )) @@ -400,8 +466,11 @@ pub(crate) async fn prepare_delta( while let Some(result) = uploads.next().await { verified_candidates.push(result?); } - verified_candidates.sort_by_key(|(ordinal, _, _, _, _)| *ordinal); - for (_, planned_hash, entry, xorb_placements, uploaded_bytes) in verified_candidates { + verified_candidates.sort_by_key(|(ordinal, _, _, _, _, _)| *ordinal); + for (_, planned_hash, entry, xorb_placements, uploaded_bytes, origin) in verified_candidates { + if let Some(origin) = origin { + origins.insert(planned_hash, origin); + } if uploaded_bytes > 0 { uploaded_xorbs.insert(planned_hash); xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -449,6 +518,7 @@ pub(crate) async fn prepare_delta( &mut placements, &mut xorb_entries, &mut uploaded_xorbs, + &mut origins, ) .await?; xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -466,6 +536,7 @@ pub(crate) async fn prepare_delta( &mut placements, &mut xorb_entries, &mut uploaded_xorbs, + &mut origins, ) .await?; xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -608,6 +679,7 @@ pub(crate) async fn prepare_delta( xorbs_uploaded: uploaded_xorbs.len() as u64, shards_uploaded, xorb_bytes_uploaded, + origins, }, )) } @@ -619,11 +691,12 @@ async fn publish_built_xorb( placements: &mut ChunkPlacementMap, xorb_entries: &mut HashMap, uploaded_xorbs: &mut HashSet, + origins: &mut HashMap, ) -> Result { let encoded_size = result.bytes.len() as u64; let body_digest = blake3::hash(&result.bytes).to_hex().to_string(); let entry = catalog_xorb_entry(result.bytes.len() as u64, body_digest, &result.placements)?; - let (entry, xorb_placements, created) = publish_xorb_candidate( + let (entry, xorb_placements, created, origin) = publish_xorb_candidate( layout, caching_store, result.hash, @@ -632,6 +705,9 @@ async fn publish_built_xorb( result.placements, ) .await?; + if let Some(origin) = origin { + origins.insert(result.hash, origin); + } if created { uploaded_xorbs.insert(result.hash); } @@ -653,6 +729,7 @@ async fn publish_xorb_candidate( crab_metadata::capsule_protocol::XorbCatalogEntry, Vec, bool, + Option, )> { let path = layout.xorb_path(&xorb_hash); let staged = layout.store().staging_write_prefix().is_some(); @@ -691,16 +768,22 @@ async fn publish_xorb_candidate( .get_with_etag_bounded(&path, MAX_XORB_SIZE as u64) .await { - Ok((existing, _)) => { - let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? - .ok_or_else(|| CrabError::CorruptObject { - path: path.to_string(), - reason: format!( - "existing xorb {} does not authenticate as the requested logical content", - xorb_hash.hex() - ), - })?; - return Ok((verified.1, verified.2, false)); + Ok((existing, identity)) => { + let verified = verify_external_xorb( + &path, + xorb_hash, + &expected_refs, + existing, + identity, + )? + .ok_or_else(|| CrabError::CorruptObject { + path: path.to_string(), + reason: format!( + "existing xorb {} does not authenticate as the requested logical content", + xorb_hash.hex() + ), + })?; + return Ok((verified.1, verified.2, false, Some(verified.3))); } Err(crab_storage::StorageError::NotFound { .. }) => {} Err(error) => return Err(error.into()), @@ -711,15 +794,18 @@ async fn publish_xorb_candidate( // canonical encoding above when present; otherwise record verified // staged bytes for service-owned promotion before ref publication. let created = layout.store().put_if_absent_verified(&path, bytes).await?; - return Ok((local_entry, local_placements, created)); + return Ok((local_entry, local_placements, created, None)); } let cache_bytes = caching_store.map(|_| bytes.clone()); + // Local builder/prepared verification already authenticated these bytes. + // Carry the payload digest with the exact immutable PUT/readback identity. + let payload_digest = XorbParser::parse(bytes.clone())?.payload_digest(); match layout .store() .create_or_read_immutable(&path, bytes, MAX_XORB_SIZE as u64) .await? { - crab_storage::ImmutableCreateOutcome::Created => { + crab_storage::ImmutableCreateOutcome::Created(identity) => { if let (Some(cache), Some(bytes)) = (caching_store, cache_bytes) { warm_published_cache_object( cache, @@ -729,18 +815,36 @@ async fn publish_xorb_candidate( ) .await; } - Ok((local_entry, local_placements, true)) + let origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + xorb_hash.into(), + payload_digest, + local_entry.encoded_size(), + identity.e_tag, + identity.version, + ); + Ok((local_entry, local_placements, true, Some(origin))) } - crab_storage::ImmutableCreateOutcome::Existing(existing) => { - let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? - .ok_or_else(|| CrabError::CorruptObject { - path: path.to_string(), - reason: format!( - "existing xorb {} does not authenticate as the requested logical content", - xorb_hash.hex() - ), - })?; - Ok((verified.1, verified.2, false)) + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, + identity, + } => { + let verified = verify_external_xorb( + &path, + xorb_hash, + &expected_refs, + existing, + identity, + )? + .ok_or_else(|| CrabError::CorruptObject { + path: path.to_string(), + reason: format!( + "existing xorb {} does not authenticate as the requested logical content", + xorb_hash.hex() + ), + })?; + Ok((verified.1, verified.2, false, Some(verified.3))) } } } @@ -842,11 +946,13 @@ fn verify_external_xorb( expected_hash: MerkleHash, expected_refs: &HashMap, bytes: Bytes, + identity: crab_storage::ETag, ) -> Result< Option<( MerkleHash, crab_metadata::capsule_protocol::XorbCatalogEntry, Vec, + crab_metadata::receipts::OriginReceipt, )>, > { let encoded_size = bytes.len() as u64; @@ -919,7 +1025,16 @@ fn verify_external_xorb( } } let entry = catalog_xorb_entry(encoded_size, body_digest, &xorb_placements)?; - Ok(Some((expected_hash, entry, xorb_placements))) + let origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + expected_hash.into(), + parser.payload_digest(), + encoded_size, + identity.e_tag, + identity.version, + ); + Ok(Some((expected_hash, entry, xorb_placements, origin))) } fn verify_prepared_xorb( @@ -1183,6 +1298,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1262,6 +1378,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1343,6 +1460,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), 1, false, &CancellationToken::new(), @@ -1460,6 +1578,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), concurrency, false, &CancellationToken::new(), @@ -1503,6 +1622,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1566,6 +1686,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1683,7 +1804,7 @@ mod tests { layout.store().clone() }; let layout = crab_storage::StoreLayout::new(store, "repo".to_owned()); - let (entry, placements, created) = publish_xorb_candidate( + let (entry, placements, created, _origin) = publish_xorb_candidate( &layout, None, candidate.hash, diff --git a/crab/src/metadata/metadb/mod.rs b/crab/src/metadata/metadb/mod.rs index a4e4db9a0..7d5344963 100644 --- a/crab/src/metadata/metadb/mod.rs +++ b/crab/src/metadata/metadb/mod.rs @@ -27,6 +27,7 @@ use tracing::{debug, info, warn}; use crate::core::error::{CrabError, MetaDbError, Result}; use crate::core::metrics::Metrics; use crab_metadata::chunk_index::ChunkIndex; +use crab_metadata::key_codec::METADB_FORMAT_VERSION; use crab_metadata::persistent_chunk_index::PersistentChunkIndex; use crab_metadata::{key_codec, value_codec}; @@ -66,8 +67,6 @@ const DEFAULT_WAL_FLUSH_SIZE: u64 = 4 * 1024 * 1024; /// lookups. const DEFAULT_BLOOM_BITS_PER_KEY: u32 = 10; -const METADB_FORMAT_VERSION: u32 = 1; - /// Aggregated snapshot of `sys:*` keys in one database. /// /// Returned by [`MetaDb::file_index_system_keys`] and diff --git a/crab/src/metadata/metadb/stores/chunk_index.rs b/crab/src/metadata/metadb/stores/chunk_index.rs index 341d53898..0b1d91563 100644 --- a/crab/src/metadata/metadb/stores/chunk_index.rs +++ b/crab/src/metadata/metadb/stores/chunk_index.rs @@ -104,6 +104,91 @@ impl ChunkIndexStore { self } + /// Load capsule hints without interpreting them as released manifest receipts. + pub(crate) async fn capsule_candidates( + &self, + hashes: &[MerkleHash], + ) -> Result { + use crab_metadata::capsule_protocol::capsule_chunk_head_key; + let requests = hashes + .iter() + .map(|hash| (*hash, capsule_chunk_head_key(&(*hash).into()), None)) + .collect::>(); + self.capsule_candidate_records(&requests).await + } + + /// Read immutable capsule placements selected during add, independent of newer heads. + pub(crate) async fn capsule_candidates_by_id( + &self, + candidates: &[(MerkleHash, [u8; 32])], + ) -> Result { + let requests = candidates + .iter() + .map(|(hash, id)| { + ( + *hash, + crab_metadata::key_codec::capsule_chunk_placement_key(id), + Some(*id), + ) + }) + .collect::>(); + self.capsule_candidate_records(&requests).await + } + + async fn capsule_candidate_records( + &self, + requests: &[(MerkleHash, [u8; 33], Option<[u8; 32]>)], + ) -> Result { + use crab_metadata::capsule_protocol::{ + CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, + }; + let mut output = CapsuleChunkCandidates::default(); + // Stay below Db's dense range-scan threshold: unrelated chunk history + // must not turn an add-time point batch into a scan of the global index. + for requests in requests.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { + let keys = requests + .iter() + .map(|(_, key, _)| Bytes::copy_from_slice(key)) + .collect::>(); + let values = self.db.get_batch(&keys).await?; + let mut anchor_ids = HashSet::new(); + for ((hash, _, expected_id), value) in requests.iter().zip(values) { + let Some(value) = value else { + continue; + }; + let placement = CommittedChunkPlacement::decode(&value)?; + if placement.chunk_hash != <[u8; 32]>::from(*hash) + || expected_id.is_some_and(|id| placement.placement_id() != id) + { + return Err(CrabError::StagingCorrupt( + "capsule candidate hash differs from index key".to_owned(), + )); + } + anchor_ids.insert(placement.source_anchor_id); + output.placements.insert(placement.chunk_hash, placement); + } + let ids = anchor_ids + .into_iter() + .filter(|id| !output.anchors.contains_key(id)) + .collect::>(); + let keys = ids + .iter() + .map(|id| Bytes::copy_from_slice(&capsule_chunk_anchor_key(id))) + .collect::>(); + let values = self.db.get_batch(&keys).await?; + for (id, value) in ids.into_iter().zip(values) { + let value = value.ok_or_else(|| { + CrabError::StagingCorrupt( + "capsule candidate source anchor is absent".to_owned(), + ) + })?; + let anchor = CapsuleChunkAnchor::decode(&id, &value)?; + output.anchors.insert(id, anchor); + } + } + Ok(output) + } + /// Look up a single chunk hash through all three tiers. /// /// Cache-first: in-memory → persistent, when available → remote. diff --git a/crab/src/optimize/xorbs/executor.rs b/crab/src/optimize/xorbs/executor.rs index cd51c621c..3c7115ccc 100644 --- a/crab/src/optimize/xorbs/executor.rs +++ b/crab/src/optimize/xorbs/executor.rs @@ -476,8 +476,10 @@ async fn upload_destination( () = cancel.cancelled() => return Err(CrabError::Cancelled), }; match publication { - crab_storage::ImmutableCreateOutcome::Created => {} - crab_storage::ImmutableCreateOutcome::Existing(existing) => { + crab_storage::ImmutableCreateOutcome::Created(_) => {} + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => { let parser = XorbParser::parse(existing).map_err(CrabError::from)?; if parser.hash() != destination_hash { return Err(CrabError::CorruptObject { diff --git a/crates/crab-auth-server/src/receive/capsule.rs b/crates/crab-auth-server/src/receive/capsule.rs index 3a6475ac3..1a6a613eb 100644 --- a/crates/crab-auth-server/src/receive/capsule.rs +++ b/crates/crab-auth-server/src/receive/capsule.rs @@ -1,6 +1,6 @@ //! Protocol-v2 protected-push candidate verification. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use crab_auth::PushRefUpdate; use crab_coordination::write_coordinator::{CommitOutcome, CoordinatedRefUpdate}; @@ -27,6 +27,7 @@ pub(super) struct VerifiedCapsuleCandidate { pub replication_objects: Vec, pub publication: Capsule, dependency_copies: Vec, + origins: HashMap, } pub(super) async fn commit_capsule_candidate( @@ -102,9 +103,29 @@ pub(super) async fn commit_capsule_candidate( .map(Some); } - promote_staged_writes(ctx.store(), &plan.staged_objects).await?; - promote_dependency_copies(ctx, &verified.dependency_copies).await?; - if let Some(delta) = capsule.pointer_catalog_delta()? { + let mut origins = verified.origins; + let mut other_objects = Vec::new(); + for object in &plan.staged_objects { + if crab_storage::content_hash_from_path(&object.canonical_key, "xorbs").is_some() { + let bytes = read_verified_staged_object(ctx.store(), object).await?; + let origin = super::capsule_dependencies::promote_xorb_origin( + ctx.store(), + &object.canonical_key, + bytes, + ) + .await?; + origins.insert( + crab_xet::hash::MerkleHash::from(origin.content_hash), + origin, + ); + } else { + other_objects.push(object.clone()); + } + } + promote_staged_writes(ctx.store(), &other_objects).await?; + origins.extend(promote_dependency_copies(ctx, &verified.dependency_copies).await?); + let delta = capsule.pointer_catalog_delta()?.unwrap_or_default(); + if !delta.is_empty() { crab_metadata::ref_registry::union_register_repo_shards( ctx.store(), ctx.router(), @@ -122,12 +143,12 @@ pub(super) async fn commit_capsule_candidate( .iter() .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); let base = view.root_snapshot().clone(); - if changes_namespace { + let result = if changes_namespace { let layout = ctx.router().clone(); let repo_prefix = ctx.repo_prefix().to_owned(); let active_active = active_active.cloned(); let replication_objects = replication_objects.to_vec(); - return crab_write::with_ref_namespaces( + crab_write::with_ref_namespaces( ctx.store(), ctx.router(), &ref_names, @@ -158,19 +179,38 @@ pub(super) async fn commit_capsule_candidate( .await }, ) - .await; + .await + } else { + publish_verified_candidate( + ctx.store(), + ctx.router(), + ctx.repo_prefix(), + base, + &transaction, + &capsule, + active_active, + replication_objects, + ) + .await + }; + if result.is_ok() { + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + ctx.repo_prefix(), + ctx.router().global_prefix(), + ); + if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( + std::sync::Arc::clone(ctx.store().inner()), + &config, + ctx.repo_prefix(), + &delta, + &origins, + ) + .await + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + } } - publish_verified_candidate( - ctx.store(), - ctx.router(), - ctx.repo_prefix(), - base, - &transaction, - &capsule, - active_active, - replication_objects, - ) - .await + result } async fn publish_verified_candidate( @@ -358,7 +398,7 @@ pub(super) async fn verify_capsule_candidate( validate_capsule_transaction(plan, candidate)?; let candidate_delta = candidate.pointer_catalog_delta()?.unwrap_or_default(); - let (changed_paths, publication, replication_objects, dependency_copies) = + let (changed_paths, publication, replication_objects, dependency_copies, origins) = match prepare.view_scope.as_ref() { None => { if prepared_root != plan.base_root_digest { @@ -382,9 +422,9 @@ pub(super) async fn verify_capsule_candidate( ) .await?; catalog.apply(&candidate_delta)?; - let (objects, copies) = + let (objects, copies, origins) = verify_pointer_dependencies(ctx, plan, &catalog, &pointers, None).await?; - (changed_paths, candidate.clone(), objects, copies) + (changed_paths, candidate.clone(), objects, copies, origins) } Some(scope) => { let filtered_router = crab_storage::StoreLayout::with_global_prefix( @@ -445,7 +485,7 @@ pub(super) async fn verify_capsule_candidate( )?; let mut final_catalog = source_catalog; final_catalog.apply(&delta)?; - let (objects, copies) = verify_pointer_dependencies( + let (objects, copies, origins) = verify_pointer_dependencies( ctx, plan, &final_catalog, @@ -465,7 +505,13 @@ pub(super) async fn verify_capsule_candidate( } let publication = Capsule::build(&materialized.transaction, materialized.git_packs, sections)?; - (materialized.changed_paths, publication, objects, copies) + ( + materialized.changed_paths, + publication, + objects, + copies, + origins, + ) } }; let publication_transaction = publication.transaction()?; @@ -496,6 +542,7 @@ pub(super) async fn verify_capsule_candidate( replication_objects: replication_objects.into_iter().collect(), publication, dependency_copies, + origins, }) } diff --git a/crates/crab-auth-server/src/receive/capsule_dependencies.rs b/crates/crab-auth-server/src/receive/capsule_dependencies.rs index a3d692b3e..059be9987 100644 --- a/crates/crab-auth-server/src/receive/capsule_dependencies.rs +++ b/crates/crab-auth-server/src/receive/capsule_dependencies.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use bytes::Bytes; use crab_metadata::capsule_protocol::PointerCatalog; @@ -130,7 +130,11 @@ pub(super) async fn verify_pointer_dependencies( catalog: &PointerCatalog, pointers: &ReachablePointerScan, fallback: Option<&crab_storage::StoreLayout>, -) -> Result<(BTreeSet, Vec)> { +) -> Result<( + BTreeSet, + Vec, + HashMap, +)> { let mut referenced = BTreeSet::from([ctx .router() .capsule_path(&plan.run_hash) @@ -139,6 +143,7 @@ pub(super) async fn verify_pointer_dependencies( let mut verified_shards = BTreeMap::::new(); let mut verified_xorbs = BTreeSet::new(); let mut copies = Vec::new(); + let mut origins = HashMap::new(); for pointer in &pointers.crab_pointers { let file_hash = MerkleHash::from(pointer.file_hash).hex(); let file = catalog @@ -163,7 +168,7 @@ pub(super) async fn verify_pointer_dependencies( Some(bytes) => bytes.clone(), None => { let fallback_key = fallback.map(|layout| layout.shard_path(&shard_hash)); - let bytes = read_candidate_object( + let (bytes, _) = read_candidate_object( ctx, plan, &shard_key, @@ -217,7 +222,7 @@ pub(super) async fn verify_pointer_dependencies( referenced.insert(key.clone()); if verified_xorbs.insert(hash.to_owned()) { let fallback_key = fallback.map(|layout| layout.xorb_path(&xorb_hash)); - let bytes = read_candidate_object( + let (bytes, identity) = read_candidate_object( ctx, plan, &key, @@ -232,6 +237,12 @@ pub(super) async fn verify_pointer_dependencies( )); } validate_staged_xorb(&key, &bytes, &chunks)?; + if let Some(identity) = identity { + origins.insert( + xorb_hash, + verified_xorb_origin(&key, bytes.clone(), identity)?, + ); + } } } } @@ -252,7 +263,8 @@ pub(super) async fn verify_pointer_dependencies( ) .to_string(); referenced.insert(key.clone()); - let bytes = read_candidate_object(ctx, plan, &key, None, pointer.size, &mut copies).await?; + let (bytes, _) = + read_candidate_object(ctx, plan, &key, None, pointer.size, &mut copies).await?; if <[u8; 32]>::from(Sha256::digest(&bytes)) != pointer.oid { return Err(invalid( "candidate LFS object digest differs from its pointer", @@ -278,7 +290,7 @@ pub(super) async fn verify_pointer_dependencies( } } copies.dedup_by(|left, right| left.target_key == right.target_key); - Ok((referenced, copies)) + Ok((referenced, copies, origins)) } async fn read_candidate_object( @@ -288,8 +300,8 @@ async fn read_candidate_object( fallback_key: Option<&object_store::path::Path>, expected_size: u64, copies: &mut Vec, -) -> Result { - let bytes = match plan +) -> Result<(Bytes, Option)> { + let (bytes, identity) = match plan .staged_objects .iter() .find(|object| object.canonical_key == canonical_key) @@ -300,7 +312,10 @@ async fn read_candidate_object( "staged dependency size differs from the candidate catalog", )); } - read_verified_staged_object(ctx.store(), object).await? + ( + read_verified_staged_object(ctx.store(), object).await?, + None, + ) } None => match ctx .store() @@ -310,7 +325,7 @@ async fn read_candidate_object( ) .await { - Ok((bytes, _)) => bytes, + Ok((bytes, identity)) => (bytes, Some(identity)), Err(crab_storage::StorageError::NotFound { .. }) => { let fallback_key = fallback_key.ok_or_else(|| AuthServerError::NotFound { path: canonical_key.to_owned(), @@ -325,7 +340,7 @@ async fn read_candidate_object( size: expected_size, digest: blake3::hash(&bytes).to_hex().to_string(), }); - bytes + (bytes, None) } Err(error) => return Err(error.into()), }, @@ -335,13 +350,14 @@ async fn read_candidate_object( "candidate dependency size differs from the candidate catalog", )); } - Ok(bytes) + Ok((bytes, identity)) } pub(super) async fn promote_dependency_copies( ctx: &ReceiveContext, copies: &[DependencyCopy], -) -> Result<()> { +) -> Result> { + let mut origins = HashMap::new(); for copy in copies { let (bytes, _) = ctx .store() @@ -356,12 +372,64 @@ pub(super) async fn promote_dependency_copies( "filtered view dependency changed after protected verification", )); } - ctx.store() - .put_if_absent_verified( - &object_store::path::Path::from(copy.target_key.clone()), - bytes, - ) - .await?; + if content_hash_from_path(©.target_key, "xorbs").is_some() { + let origin = promote_xorb_origin(ctx.store(), ©.target_key, bytes).await?; + origins.insert(MerkleHash::from(origin.content_hash), origin); + } else { + ctx.store() + .put_if_absent_verified( + &object_store::path::Path::from(copy.target_key.clone()), + bytes, + ) + .await?; + } } - Ok(()) + Ok(origins) +} + +// Verification checked the complete body before promotion. Keep the immutable +// operation identity with that body so a later HEAD cannot certify a replacement. +pub(super) async fn promote_xorb_origin( + store: &crab_storage::Store, + key: &str, + bytes: Bytes, +) -> Result { + let expected = *blake3::hash(&bytes).as_bytes(); + let maximum = bytes.len() as u64; + let outcome = store + .create_or_read_immutable(&object_store::path::Path::from(key), bytes.clone(), maximum) + .await?; + let identity = match outcome { + crab_storage::ImmutableCreateOutcome::Created(identity) => identity, + crab_storage::ImmutableCreateOutcome::Existing { + bytes: stored, + identity, + } => { + if *blake3::hash(&stored).as_bytes() != expected { + return Err(conflict( + "canonical xorb differs from the verified protected body", + )); + } + identity + } + }; + verified_xorb_origin(key, bytes, identity) +} + +fn verified_xorb_origin( + key: &str, + bytes: Bytes, + identity: crab_storage::ETag, +) -> Result { + let size = bytes.len() as u64; + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes)?; + Ok(crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + key.to_owned(), + parser.hash().into(), + parser.payload_digest(), + size, + identity.e_tag, + identity.version, + )) } diff --git a/crates/crab-http-server/src/receive_tests.rs b/crates/crab-http-server/src/receive_tests.rs index 498dcb05e..60240fca3 100644 --- a/crates/crab-http-server/src/receive_tests.rs +++ b/crates/crab-http-server/src/receive_tests.rs @@ -415,17 +415,25 @@ async fn exercise(server: Arc, branch: &str) { .await .unwrap(); - let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&repo.layout) - .await - .unwrap() - .unwrap(); - let path_state_hash = indexes.path_state_hash().unwrap(); for corruption in [ "missing descriptor", "corrupt descriptor", "malformed record", "oversized record", ] { + // Readiness can precede its repair worker's completion. Drain it before + // injecting another fault, or that worker can replace the tested record + // with legitimate graph-only readiness before the corruption probe. + server.finish_maintenance().await.unwrap(); + let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&repo.layout) + .await + .unwrap() + .unwrap(); + assert_eq!( + indexes.state_digest(), + repo.open_view().await.unwrap().state_digest() + ); + let path_state_hash = indexes.path_state_hash().unwrap(); let descriptor = repo .layout .bulk_manifest_path("path-state", path_state_hash); diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 62c676373..993fdbcbb 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -130,6 +130,13 @@ foreground reads still load only the current bounded readiness record. An unchanged capture does not append a node. Missing/corrupt lifetime dependencies stop GC before deletion. A valid readiness record without lifetime ownership requires background browse maintenance before GC; there is no scan fallback. +Full capsule fsck also authenticates the retained chain, decodes complete +graph/path-state indexes, and verifies physical-index and original source bytes, +including superseded ref-only runs. Shared captures stream each source body once +during this phase. Fsck holds the verified lifetime/readiness capture through +payload verification and the final snapshot check. Revalidation reads only bounded +control records; valid movement requires a retry and corruption fails validation. +Mirror pointer-recipe probes do not initialize this full maintenance workload. Explicit history pruning trims superseded captures below the oldest retained checkpoint generation only after the authoritative history CAS succeeds. Current @@ -145,6 +152,9 @@ a per-ref head under `v2/history/source-heads/` and authenticated source-pointer segments under `v2/history/source-roots/`. Foreground catalog sessions stay read-only and keep their original source ranges. GC discovers these roots in its existing history enumeration; fsck authenticates their original run proofs. +Fsck retains the decoded heads' CAS tokens and the pruning floor through later +data and Git checks. It rechecks mutable authority without decoding history again; +concurrent movement requests a retry, while invalid control bytes fail validation. Explicit history pruning advances a persistent generation floor after history CAS, trims older segments, and prevents expired publishers from restoring them. These new lifetime records do not change capsule, checkpoint, or catalog formats. diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 89a06c4b9..407d15424 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -11,7 +11,7 @@ use super::{ use crate::error::{MetadataError, Result}; use crate::validation::{corrupt_object, validate_content_hash}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Head { version: u32, @@ -285,6 +285,113 @@ pub async fn load_retained_browse_indexes( } } +/// Verify retained index/source integrity against stable lifetime and readiness heads. +/// +/// Maintenance decodes complete graph/path indexes and streams original source +/// and physical-index bodies. Retain the returned capture and recheck it after +/// later phases. This proves retained bytes, never ref visibility. +#[cfg(feature = "remote-index")] +pub async fn verify_retained_browse_indexes( + layout: &StoreLayout, + maximum_bytes: u64, +) -> Result { + let head = load_head(layout).await?; + let current = super::load_browse_indexes(layout).await?; + let records = match &head { + Some((head, _)) => load_chain(layout, head).await?, + None => Vec::new(), + }; + if current + .as_ref() + .is_some_and(|current| !records.iter().any(|record| record.indexes() == current)) + { + return Err(corrupt("published browse indexes have no retained capture")); + } + let mut sources = BTreeSet::new(); + for record in records { + let graph = crate::split_commit_graph::load_split_commit_graph( + layout.store(), + layout, + record.indexes().commit_graph_hash(), + maximum_bytes.min(crate::split_commit_graph::DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES), + ) + .await?; + if graph.descriptor.generation != record.generation() { + return Err(corrupt( + "retained graph generation differs from its capture", + )); + } + if let Some(hash) = record.indexes().path_state_hash() { + crate::path_state::load_path_state( + layout.store(), + layout, + hash, + &graph, + maximum_bytes.min(crate::path_state::DEFAULT_MAX_PATH_STATE_BYTES), + ) + .await?; + } + if let Some(hash) = record.indexes().object_index_hash() { + crate::git_object_locator::PagedGitObjectIndex::verify_retained( + layout.store(), + layout, + hash, + record.indexes().state_digest(), + &graph.descriptor, + maximum_bytes, + ) + .await?; + } + // Captures may share original multi-gigabyte sources. Authenticate each + // immutable body once, including ref-only sources absent from current history. + for path in record.source_paths(layout) { + if !sources.insert(path.clone()) { + continue; + } + let size = layout.store().head(&path).await?.size; + if size > maximum_bytes { + return Err(corrupt("retained source exceeds byte limit")); + } + let hash = path + .filename() + .and_then(|hash| blake3::Hash::from_hex(hash).ok()) + .ok_or_else(|| corrupt("retained source path has no content identity"))?; + layout + .store() + .verify_size_and_hash(&path, size, hash.as_bytes()) + .await?; + } + } + // Browse maintenance can change lifetime without moving a Git ref. A + // concurrent prune/rebuild must yield a retry, never a clean mixed capture. + let capture = BrowseRetentionCapture { head, current }; + if !capture.is_current(layout).await? { + return Err(corrupt( + "browse retention changed during verification; retry fsck", + )); + } + Ok(capture) +} + +/// Verified browse lifetime and readiness authority retained across maintenance phases. +#[cfg(feature = "remote-index")] +#[derive(Debug)] +pub struct BrowseRetentionCapture { + head: Option<(Head, ETag)>, + current: Option, +} + +#[cfg(feature = "remote-index")] +impl BrowseRetentionCapture { + /// Revalidate control identity without decoding indexes or original sources again. + /// + /// Invalid records return an error; valid concurrent movement returns false. + pub async fn is_current(&self, layout: &StoreLayout) -> Result { + Ok(load_head(layout).await? == self.head + && super::load_browse_indexes(layout).await? == self.current) + } +} + /// Trim captured generations after authoritative history pruning commits. /// /// The caller holds the exclusive repository GC lease. Failure leaves extra diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index 61412516f..4fb9f5491 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -15,6 +15,332 @@ fn indexes(byte: char) -> BrowseIndexes { BrowseIndexes::new(byte.to_string().repeat(64), "b".repeat(64), None).unwrap() } +#[cfg(feature = "remote-index")] +async fn integrity_fixture() -> (StoreLayout, Vec) { + use super::super::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleRun, CapsuleTransaction, PackLayer, + PointerCatalog, + }; + use crate::git_object_locator::{ + GitObjectLocation, GitObjectLocator, GitObjectMetadata, GitPackInventoryEntry, + PagedGitObjectIndexWrite, + }; + use crate::path_state::{PathStateInput, append_path_state, upload_path_state}; + use crate::split_commit_graph::{ + CommitGraphInput, append_paged_commit_graph, load_split_commit_graph, + upload_split_commit_graph, + }; + + let layout = fixture(); + let mut manifest = crate::manifests::Manifest::default_for_repo("refs/heads/main"); + manifest.generation = 1; + let pack_id = crab_xet::hash::MerkleHash::from([3; 32]); + let inventory = [GitPackInventoryEntry { + pack_id, + object_count: 1, + pack_size: 42, + }]; + manifest.pack_index_hash = crate::manifests::compact_pack_index( + 1, + &[crate::manifests::PackManifestEntry { + pack_id: pack_id.to_string(), + size: 42, + content_hash: pack_id.to_string(), + ref_tips: Vec::new(), + object_count: 1, + }], + ) + .unwrap() + .0; + manifest.git_validation_digest = "b".repeat(64); + let graph = append_paged_commit_graph( + None, + 1, + manifest.pack_index_hash.clone(), + manifest.git_validation_digest.clone(), + &[[1; 20]], + vec![CommitGraphInput { + oid: [1; 20], + tree_oid: [2; 20], + commit_time: 1, + parents: Vec::new(), + }], + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(layout.store(), &layout, &graph) + .await + .unwrap(); + let graph_hash = graph.descriptor_hash.clone(); + let mut paths = vec![layout.repo_path(&graph.layers[0].reference.path)]; + let graph = load_split_commit_graph(layout.store(), &layout, &graph_hash, 1024 * 1024) + .await + .unwrap(); + let path_state = append_path_state( + None, + &graph, + vec![PathStateInput { + oid: [1; 20], + first_parent: None, + author: Vec::new(), + author_seconds: 1, + message: Vec::new(), + mutations: Vec::new(), + }], + ) + .unwrap(); + upload_path_state(layout.store(), &layout, &path_state, 1024 * 1024) + .await + .unwrap(); + paths.push(layout.repo_path(&path_state.layers[0].reference.path)); + let entries = [( + [1; 20], + GitObjectLocator { + ordinal: 0, + pack_id, + location: GitObjectLocation { + pack_offset: 12, + entry_len: 10, + crc32: 0, + }, + metadata: GitObjectMetadata::default(), + }, + )]; + let physical = + PagedGitObjectIndexWrite::build(&"a".repeat(64), &manifest, &inventory, &entries).unwrap(); + let hash = physical.upload(layout.store(), &layout).await.unwrap(); + paths.push( + crate::git_object_locator::PagedGitObjectIndex::retained_paths( + layout.store(), + &layout, + &hash, + &"a".repeat(64), + 1024 * 1024, + ) + .await + .unwrap()[1] + .clone(), + ); + let indexes = BrowseIndexes::new( + "a".repeat(64), + graph_hash.clone(), + Some(path_state.descriptor_hash), + ) + .unwrap() + .with_object_index(hash) + .unwrap(); + let transaction = CapsuleTransaction::new( + &"c".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("1".repeat(40)), + None, + )], + ) + .unwrap(); + let run = + CapsuleRun::leaf(Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap()).unwrap(); + let run_path = layout.capsule_path(run.hash()); + layout + .store() + .put(&run_path, run.bytes().clone()) + .await + .unwrap(); + paths.push(run_path); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from(vec![0x42; 1024]), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "3".repeat(40), + 1, + ) + .unwrap(); + let layer = PackLayer::build(&pack).unwrap(); + let layer_path = layout.capsule_pack_layer_path(layer.hash()); + layout + .store() + .put(&layer_path, layer.bytes().clone()) + .await + .unwrap(); + paths.push(layer_path); + let checkpoint = LayeredCheckpoint::build( + 1, + &"c".repeat(64), + vec![layer.source_descriptor().unwrap()], + PointerCatalog::new(), + None, + ) + .unwrap(); + let checkpoint_path = layout.capsule_checkpoint_path(checkpoint.hash()); + layout + .store() + .put(&checkpoint_path, checkpoint.bytes().clone()) + .await + .unwrap(); + paths.push(checkpoint_path); + retain_browse_indexes(&layout, &indexes, 1, Some(&checkpoint), &[pointer]) + .await + .unwrap(); + // The old bodies now belong only to retained capture lifetime, not the + // latest readiness or any authoritative checkpoint-history chain. + let latest = BrowseIndexes::new("d".repeat(64), graph_hash, None).unwrap(); + retain_browse_indexes(&layout, &latest, 1, None, &[]) + .await + .unwrap(); + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + latest.encode().unwrap(), + ) + .await + .unwrap(); + (layout, paths) +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn integrity_checks_every_retained_body_after_readiness_replacement() { + let (layout, paths) = integrity_fixture().await; + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); + for path in paths { + let original = layout.store().get_with_etag(&path).await.unwrap().0; + for missing in [false, true] { + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + } + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err(), + "missing={missing}, path={path}" + ); + layout + .store() + .put_overwrite(&path, original.clone()) + .await + .unwrap(); + } + } + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn verified_capture_distinguishes_control_movement_from_corruption() { + for readiness in [false, true] { + let layout = fixture(); + let capture = verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); + assert!(capture.is_current(&layout).await.unwrap()); + let path = if readiness { + let path = layout.capsule_browse_indexes_path(); + layout + .store() + .put(&path, indexes('a').encode().unwrap()) + .await + .unwrap(); + path + } else { + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + layout.browse_retained_head_path() + }; + assert!(!capture.is_current(&layout).await.unwrap()); + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + assert!(capture.is_current(&layout).await.is_err()); + } +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn advertised_readiness_requires_authenticated_lifetime_ownership() { + let layout = fixture(); + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + indexes('a').encode().unwrap(), + ) + .await + .unwrap(); + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err() + ); +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn retained_physical_index_must_match_its_graph_and_state() { + for field in [ + "state_digest", + "generation", + "pack_index_hash", + "git_validation_digest", + ] { + let (layout, _) = integrity_fixture().await; + let records = load_retained_browse_indexes(&layout).await.unwrap(); + let indexes = records[1].indexes(); + let path = + layout.bulk_manifest_path("git-object-index", indexes.object_index_hash().unwrap()); + let bytes = layout.store().get_with_etag(&path).await.unwrap().0; + let mut descriptor: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + descriptor[field] = if field == "generation" { + serde_json::json!(2) + } else { + serde_json::json!("e".repeat(64)) + }; + let bytes = Bytes::from(serde_json::to_vec(&descriptor).unwrap()); + let hash = blake3::hash(&bytes).to_hex().to_string(); + layout + .store() + .put(&layout.bulk_manifest_path("git-object-index", &hash), bytes) + .await + .unwrap(); + let indexes = indexes.clone().with_object_index(hash).unwrap(); + retain_browse_indexes(&layout, &indexes, 1, None, &[]) + .await + .unwrap(); + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err(), + "field={field}" + ); + } +} + #[tokio::test] async fn unchanged_capture_does_not_grow_retention_or_rewrite_its_head() { let layout = fixture(); diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs new file mode 100644 index 000000000..7ba9ebd52 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs @@ -0,0 +1,180 @@ +//! Rebuildable chunk hints whose source authority is a capsule catalog. +#[cfg(any(test, feature = "remote-index"))] +use std::collections::BTreeMap; +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::XorbCatalogEntry; +use crate::error::{MetadataError, Result}; +#[cfg(any(test, feature = "remote-index"))] +use crate::receipts::RECEIPT_SCHEMA_VERSION; +use crate::receipts::{CommittedChunkPlacement, OriginReceipt}; +use crab_xet::hash::MerkleHash; + +pub use crate::key_codec::{capsule_chunk_anchor_key, capsule_chunk_head_key}; + +/// Capsule source and verified physical origin for one xorb. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapsuleChunkAnchor { + pub source_repo_prefix: String, + pub source_shard_hash: String, + pub body_digest: String, + pub origin: OriginReceipt, +} + +impl CapsuleChunkAnchor { + /// Decode bounded provenance and verify its immutable index identity. + pub fn decode(id: &[u8; 32], bytes: &[u8]) -> Result { + if bytes.len() > 16 * 1024 { + return Err(corrupt("capsule source anchor exceeds its byte bound")); + } + let anchor: Self = serde_json::from_slice(bytes) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if anchor.id()? != *id { + return Err(corrupt("capsule source anchor digest differs")); + } + Ok(anchor) + } + + /// Identify this immutable source and origin assertion. + pub fn id(&self) -> Result<[u8; 32]> { + if self.source_repo_prefix.len() > 4096 + || self.source_shard_hash.len() != 64 + || self.body_digest.len() != 64 + { + return Err(corrupt("invalid capsule source identity")); + } + let bytes = serde_json::to_vec(self) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if bytes.len() > 16 * 1024 { + return Err(corrupt("capsule source anchor exceeds its byte bound")); + } + let mut hash = blake3::Hasher::new_derive_key("crab capsule chunk source v1"); + hash.update(&bytes); + Ok(*hash.finalize().as_bytes()) + } + + /// Check an exact placement against its committed shard closure and origin assertion. + pub fn validate_placement( + &self, + placement: &CommittedChunkPlacement, + catalog: &super::PointerCatalog, + object_key: &str, + ) -> Result<()> { + placement.validate()?; + let xorb_hash = MerkleHash::from(placement.xorb_hash).hex(); + let rooted = catalog + .shards() + .get(&self.source_shard_hash) + .is_some_and(|shard| shard.xorb_hashes().binary_search(&xorb_hash).is_ok()); + let entry = catalog + .xorbs() + .get(&xorb_hash) + .ok_or_else(|| corrupt("capsule placement xorb is absent from source catalog"))?; + self.validate_catalog_xorb(object_key, entry)?; + let chunk = entry.chunks().get(placement.chunk_index as usize); + if !rooted + || placement.source_anchor_id != self.id()? + || placement.origin_proof_id != self.origin.proof_id() + || placement.xorb_hash != self.origin.content_hash + || !chunk.is_some_and(|chunk| { + chunk.hash() == MerkleHash::from(placement.chunk_hash).hex() + && chunk.uncompressed_size() == placement.uncompressed_size + }) + { + return Err(corrupt( + "capsule placement differs from its source shard or origin", + )); + } + Ok(()) + } + + /// Check the source catalog's physical descriptor before verifying origin availability. + pub fn validate_catalog_xorb(&self, object_key: &str, entry: &XorbCatalogEntry) -> Result<()> { + if self.source_repo_prefix.is_empty() || self.body_digest != entry.body_digest() { + return Err(corrupt( + "source catalog differs from the capsule chunk anchor", + )); + } + self.origin.validate( + "canonical-origin", + object_key, + self.origin.content_hash, + entry.encoded_size(), + ) + } +} + +/// Untrusted hints, requiring current source-catalog and canonical-origin checks. +#[derive(Debug, Default)] +pub struct CapsuleChunkCandidates { + pub placements: HashMap<[u8; 32], CommittedChunkPlacement>, + pub anchors: HashMap<[u8; 32], CapsuleChunkAnchor>, +} + +#[cfg(any(test, feature = "remote-index"))] +/// Encode one bounded xorb's candidates after full verification and ref publication. +pub(crate) fn capsule_chunk_candidate_rows( + repo_prefix: &str, + source_shard_hash: &str, + xorb_hash: MerkleHash, + entry: &XorbCatalogEntry, + origin: &OriginReceipt, +) -> Result, Vec)>> { + let anchor = CapsuleChunkAnchor { + source_repo_prefix: repo_prefix.to_owned(), + source_shard_hash: source_shard_hash.to_owned(), + body_digest: entry.body_digest().to_owned(), + origin: origin.clone(), + }; + anchor.validate_catalog_xorb(&origin.object_key, entry)?; + if origin.content_hash != <[u8; 32]>::from(xorb_hash) { + return Err(corrupt("origin names a different xorb")); + } + let anchor_id = anchor.id()?; + let mut rows = vec![( + capsule_chunk_anchor_key(&anchor_id).to_vec(), + serde_json::to_vec(&anchor) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?, + )]; + let mut placements = BTreeMap::new(); + for (index, chunk) in entry.chunks().iter().enumerate() { + let chunk_hash = + MerkleHash::from_hex(chunk.hash()).map_err(|_| corrupt("invalid chunk hash"))?; + placements + .entry(<[u8; 32]>::from(chunk_hash)) + .or_insert(CommittedChunkPlacement { + schema_version: RECEIPT_SCHEMA_VERSION, + chunk_hash: chunk_hash.into(), + xorb_hash: xorb_hash.into(), + chunk_index: u32::try_from(index) + .map_err(|_| corrupt("chunk position overflows"))?, + uncompressed_size: chunk.uncompressed_size(), + origin_proof_id: origin.proof_id(), + source_anchor_id: anchor_id, + }); + } + for (hash, placement) in placements { + let encoded = placement.encode()?.to_vec(); + // Preserve the exact add-time assertion when another publication replaces + // the shared head. The placement digest also commits to the chunk key. + rows.push(( + crate::key_codec::capsule_chunk_placement_key(&placement.placement_id()).to_vec(), + encoded.clone(), + )); + rows.push((capsule_chunk_head_key(&hash).to_vec(), encoded)); + } + Ok(rows) +} + +fn corrupt(reason: &str) -> MetadataError { + MetadataError::CorruptObject { + path: "capsule chunk candidates".to_owned(), + reason: reason.to_owned(), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs new file mode 100644 index 000000000..c0f82c629 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs @@ -0,0 +1,147 @@ +use super::*; +use crate::capsule_protocol::XorbChunkEntry; + +fn fixture() -> (MerkleHash, XorbCatalogEntry, OriginReceipt) { + let hash = MerkleHash::from([3; 32]); + let entry = XorbCatalogEntry::new( + 100, + blake3::hash(b"whole body").to_hex().to_string(), + vec![ + XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9), + XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9), + ], + ); + let mut origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", hash.hex()), + hash.into(), + *blake3::hash(b"payload").as_bytes(), + 100, + Some("etag".to_owned()), + Some("version".to_owned()), + ); + origin.proven_at_unix_secs = 1; + (hash, entry, origin) +} + +#[test] +fn capsule_candidates_preserve_distinct_digests_and_first_occurrence() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + assert_eq!( + ( + rows.len(), + placement.chunk_index, + anchor.body_digest, + anchor.origin.payload_digest + ), + (3, 0, entry.body_digest().to_owned(), origin.payload_digest) + ); +} + +#[test] +fn capsule_anchor_rejects_changed_source_identity() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let mut anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + anchor.source_repo_prefix = "other".to_owned(); + assert!( + CapsuleChunkAnchor::decode( + &placement.source_anchor_id, + &serde_json::to_vec(&anchor).unwrap() + ) + .is_err() + ); +} + +#[test] +fn capsule_anchor_requires_the_authenticated_physical_descriptor() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + let other = XorbCatalogEntry::new(100, "4".repeat(64), entry.chunks().to_vec()); + assert!( + anchor + .validate_catalog_xorb(&origin.object_key, &other) + .is_err() + ); +} + +#[test] +fn capsule_anchor_rejects_oversized_records_before_decoding() { + assert!(CapsuleChunkAnchor::decode(&[1; 32], &vec![b' '; 16 * 1024 + 1]).is_err()); +} + +fn rooted_fixture() -> ( + CapsuleChunkAnchor, + CommittedChunkPlacement, + crate::capsule_protocol::PointerCatalog, +) { + let (hash, entry, origin) = fixture(); + let shard = "2".repeat(64); + let rows = capsule_chunk_candidate_rows("repo", &shard, hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + let mut catalog = crate::capsule_protocol::PointerCatalog::new(); + catalog.insert_xorb(hash.hex(), entry).unwrap(); + catalog + .insert_shard( + shard, + crate::capsule_protocol::ShardCatalogEntry::new(20, vec![hash.hex()]), + ) + .unwrap(); + (anchor, placement, catalog) +} + +#[test] +fn capsule_placement_requires_an_authenticated_source_shard() { + let (anchor, placement, rooted) = rooted_fixture(); + let mut unrooted = crate::capsule_protocol::PointerCatalog::new(); + for (hash, entry) in rooted.xorbs() { + unrooted.insert_xorb(hash.clone(), entry.clone()).unwrap(); + } + assert!( + anchor + .validate_placement(&placement, &unrooted, &anchor.origin.object_key) + .is_err() + ); +} + +#[test] +fn capsule_placement_accepts_exact_rooted_content() { + let (anchor, placement, catalog) = rooted_fixture(); + assert!( + anchor + .validate_placement(&placement, &catalog, &anchor.origin.object_key) + .is_ok() + ); +} + +#[test] +fn capsule_placement_rejects_tampered_occurrences_and_proofs() { + let (anchor, placement, catalog) = rooted_fixture(); + for field in ["size", "position", "chunk", "origin", "source"] { + let mut changed = placement.clone(); + match field { + "size" => changed.uncompressed_size += 1, + "position" => changed.chunk_index = 2, + "chunk" => changed.chunk_hash = [9; 32], + "origin" => changed.origin_proof_id = [9; 32], + "source" => changed.source_anchor_id = [9; 32], + _ => unreachable!(), + } + assert!( + anchor + .validate_placement(&changed, &catalog, &anchor.origin.object_key) + .is_err(), + "accepted changed {field}" + ); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index d456fb982..bbecfb74f 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -10,6 +10,7 @@ mod capsule; mod captured_visibility; #[cfg(feature = "storage")] mod catalog_capture; +mod chunk_candidates; mod history; mod layered; #[cfg(feature = "storage")] @@ -31,6 +32,8 @@ mod visibility; #[cfg(feature = "storage")] pub use browse::load_browse_indexes; pub use browse::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES}; +#[cfg(feature = "remote-index")] +pub use browse_retention::{BrowseRetentionCapture, verify_retained_browse_indexes}; #[cfg(feature = "storage")] pub use browse_retention::{ RetainedBrowseIndexes, load_retained_browse_indexes, prune_retained_browse_indexes, @@ -67,8 +70,8 @@ pub use plan::{ #[cfg(feature = "storage")] pub use pointer::PointerCatalogLookup; pub use pointer::{ - FileCatalogEntry, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, XorbCatalogEntry, - XorbChunkEntry, + FileCatalogEntry, MAX_CATALOG_BYTES, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, @@ -121,3 +124,10 @@ fn valid_ref_namespace<'a>(names: impl IntoIterator) -> bool { .all(|(index, _)| !names.contains(&name[..index])) }) } + +pub use chunk_candidates::{ + CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, capsule_chunk_head_key, +}; + +#[cfg(feature = "remote-index")] +pub(crate) use chunk_candidates::capsule_chunk_candidate_rows; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 2b359154a..85255d410 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -10,6 +10,7 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; pub(super) mod paged; +pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] pub use paged::PointerCatalogLookup; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index b7e3e9518..66e273430 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -10,7 +10,8 @@ pub use reader::PointerCatalogLookup; const MAGIC: &[u8; 8] = b"CRBPCP01"; const TRAILER_BYTES: usize = 12; const MAX_DESCRIPTOR_BYTES: usize = 4096; -pub(in crate::capsule_protocol) const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; +/// Maximum byte length of one encoded pointer catalog. +pub const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index f9e3bf451..3b38e2e0d 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -280,7 +280,7 @@ impl PointerCatalogLookup { )?; let hashes = requested(files, &mut work)?; let mut shards = requested(shards, &mut work)?; - let mut xorbs = requested(xorbs, &mut work)?; + let xorbs = requested(xorbs, &mut work)?; if hashes.is_empty() && shards.is_empty() && xorbs.is_empty() { return Ok(PointerCatalog::new()); } @@ -320,13 +320,43 @@ impl PointerCatalogLookup { catalog.insert_file(hash.clone(), entry)?; } } + let selected = self.selected_shards(store, &shards, &mut work).await?; + for (hash, entry) in selected.shards { + catalog.insert_shard(hash, entry)?; + } + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + catalog.validate(true, true)?; + Ok(catalog) + } + + /// Select requested shard roots and their complete authenticated xorb closures. + pub async fn shards(&self, store: &Store, hashes: &[String]) -> Result { + let mut work = self.work(); + let hashes = requested(hashes, &mut work)?; + self.selected_shards(store, &hashes, &mut work).await + } + + async fn selected_shards( + &self, + store: &Store, + shards: &BTreeSet, + work: &mut PageWork, + ) -> Result { + let mut catalog = PointerCatalog::new(); let root = self .checkpoint .as_ref() .and_then(|s| s.descriptor.shards.as_ref()); let rows = self - .rows(store, root, Kind::CatalogShards, &keys(&shards)?, &mut work) + .rows(store, root, Kind::CatalogShards, &keys(shards)?, work) .await?; + let mut xorbs = BTreeSet::new(); for (hash, row) in shards.iter().zip(rows) { for value in row { let row: ShardRow = from_json(&value.bytes)?; @@ -336,7 +366,7 @@ impl PointerCatalogLookup { &value.origin, row.xorbs.as_ref(), Kind::CatalogShardXorbs, - &mut work, + work, ) .await?; let closure = closure @@ -352,20 +382,12 @@ impl PointerCatalogLookup { verify_identity(&value, Kind::CatalogShards, &json(&entry)?)?; catalog.insert_shard(hash.clone(), entry)?; } + let Some(entry) = catalog.shards.get(hash) else { + continue; + }; + xorbs.extend(entry.xorb_hashes.iter().cloned()); } - for file in catalog.files.values() { - catalog - .shards - .get(&file.shard_hash) - .ok_or_else(|| corrupt("selected file references an absent shard"))?; - } - xorbs.extend( - catalog - .shards - .values() - .flat_map(|entry| entry.xorb_hashes.iter().cloned()), - ); - let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + let selected = self.selected_xorbs(store, &xorbs, work).await?; for (hash, entry) in selected.xorbs { catalog.insert_xorb(hash, entry)?; } diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index e20d5eef9..c07b638c0 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -144,6 +144,7 @@ pub struct CapsuleControl { #[derive(Debug, Clone)] enum ControlVisibility { + #[cfg(any(feature = "storage", test))] Complete(std::sync::Arc), #[cfg(feature = "storage")] Paged(std::sync::Arc), @@ -768,6 +769,9 @@ impl CapsuleControl { pub fn visibility_delta(&self) -> Result> { match &self.visibility { None => Ok(None), + #[cfg(not(any(feature = "storage", test)))] + Some(never) => match *never {}, + #[cfg(any(feature = "storage", test))] Some(ControlVisibility::Complete(delta)) => Ok(Some(delta)), #[cfg(feature = "storage")] Some(ControlVisibility::Paged(_)) => Err(corrupt( diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention.rs b/crates/crab-metadata/src/capsule_protocol/source_retention.rs index 65bf1df2e..1053e3a64 100644 --- a/crates/crab-metadata/src/capsule_protocol/source_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/source_retention.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crab_storage::{ETag, Store, StoreLayout}; +use futures_util::{StreamExt, TryStreamExt}; use object_store::path::Path; use serde::{Deserialize, Serialize}; @@ -48,9 +49,12 @@ struct Segment { } /// Authenticated lifetime dependencies; they never authorize ref or object visibility. +#[derive(Debug)] pub struct RetainedCapsuleSources { paths: Vec, runs: Vec, + head_versions: BTreeMap, + floor: Option<(u64, ETag)>, } impl RetainedCapsuleSources { @@ -65,6 +69,38 @@ impl RetainedCapsuleSources { pub fn runs(&self) -> &[CapsulePointer] { &self.runs } + + /// Check whether a complete capture still names the current lifetime authority. + /// + /// The loader must have received every source head. Invalid control records + /// return an error; valid concurrent movement returns false for caller retry. + pub async fn is_current(&self, layout: &StoreLayout) -> Result { + let heads = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await?; + if heads.len() != self.head_versions.len() + || heads + .iter() + .any(|meta| !self.head_versions.contains_key(&meta.location)) + { + return Ok(false); + } + // LIST can omit version IDs. Compare the GET CAS tokens that bound + // decoded heads so versioned buckets keep the same stability proof. + let mut versions = futures_util::stream::iter(self.head_versions.clone()) + .map(|(path, version)| async move { + let current = load_head(layout, &path).await?; + Ok::<_, MetadataError>(current.is_some_and(|(_, current)| current == version)) + }) + .buffer_unordered(16); + while let Some(current) = versions.try_next().await? { + if !current { + return Ok(false); + } + } + Ok(load_floor(layout).await?.as_ref() == self.floor.as_ref()) + } } impl Tip { @@ -347,20 +383,22 @@ pub async fn load_retained_capsule_sources( ) -> Result { let mut paths = BTreeSet::new(); let mut runs = BTreeMap::new(); + let mut head_versions = BTreeMap::new(); let mut roots = 0usize; let mut bytes = 0u64; let floor = load_floor(layout).await?; if floor.is_some() { paths.insert(layout.capsule_source_floor_path()); } - let minimum_generation = floor.map_or(0, |(floor, _)| floor); + let minimum_generation = floor.as_ref().map_or(0, |(floor, _)| *floor); for path in heads { if !paths.insert(path.clone()) { continue; } - let (head, _) = load_head(layout, path) + let (head, version) = load_head(layout, path) .await? .ok_or_else(|| corrupt("source lifetime head disappeared"))?; + head_versions.insert(path.clone(), version); if head.minimum_generation > minimum_generation { return Err(corrupt("source head exceeds the repository pruning floor")); } @@ -392,6 +430,8 @@ pub async fn load_retained_capsule_sources( Ok(RetainedCapsuleSources { paths: paths.into_iter().collect(), runs: runs.into_values().collect(), + head_versions, + floor, }) } diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs index 62d06c3d0..e12965453 100644 --- a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs @@ -75,6 +75,56 @@ async fn repeated_retirement_does_not_append_or_rewrite_lifetime() { assert_eq!(layout.store().get_with_etag(&path).await.unwrap(), before); } +#[tokio::test] +async fn source_snapshot_detects_appends_and_new_ref_heads() { + let layout = fixture(); + let first = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[first]) + .await + .unwrap(); + let captured = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + assert!(captured.is_current(&layout).await.unwrap()); + let second = source(&layout, 2).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, std::slice::from_ref(&second)) + .await + .unwrap(); + assert!(!captured.is_current(&layout).await.unwrap()); + let captured = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + retain_capsule_sources(&layout, "refs/heads/other", 0, &[second]) + .await + .unwrap(); + assert!(!captured.is_current(&layout).await.unwrap()); +} + +#[tokio::test] +async fn source_snapshot_tracks_floor_without_rewriting_ref_heads() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[pointer]) + .await + .unwrap(); + let paths = heads(&layout).await; + let before = layout.store().get_with_etag(&paths[0]).await.unwrap(); + let captured = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 0).await.unwrap(); + assert_eq!( + layout.store().get_with_etag(&paths[0]).await.unwrap(), + before + ); + assert!(!captured.is_current(&layout).await.unwrap()); + let captured = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 0).await.unwrap(); + assert!(captured.is_current(&layout).await.unwrap()); +} + #[tokio::test] async fn independent_ref_heads_share_source_identity_without_sharing_cas() { let layout = fixture(); diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs index 3ae87ee33..105e8cad5 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs @@ -224,6 +224,7 @@ impl Descriptor { } } +#[cfg(feature = "storage")] impl VisibilityEditHeader { /// Return the prior tip used by the edit. pub fn old_oid(&self) -> Option<&str> { @@ -260,7 +261,9 @@ impl VisibilityEditHeader { _ => false, } } +} +impl VisibilityEditHeader { fn decode(key: &[u8], bytes: &[u8], body_bytes: u64) -> Result { let row: Self = serde_json::from_slice(bytes) .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index f26164933..2326d40e4 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -53,6 +53,36 @@ impl PagedGitObjectIndex { ]) } + /// Authenticate a retained body's bytes and graph binding without granting visibility. + /// + /// The caller supplies an authenticated graph descriptor from the same capture. + pub async fn verify_retained( + store: &Store, + layout: &StoreLayout, + hash: &str, + state_digest: &str, + graph: &crate::split_commit_graph::CommitGraphDescriptor, + maximum: u64, + ) -> Result<()> { + let (descriptor, _) = Self::descriptor(store, layout, hash, maximum).await?; + if descriptor.state_digest != state_digest + || descriptor.generation != graph.generation + || descriptor.pack_index_hash != graph.pack_index_hash + || descriptor.git_validation_digest != graph.git_validation_digest + || descriptor.body_bytes > maximum + { + return bad("retained object index does not match graph or byte limit"); + } + store + .verify_size_and_hash( + &body_path(layout, &descriptor.body_hash), + descriptor.body_bytes, + &hash_bytes(&descriptor.body_hash)?, + ) + .await?; + Ok(()) + } + async fn descriptor( store: &Store, layout: &StoreLayout, diff --git a/crates/crab-metadata/src/key_codec.rs b/crates/crab-metadata/src/key_codec.rs index 19301a0a6..e95d2d79f 100644 --- a/crates/crab-metadata/src/key_codec.rs +++ b/crates/crab-metadata/src/key_codec.rs @@ -11,6 +11,9 @@ //! - `0x03 || hash[32]` — latest committed chunk-receipt candidate. //! - `0x04 || proof_id[32]` — immutable xorb origin proof. //! - `0x05 || anchor_id[32]` — immutable source-shard anchor. +//! - `0x06 || hash[32]` — point-readable capsule chunk candidate. +//! - `0x07 || anchor_id[32]` — immutable capsule source/origin anchor. +//! - `0x08 || placement_id[32]` — immutable capsule chunk placement. //! - `0xFF || b"sys:" || name` — system key (format_version, epoch, //! created_at, gc_generation). //! @@ -22,6 +25,9 @@ use crab_xet::xorb::format::MerkleHash; use crate::error::{MetadataError, Result}; +/// Canonical metadata database format shared by all reader and writer adapters. +pub const METADB_FORMAT_VERSION: u32 = 1; + /// Prefix byte for content-addressed keys (file_hash or chunk_hash). pub const PREFIX_CONTENT: u8 = 0x01; @@ -41,6 +47,30 @@ pub const PREFIX_ORIGIN_PROOF: u8 = 0x04; /// Prefix byte for immutable repository-shard source anchors. pub const PREFIX_SOURCE_ANCHOR: u8 = 0x05; +/// Capsule candidates use distinct keys so released manifest readers cannot +/// mistake capsule provenance for a manifest-generation assertion. +const PREFIX_CAPSULE_CHUNK_HEAD: u8 = 0x06; +const PREFIX_CAPSULE_CHUNK_ANCHOR: u8 = 0x07; +const PREFIX_CAPSULE_CHUNK_PLACEMENT: u8 = 0x08; + +/// Encode a point-readable capsule candidate key. +#[must_use] +pub fn capsule_chunk_head_key(hash: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_HEAD, hash) +} + +/// Encode an immutable capsule provenance key. +#[must_use] +pub fn capsule_chunk_anchor_key(hash: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_ANCHOR, hash) +} + +/// Encode an immutable capsule placement independently of its replaceable head. +#[must_use] +pub fn capsule_chunk_placement_key(id: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_PLACEMENT, id) +} + /// Prefix byte for the `sys:*` system-key namespace. Chosen as `0xFF` /// so it sorts after every content key and is trivial to filter out of /// content-only scans. diff --git a/crates/crab-metadata/src/remote_index.rs b/crates/crab-metadata/src/remote_index.rs index e33763806..4c94dc6a8 100644 --- a/crates/crab-metadata/src/remote_index.rs +++ b/crates/crab-metadata/src/remote_index.rs @@ -100,6 +100,73 @@ pub async fn write_index_entries( close_result } +/// Publish rebuildable capsule chunk hints using origin receipts captured during byte verification. +/// +/// Call after ref visibility. Every opened database is closed on success or error; +/// callers must await this future through completion. Readers still revalidate source +/// visibility and exact origin identity, so partial batches never confer authority. +pub async fn write_capsule_chunk_candidates( + store: Arc, + config: &RemoteIndexConfig, + repo_prefix: &str, + catalog: &crate::capsule_protocol::PointerCatalog, + origins: &HashMap, +) -> Result<()> { + let mut source_shards = HashMap::new(); + for (shard_hash, shard) in catalog.shards() { + for xorb in shard.xorb_hashes() { + source_shards + .entry(xorb.as_str()) + .or_insert(shard_hash.as_str()); + } + } + let mut selected = Vec::new(); + for (hash, entry) in catalog.xorbs().iter().rev() { + let parsed = MerkleHash::from_hex(hash).map_err(|_| MetadataError::CorruptObject { + path: CHUNK_INDEX_DB_LABEL.to_owned(), + reason: "invalid capsule xorb hash".to_owned(), + })?; + if let Some(origin) = origins.get(&parsed) { + selected.push((hash, parsed, entry, origin)); + } + } + if selected.is_empty() { + return Ok(()); + } + for (hash, _, _, _) in &selected { + if !source_shards.contains_key(hash.as_str()) { + return Err(MetadataError::CorruptObject { + path: CHUNK_INDEX_DB_LABEL.to_owned(), + reason: "capsule xorb candidate has no source shard root".to_owned(), + }); + } + } + let writer = RemoteIndexWriter::open(store, config, false, true).await?; + let result = async { + // Last-write heads choose the lowest canonical xorb and first occurrence. + for (hash, parsed, entry, origin) in selected { + let source_shard = source_shards.get(hash.as_str()).ok_or_else(|| { + MetadataError::Internal( + "admitted capsule candidate lost its source shard".to_owned(), + ) + })?; + let rows = crate::capsule_protocol::capsule_chunk_candidate_rows( + repo_prefix, + source_shard, + parsed, + entry, + origin, + )?; + writer.write_capsule_candidates(&rows).await?; + } + Ok::<_, MetadataError>(()) + } + .await; + let close = writer.close().await; + result?; + close +} + /// Long-lived writer for caller-bounded remote index batches. /// /// Writes buffer data without waiting for durability. Always await [`Self::close`] @@ -164,6 +231,33 @@ impl RemoteIndexWriter { .await } + /// Buffer a bounded set of capsule candidate rows; close remains the durability barrier. + async fn write_capsule_candidates(&self, rows: &[(Vec, Vec)]) -> Result<()> { + let db = self + .chunk_db + .as_ref() + .ok_or_else(|| MetadataError::Internal("chunk index was not opened".to_owned()))?; + for rows in rows.chunks(4096) { + let mut batch = slatedb::WriteBatch::new(); + for (key, value) in rows { + batch.put(key.as_slice(), value.as_slice()); + } + db.write_with_options( + batch, + &slatedb::config::WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .map_err(|source| MetadataError::SlateDbWrite { + db: CHUNK_INDEX_DB_LABEL.to_owned(), + source, + })?; + } + Ok(()) + } + /// Flush and close every database opened by this writer. /// /// Both closes are attempted. If both fail, return the file-index error. @@ -474,6 +568,34 @@ async fn open_writer( path: &str, db: &'static str, ) -> Result { + let fresh = match open_reader(Arc::clone(&store), path, db).await? { + None => true, + Some(reader) => { + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let validation = async { + let value = + reader + .get(&key) + .await + .map_err(|source| MetadataError::SlateDbRead { + db: db.to_owned(), + source, + })?; + validate_writer_format(value.as_deref(), db) + } + .await; + let close = reader + .close() + .await + .map_err(|source| MetadataError::SlateDbClose { + db: db.to_owned(), + source, + }); + validation?; + close?; + false + } + }; let l0_sst_size_bytes = usize::try_from(DEFAULT_CHUNK_INDEX_L0_SST_SIZE_BYTES) .map_err(|error| MetadataError::Internal(format!("L0 SST size unsupported: {error}")))?; let settings = slatedb::config::Settings { @@ -484,7 +606,7 @@ async fn open_writer( l0_sst_size_bytes, ..slatedb::config::Settings::default() }; - slatedb::Db::builder(ObjectPath::from(path), store) + let writer = slatedb::Db::builder(ObjectPath::from(path), store) .with_settings(settings) .build() .await @@ -492,7 +614,61 @@ async fn open_writer( db: db.to_owned(), path: path.to_owned(), source, - }) + })?; + if fresh { + let initialized = async { + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = writer + .get(&key) + .await + .map_err(|source| MetadataError::SlateDbRead { + db: db.to_owned(), + source, + })?; + if let Some(value) = value { + return validate_writer_format(Some(&value), db); + } + let mut batch = slatedb::WriteBatch::new(); + batch.put(key, crate::key_codec::METADB_FORMAT_VERSION.to_le_bytes()); + writer + .write_with_options( + batch, + &slatedb::config::WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .map(|_| ()) + .map_err(|source| MetadataError::SlateDbWrite { + db: db.to_owned(), + source, + }) + } + .await; + if let Err(error) = initialized { + let _ = close_writer(Some(writer), db).await; + return Err(error); + } + } + Ok(writer) +} + +fn validate_writer_format(value: Option<&[u8]>, db: &str) -> Result<()> { + if value + != Some( + crate::key_codec::METADB_FORMAT_VERSION + .to_le_bytes() + .as_slice(), + ) + { + return Err(MetadataError::CorruptObject { + path: db.to_owned(), + reason: "metadata database is missing or has an unsupported canonical format marker" + .to_owned(), + }); + } + Ok(()) } async fn open_reader( @@ -790,3 +966,6 @@ mod tests { assert_eq!(config.chunk_index_path, ".crab/chunk_index_db/"); } } + +#[cfg(test)] +mod capsule_tests; diff --git a/crates/crab-metadata/src/remote_index/capsule_tests.rs b/crates/crab-metadata/src/remote_index/capsule_tests.rs new file mode 100644 index 000000000..269567268 --- /dev/null +++ b/crates/crab-metadata/src/remote_index/capsule_tests.rs @@ -0,0 +1,165 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use crate::capsule_protocol::{PointerCatalog, XorbCatalogEntry, XorbChunkEntry}; +use crate::receipts::OriginReceipt; +use crab_xet::hash::MerkleHash; +use object_store::memory::InMemory; + +use super::{RemoteIndexConfig, write_capsule_chunk_candidates}; + +#[tokio::test] +async fn capsule_candidates_reject_xorbs_without_a_source_shard_root() { + let hash = MerkleHash::from([3; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + hash.hex(), + XorbCatalogEntry::new( + 100, + "4".repeat(64), + vec![XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9)], + ), + ) + .unwrap(); + let origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", hash.hex()), + hash.into(), + [5; 32], + 100, + Some("etag".to_owned()), + None, + ); + let result = write_capsule_chunk_candidates( + Arc::new(InMemory::new()), + &RemoteIndexConfig::for_repo("repo"), + "repo", + &catalog, + &HashMap::from([(hash, origin)]), + ) + .await; + assert!(result.is_err(), "an xorb descriptor alone is not a GC root"); +} + +#[tokio::test] +async fn capsule_candidates_writer_initializes_canonical_format() { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let writer = super::RemoteIndexWriter::open(Arc::clone(&store), &config, false, true) + .await + .unwrap(); + writer.close().await.unwrap(); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = reader.get(&key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!(value.as_deref(), Some(1u32.to_le_bytes().as_slice())); +} + +#[tokio::test] +async fn capsule_candidates_writer_rejects_noncanonical_existing_databases() { + for marker in [None, Some(2u32.to_le_bytes().to_vec()), Some(vec![1])] { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let seed = slatedb::Db::open(config.chunk_index_path.clone(), Arc::clone(&store)) + .await + .unwrap(); + if let Some(marker) = marker { + seed.put( + crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION), + marker, + ) + .await + .unwrap(); + } + seed.close().await.unwrap(); + let result = super::RemoteIndexWriter::open(store, &config, false, true).await; + if let Ok(writer) = result { + writer.close().await.unwrap(); + panic!("invalid existing format was accepted"); + } + } +} + +#[tokio::test] +async fn capsule_exact_placement_survives_head_replacement() { + use crate::capsule_protocol::ShardCatalogEntry; + use crate::receipts::CommittedChunkPlacement; + + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("source"); + let xorb = MerkleHash::from([3; 32]); + let chunk = MerkleHash::from([1; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 100, + "4".repeat(64), + vec![XorbChunkEntry::new(chunk.hex(), 9)], + ), + ) + .unwrap(); + catalog + .insert_shard("2".repeat(64), ShardCatalogEntry::new(20, vec![xorb.hex()])) + .unwrap(); + let mut origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", xorb.hex()), + xorb.into(), + [5; 32], + 100, + Some("first".to_owned()), + None, + ); + let mut placements = Vec::new(); + for repo in ["source", "newer-source"] { + write_capsule_chunk_candidates( + Arc::clone(&store), + &config, + repo, + &catalog, + &HashMap::from([(xorb, origin.clone())]), + ) + .await + .unwrap(); + let reader = super::open_reader( + Arc::clone(&store), + &config.chunk_index_path, + "chunk_index_db", + ) + .await + .unwrap() + .unwrap(); + let head = reader + .get(crate::key_codec::capsule_chunk_head_key(&chunk.into())) + .await + .unwrap() + .unwrap(); + reader.close().await.unwrap(); + placements.push(CommittedChunkPlacement::decode(&head).unwrap()); + origin.etag = Some("second".to_owned()); + } + let first = &placements[0]; + let mut key = vec![0x08]; + key.extend_from_slice(&first.placement_id()); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let exact = reader.get(key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!( + ( + placements[0] != placements[1], + exact.map(|value| CommittedChunkPlacement::decode(&value).unwrap()) + ), + (true, Some(first.clone())), + "a changed shared head must not erase the proof selected by an earlier add", + ); +} diff --git a/crates/crab-read/src/upload_pack.rs b/crates/crab-read/src/upload_pack.rs index 5fb59eeb3..3bed3dc62 100644 --- a/crates/crab-read/src/upload_pack.rs +++ b/crates/crab-read/src/upload_pack.rs @@ -1128,6 +1128,7 @@ async fn plan_with_operation( } let mut selected = HashSet::new(); + let mut selected_commit = false; let mut object_ids = Vec::new(); let mut shallow = HashSet::new(); let mut unshallow = HashSet::new(); @@ -1141,7 +1142,11 @@ async fn plan_with_operation( let Some(item) = queue.pop_front() else { break; }; - if common_haves.contains(&item.oid) && request.deepen.is_none() { + if common_haves.contains(&item.oid) + && request.deepen.is_none() + && request.deepen_since.is_none() + && request.deepen_not.is_empty() + { continue; } // Tree entries prove that blobs are leaves. Rejecting a known omitted blob here @@ -1241,9 +1246,29 @@ async fn plan_with_operation( invariant: "batched upload-pack read is missing an object", })? }; - if excluded_commits.contains(&item.oid) && !roots.contains(&item.oid) { + // Wants authorize access but do not override shallow selectors. Applying + // these before selection also prevents an excluded tip's tree leaking in. + if excluded_commits.contains(&item.oid) { continue; } + if object.kind == gix_object::Kind::Commit + && let Some(since) = request.deepen_since + { + let commit = gix_object::CommitRef::from_bytes(&object.data, gix_hash::Kind::Sha1) + .map_err(|_| RemoteGitError::Corrupt { + stage: CorruptionStage::Commit, + })?; + let timestamp = commit + .time() + .map_err(|_| RemoteGitError::Corrupt { + stage: CorruptionStage::Commit, + })? + .seconds; + if timestamp < since { + continue; + } + } + selected_commit |= object.kind == gix_object::Kind::Commit; let encountered_client_have = matches!(visibility, VisibilitySource::TipBound { .. }) && client_haves.contains(&item.oid) && object.kind == gix_object::Kind::Commit; @@ -1278,6 +1303,10 @@ async fn plan_with_operation( } } + if (request.deepen_since.is_some() || !request.deepen_not.is_empty()) && !selected_commit { + return Err(RemoteGitError::EmptyShallowSelection); + } + if request.include_tags { for reference in &repository.refs().entries { if !reference.name.starts_with("refs/tags/") @@ -2396,18 +2425,6 @@ async fn enqueue_children( maximum_objects, deduplicate_by_oid, )?; - if let Some(since) = request.deepen_since { - let timestamp = commit - .time() - .map_err(|_| RemoteGitError::Corrupt { - stage: CorruptionStage::Commit, - })? - .seconds; - if timestamp < since { - shallow.insert(item.oid); - return Ok(()); - } - } match item.depth { TraversalDepth::RelativeBoundary => { if existing_shallow.contains(&item.oid) { diff --git a/crates/crab-remote-git/src/error.rs b/crates/crab-remote-git/src/error.rs index 15bcdeb54..0e40a9f61 100644 --- a/crates/crab-remote-git/src/error.rs +++ b/crates/crab-remote-git/src/error.rs @@ -237,6 +237,10 @@ pub enum Error { #[error("repository is empty")] EmptyRepository, + /// Date and ref exclusions selected no commits for a shallow fetch. + #[error("no commits selected for shallow requests")] + EmptyShallowSelection, + /// Locator publication has not reached the required manifest generation. #[error( "repository object locator is not ready: observed generation {observed:?}, required {required}" @@ -516,6 +520,7 @@ impl Error { | Self::InvalidPath { .. } | Self::InvalidCursor { .. } | Self::InvalidLimit { .. } + | Self::EmptyShallowSelection | Self::Revision { reason: RevisionError::InvalidReference diff --git a/crates/crab-remote/Cargo.toml b/crates/crab-remote/Cargo.toml index 67659fc1a..bf30ec429 100644 --- a/crates/crab-remote/Cargo.toml +++ b/crates/crab-remote/Cargo.toml @@ -41,6 +41,7 @@ futures-util = { workspace = true, optional = true } object_store = { workspace = true, optional = true } [dev-dependencies] +slatedb = "=0.15.0" sha1 = "0.10" tracing-subscriber = "0.3" async-trait = { workspace = true } diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 1ba312482..5c391f6d2 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -58,6 +58,19 @@ The pinned-view outcome also reports the last inventory published by that pass, or the input inventory if neither phase published. It deliberately excludes later ref-head updates instead of rescanning them for CLI statistics. +Capsule preparation resolves outgoing pointer files and only the prepared +shard/xorb identities against a captured paged catalog. Ordinary Git and LFS-only +uploads do not read historical Xet catalog pages. Selected dependencies retain +complete closure, size and immutable-descriptor conflict checks. + +Scratch ownership does not freeze artifact bytes. Bounded reads revalidate the +inspected raw Blake3 body identity before upload; Xet protocol identities retain +their distinct word-ordered representation. Capsule commitment retains exact +verified create/read identities with their source catalog, publishes rebuildable +chunk hints only after proven ref visibility, and preserves a committed outcome +if hint-index maintenance fails. Uncertain publication does not publish hints. +Caller-owned ref leases and GC fences cover this complete operation. + `objects` owns bounded tree reads, deterministic tree edits and encoding, commit encoding, and Git object identities. `prepare` owns normalized packs and hydrated content artifacts used by remote commits and recovery. File bodies and diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 3c526c14e..7210a1e20 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, fs::File, io::{BufRead, BufReader}, path::PathBuf, @@ -20,7 +20,8 @@ use crab_git::{ }; use crab_metadata::{ capsule_protocol::{ - FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + FileCatalogEntry, PointerCatalog, PointerCatalogLookup, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }, git_object_locator::GitObjectKind, git_visibility::{GitCatalogVisibilityIndex, GitVisibilityEdit, GitVisibilityIndex}, @@ -28,8 +29,11 @@ use crab_metadata::{ use crab_remote_git::{OperationContext, OperationKind, RemoteGitRepository}; use gix_hash::ObjectId; use gix_object::Kind; +use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; +mod capsule; + /// Failure while validating and preparing incoming Git objects. #[derive(Debug, thiserror::Error)] pub enum Error { @@ -125,6 +129,41 @@ impl ContentArtifact { pub const fn size(&self) -> u64 { self.size } + + // Scratch ownership does not freeze file bytes. Revalidate the inspected + // identity with a bounded read before publishing content or its receipt. + async fn read_verified(&self, maximum: u64, cancel: &CancellationToken) -> Result { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + if self.size > maximum { + return Err(Error::Content( + "prepared artifact exceeds its byte limit".to_owned(), + )); + } + let mut bytes = Vec::new(); + tokio::fs::File::open(&self.path) + .await? + .take(self.size + 1) + .read_to_end(&mut bytes) + .await?; + let bytes = Bytes::from(bytes); + let size = self.size; + let hash = self.body_hash; + let bytes = tokio::task::spawn_blocking(move || { + if bytes.len() as u64 != size || blake3::hash(&bytes).as_bytes() != &hash { + return Err(Error::Content( + "prepared artifact changed after inspection".to_owned(), + )); + } + Ok(bytes) + }) + .await??; + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + Ok(bytes) + } } /// One Crab pointer supplied by a prepared reconstruction shard. @@ -236,15 +275,14 @@ impl PreparedContent { { return Err(Error::Content("shard identity is invalid".to_owned())); } - let bytes = tokio::fs::read(&artifact.path).await?; - if bytes.len() as u64 != artifact.size - || blake3::hash(&bytes).as_bytes() != &artifact.body_hash - || <[u8; 32]>::from(crab_xet::hash::compute_data_hash(&bytes)) - != artifact.protocol_hash + let bytes = artifact + .read_verified(crab_xet::shard::MAX_PUSH_SHARD_SIZE_BYTES, cancel) + .await?; + if <[u8; 32]>::from(crab_xet::hash::compute_data_hash(&bytes)) != artifact.protocol_hash { return Err(Error::Content("shard binding changed".to_owned())); } - shard_bodies.insert(artifact.protocol_hash, Bytes::from(bytes)); + shard_bodies.insert(artifact.protocol_hash, bytes); } let mut seen_files = std::collections::BTreeSet::new(); let mut used_shards = std::collections::BTreeSet::new(); @@ -330,7 +368,9 @@ impl PreparedContent { crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), XorbCatalogEntry::new( artifact.size, - crab_xet::hash::MerkleHash::from(artifact.body_hash).hex(), + blake3::Hash::from_bytes(artifact.body_hash) + .to_hex() + .to_string(), inspected .chunks .iter() @@ -428,6 +468,8 @@ pub struct CapsuleArtifacts { edits: Vec, packs: Vec, sections: Vec, + pointer_delta: PointerCatalog, + origins: HashMap, changes_namespace: bool, } @@ -809,427 +851,6 @@ impl Prepared { shards, }) } - - /// Upload artifacts for one captured capsule-protocol repository view. - /// - /// The caller must hold ref leases and GC fences from before opening - /// `view` through commitment. This validates all historical dependencies - /// against that view, uploads same-publication Xet content before Git - /// visibility, and retains the exact root snapshot for per-ref CAS. - pub async fn upload_capsule( - &self, - view: &crab_read::capsule_protocol::CapsuleRepositoryView, - limits: crab_read::dependency_proof::DependencyProofLimits, - cancel: &CancellationToken, - ) -> Result { - let started = Instant::now(); - let repository_refs = self - .repository - .refs() - .entries - .iter() - .map(|reference| (reference.name.clone(), reference.target.to_string())) - .collect::>(); - if repository_refs != *view.refs() - || self.repository.generation() != view.root().root().generation() - { - return Err(Error::Request( - "Capsule view differs from the prepared repository version", - )); - } - let lookup = view.pointer_catalog_lookup(&self.layout, 2 * 1024 * 1024 * 1024)?; - let local = self.content.as_ref().map(PreparedContent::pointer_catalog); - let mut files = self - .plan - .pointers() - .iter() - .filter_map(|dependency| match &dependency.pointer { - crab_git::pointer_detect::PointerKind::Crab(pointer) => { - Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) - } - _ => None, - }) - .collect::>(); - files.extend( - local - .into_iter() - .flat_map(|catalog| catalog.files().keys().cloned()), - ); - let files = files.into_iter().collect::>(); - let shards = local - .into_iter() - .flat_map(|catalog| catalog.shards().keys().cloned()) - .collect::>(); - let xorbs = local - .into_iter() - .flat_map(|catalog| catalog.xorbs().keys().cloned()) - .collect::>(); - // Local artifacts can collide with historical identities even when - // none of their files are present. Select those keys and the outgoing - // file closure together; unrelated history must stay cold. - let base_catalog = lookup - .dependencies(self.layout.store(), &files, &shards, &xorbs) - .await?; - let prepared_files = self - .content - .as_ref() - .map(|content| { - content - .files - .iter() - .map(|file| file.file_hash) - .collect::>() - }) - .unwrap_or_default(); - crab_read::dependency_proof::verify_capsule_dependencies_except_crab( - &self.layout, - &base_catalog, - self.plan.pointers(), - &prepared_files, - limits, - cancel, - ) - .await - .map_err(|error| Error::Dependency(Box::new(error)))?; - - let pointer_delta = match &self.content { - Some(content) => { - upload_capsule_content(&self.layout, &base_catalog, content, cancel).await? - } - None => PointerCatalog::new(), - }; - let mut packs = Vec::new(); - if let Some(pack) = &self.pack { - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let (pack_bytes, index, reverse_index, locator) = tokio::try_join!( - tokio::fs::read(pack.pack_path()), - tokio::fs::read(pack.index_path()), - tokio::fs::read(pack.reverse_path()), - tokio::fs::read(pack.kinds_path()), - )?; - let external_delta_bases = pack - .external_delta_bases() - .iter() - .map(ToString::to_string) - .collect(); - packs.push( - crab_metadata::capsule_protocol::CapsuleGitPack::new_with_external_delta_bases( - Bytes::from(pack_bytes), - Bytes::from(index), - Bytes::from(reverse_index), - Bytes::from(locator), - pack.git_sha1().to_string(), - u64::from(pack.object_count()), - external_delta_bases, - )?, - ); - } - let mut sections = Vec::with_capacity(2); - if !pointer_delta.is_empty() { - sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( - crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta, - pointer_delta.encode_delta()?, - )); - } - if !self.visibility.is_empty() { - let visibility = crab_metadata::capsule_protocol::CapsuleVisibilityDelta::new( - self.visibility.clone(), - )?; - sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( - crab_metadata::capsule_protocol::CapsuleSectionKind::VisibilityDelta, - visibility.encode()?, - )); - } - let edits = self - .updates - .iter() - .map(|update| { - crab_metadata::capsule_protocol::CapsuleRefEdit::new( - update.name.clone(), - update.old.map(|oid| oid.to_string()), - update.new.map(|oid| oid.to_string()), - self.plan - .peeled() - .get(&update.name) - .map(ToString::to_string), - ) - }) - .collect::>(); - let changes_namespace = edits - .iter() - .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); - tracing::info!( - phase = "capsule_input", - packs = packs.len(), - elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, - "Git receive phase completed" - ); - Ok(CapsuleArtifacts { - layout: self.layout.clone(), - base: view.root_snapshot().clone(), - edits, - packs, - sections, - changes_namespace, - }) - } -} - -impl CapsuleArtifacts { - /// Commit through independently mutable ref heads and one capsule authority. - /// - /// Revalidate authorization and repository policy immediately before this - /// call. The caller must retain its original ref leases and GC fences until - /// the returned outcome is known and all lease cleanup has completed. - pub async fn commit( - self, - plan_id: Option<&str>, - namespace_ttl: Duration, - cancel: &CancellationToken, - ) -> Result { - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let transaction = match plan_id { - Some(plan_id) => crab_metadata::capsule_protocol::CapsuleTransaction::for_plan( - self.base.record().digest(), - plan_id, - self.edits, - )?, - None => crab_metadata::capsule_protocol::CapsuleTransaction::new( - self.base.record().digest(), - self.edits, - )?, - }; - let transaction_id = transaction.id()?; - let started = Instant::now(); - let capsule = crab_metadata::capsule_protocol::Capsule::build( - &transaction, - self.packs, - self.sections, - )?; - tracing::info!( - phase = "capsule_encode", - %transaction_id, - bytes = capsule.bytes().len(), - elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, - "Git receive phase completed" - ); - let publish = if self.changes_namespace { - let ref_names = transaction - .edits() - .iter() - .map(|edit| edit.ref_name().to_owned()) - .collect::>(); - let layout = self.layout.clone(); - let commit_layout = layout.clone(); - let base = self.base; - crab_write::with_ref_namespaces( - layout.store(), - &layout, - &ref_names, - namespace_ttl, - cancel, - move |scoped| async move { - if scoped.is_cancelled() { - return Err(crab_write::WriteError::Cancelled); - } - crab_write::capsule_protocol::validate_ref_namespace( - &commit_layout, - base.record().root(), - transaction.edits(), - ) - .await?; - crab_write::capsule_protocol::publish( - &commit_layout, - base, - &transaction, - &capsule, - ) - .await - }, - ) - .await - } else { - crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) - .await - }; - match publish { - Ok(_) => Ok(CapsuleCommitOutcome::Committed { transaction_id }), - Err(error @ crab_write::WriteError::CapsuleCommitUncertain { .. }) => { - Ok(CapsuleCommitOutcome::Indeterminate { - transaction_id, - source: Box::new(error), - }) - } - Err(error) => Err(error.into()), - } - } -} - -async fn upload_capsule_content( - layout: &crab_storage::StoreLayout, - base: &PointerCatalog, - content: &PreparedContent, - cancel: &CancellationToken, -) -> Result { - let local = content.pointer_catalog(); - let mut delta = PointerCatalog::new(); - let mut needed_shards = std::collections::BTreeSet::new(); - for (file_hash, file) in local.files() { - if let Some(existing) = base.files().get(file_hash) { - if existing.size() != file.size() { - return Err(Error::Content(format!( - "file {file_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - needed_shards.insert(file.shard_hash().to_owned()); - delta.insert_file(file_hash.clone(), file.clone())?; - } - let mut needed_xorbs = std::collections::BTreeSet::new(); - for shard_hash in &needed_shards { - let shard = local - .shards() - .get(shard_hash) - .ok_or_else(|| Error::Content("prepared shard catalog entry is missing".to_owned()))?; - if let Some(existing) = base.shards().get(shard_hash) { - if existing != shard { - return Err(Error::Content(format!( - "shard {shard_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - needed_xorbs.extend(shard.xorb_hashes().iter().cloned()); - delta.insert_shard(shard_hash.clone(), shard.clone())?; - } - let xorb_artifacts = content - .xorbs - .iter() - .map(|artifact| { - ( - crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), - artifact, - ) - }) - .collect::>(); - for xorb_hash in needed_xorbs { - let local_entry = local - .xorbs() - .get(&xorb_hash) - .ok_or_else(|| Error::Content("prepared xorb catalog entry is missing".to_owned()))?; - if let Some(existing) = base.xorbs().get(&xorb_hash) { - if existing.chunks() != local_entry.chunks() { - return Err(Error::Content(format!( - "xorb {xorb_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let artifact = xorb_artifacts - .get(&xorb_hash) - .ok_or_else(|| Error::Content("prepared xorb artifact is missing".to_owned()))?; - let bytes = Bytes::from(tokio::fs::read(&artifact.path).await?); - let hash = crab_xet::hash::MerkleHash::from(artifact.protocol_hash); - let entry = match layout - .store() - .create_or_read_immutable( - &layout.xorb_path(&hash), - bytes, - crab_xet::xorb::format::MAX_XORB_SIZE as u64, - ) - .await? - { - crab_storage::ImmutableCreateOutcome::Created => local_entry.clone(), - crab_storage::ImmutableCreateOutcome::Existing(existing) => { - catalog_xorb_from_existing(&xorb_hash, local_entry, existing)? - } - }; - delta.insert_xorb(xorb_hash, entry)?; - } - let shard_artifacts = content - .shards - .iter() - .map(|artifact| { - ( - crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), - artifact, - ) - }) - .collect::>(); - for shard_hash in needed_shards { - if base.shards().contains_key(&shard_hash) { - continue; - } - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let artifact = shard_artifacts - .get(&shard_hash) - .ok_or_else(|| Error::Content("prepared shard artifact is missing".to_owned()))?; - let bytes = Bytes::from(tokio::fs::read(&artifact.path).await?); - layout - .store() - .put_if_absent_verified( - &layout.shard_path(&crab_xet::hash::MerkleHash::from(artifact.protocol_hash)), - bytes, - ) - .await?; - } - if !delta.shards().is_empty() { - crab_metadata::ref_registry::union_register_repo_shards( - layout.store(), - layout, - delta.shards().keys().cloned().collect(), - ) - .await?; - } - let mut complete = base.clone(); - complete.apply(&delta)?; - Ok(delta) -} - -fn catalog_xorb_from_existing( - xorb_hash: &str, - local: &XorbCatalogEntry, - bytes: Bytes, -) -> Result { - let parser = - crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).map_err(Error::ContentFormat)?; - parser - .verify_payload_digest() - .map_err(Error::ContentFormat)?; - parser.verify_all_chunks().map_err(Error::ContentFormat)?; - if parser.hash().hex() != xorb_hash { - return Err(Error::Content( - "existing xorb has the wrong logical identity".to_owned(), - )); - } - let chunks = (0..parser.num_chunks()) - .map(|index| { - parser - .chunk_meta(index) - .map(|chunk| XorbChunkEntry::new(chunk.hash.hex(), chunk.uncompressed_len)) - }) - .collect::, _>>() - .map_err(Error::ContentFormat)?; - if chunks != local.chunks() { - return Err(Error::Content( - "existing xorb has a conflicting chunk layout".to_owned(), - )); - } - Ok(XorbCatalogEntry::new( - bytes.len() as u64, - blake3::hash(&bytes).to_hex().to_string(), - chunks, - )) } impl Artifacts<'_> { diff --git a/crates/crab-remote/src/prepare/capsule.rs b/crates/crab-remote/src/prepare/capsule.rs new file mode 100644 index 000000000..dcf26b988 --- /dev/null +++ b/crates/crab-remote/src/prepare/capsule.rs @@ -0,0 +1,495 @@ +use super::*; + +impl Prepared { + /// Upload artifacts for one captured capsule-protocol repository view. + /// + /// The caller must hold ref leases and GC fences from before opening + /// `view` through commitment. This validates all historical dependencies + /// against that view, uploads same-publication Xet content before Git + /// visibility, and retains the exact root snapshot for per-ref CAS. + pub async fn upload_capsule( + &self, + view: &crab_read::capsule_protocol::CapsuleRepositoryView, + limits: crab_read::dependency_proof::DependencyProofLimits, + cancel: &CancellationToken, + ) -> Result { + let started = Instant::now(); + let repository_refs = self + .repository + .refs() + .entries + .iter() + .map(|reference| (reference.name.clone(), reference.target.to_string())) + .collect::>(); + if repository_refs != *view.refs() + || self.repository.generation() != view.root().root().generation() + { + return Err(Error::Request( + "Capsule view differs from the prepared repository version", + )); + } + if self.plan.pointers().len() > limits.max_dependencies { + return Err(Error::Dependency(Box::new( + crab_read::dependency_proof::DependencyProofError::Limit("pointer count"), + ))); + } + let lookup = view.pointer_catalog_lookup( + &self.layout, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + )?; + let hashes = self + .plan + .pointers() + .iter() + .filter_map(|dependency| match &dependency.pointer { + crab_git::pointer_detect::PointerKind::Crab(pointer) => { + Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) + } + _ => None, + }) + .collect::>(); + // Ordinary Git and LFS-only changes need no Xet catalog pages. Selected + // Crab pointers retain complete authenticated shard/xorb closure checks. + let base_catalog = if hashes.is_empty() { + PointerCatalog::new() + } else { + lookup.files(self.layout.store(), &hashes).await? + }; + let prepared_files = self + .content + .as_ref() + .map(|content| { + content + .files + .iter() + .map(|file| file.file_hash) + .collect::>() + }) + .unwrap_or_default(); + crab_read::dependency_proof::verify_capsule_dependencies_except_crab( + &self.layout, + &base_catalog, + self.plan.pointers(), + &prepared_files, + limits, + cancel, + ) + .await + .map_err(|error| Error::Dependency(Box::new(error)))?; + + let (pointer_delta, origins) = match &self.content { + Some(content) => { + upload_capsule_content(&self.layout, &lookup, base_catalog, content, cancel).await? + } + None => (PointerCatalog::new(), HashMap::new()), + }; + let mut packs = Vec::new(); + if let Some(pack) = &self.pack { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let (pack_bytes, index, reverse_index, locator) = tokio::try_join!( + tokio::fs::read(pack.pack_path()), + tokio::fs::read(pack.index_path()), + tokio::fs::read(pack.reverse_path()), + tokio::fs::read(pack.kinds_path()), + )?; + let external_delta_bases = pack + .external_delta_bases() + .iter() + .map(ToString::to_string) + .collect(); + packs.push( + crab_metadata::capsule_protocol::CapsuleGitPack::new_with_external_delta_bases( + Bytes::from(pack_bytes), + Bytes::from(index), + Bytes::from(reverse_index), + Bytes::from(locator), + pack.git_sha1().to_string(), + u64::from(pack.object_count()), + external_delta_bases, + )?, + ); + } + let mut sections = Vec::with_capacity(1); + if !self.visibility.is_empty() { + let visibility = crab_metadata::capsule_protocol::CapsuleVisibilityDelta::new( + self.visibility.clone(), + )?; + sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( + crab_metadata::capsule_protocol::CapsuleSectionKind::VisibilityDelta, + visibility.encode()?, + )); + } + let edits = self + .updates + .iter() + .map(|update| { + crab_metadata::capsule_protocol::CapsuleRefEdit::new( + update.name.clone(), + update.old.map(|oid| oid.to_string()), + update.new.map(|oid| oid.to_string()), + self.plan + .peeled() + .get(&update.name) + .map(ToString::to_string), + ) + }) + .collect::>(); + let changes_namespace = edits + .iter() + .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); + tracing::info!( + phase = "capsule_input", + packs = packs.len(), + elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, + "Git receive phase completed" + ); + Ok(CapsuleArtifacts { + layout: self.layout.clone(), + base: view.root_snapshot().clone(), + edits, + packs, + sections, + pointer_delta, + origins, + changes_namespace, + }) + } +} + +impl CapsuleArtifacts { + /// Commit through independently mutable ref heads and one capsule authority. + /// + /// Revalidate authorization and repository policy immediately before this + /// call. The caller must retain its original ref leases and GC fences until + /// the returned outcome is known and all lease cleanup has completed. + pub async fn commit( + self, + plan_id: Option<&str>, + namespace_ttl: Duration, + cancel: &CancellationToken, + ) -> Result { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let transaction = match plan_id { + Some(plan_id) => crab_metadata::capsule_protocol::CapsuleTransaction::for_plan( + self.base.record().digest(), + plan_id, + self.edits, + )?, + None => crab_metadata::capsule_protocol::CapsuleTransaction::new( + self.base.record().digest(), + self.edits, + )?, + }; + let transaction_id = transaction.id()?; + let started = Instant::now(); + let mut sections = self.sections; + if !self.pointer_delta.is_empty() { + sections.insert( + 0, + crab_metadata::capsule_protocol::CapsuleSection::new( + crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta, + self.pointer_delta.encode_delta()?, + ), + ); + } + let capsule = + crab_metadata::capsule_protocol::Capsule::build(&transaction, self.packs, sections)?; + tracing::info!( + phase = "capsule_encode", + %transaction_id, + bytes = capsule.bytes().len(), + elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, + "Git receive phase completed" + ); + let publish = if self.changes_namespace { + let ref_names = transaction + .edits() + .iter() + .map(|edit| edit.ref_name().to_owned()) + .collect::>(); + let layout = self.layout.clone(); + let commit_layout = layout.clone(); + let base = self.base; + crab_write::with_ref_namespaces( + layout.store(), + &layout, + &ref_names, + namespace_ttl, + cancel, + move |scoped| async move { + if scoped.is_cancelled() { + return Err(crab_write::WriteError::Cancelled); + } + crab_write::capsule_protocol::validate_ref_namespace( + &commit_layout, + base.record().root(), + transaction.edits(), + ) + .await?; + crab_write::capsule_protocol::publish( + &commit_layout, + base, + &transaction, + &capsule, + ) + .await + }, + ) + .await + } else { + crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) + .await + }; + match publish { + Ok(_) => { + // Ref visibility owns the payload before hints become discoverable. + // Failed hint maintenance cannot reject an already committed Git push. + let config = + crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + self.layout.repo_prefix(), + self.layout.global_prefix(), + ); + if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(self.layout.store().inner()), + &config, + self.layout.repo_prefix(), + &self.pointer_delta, + &self.origins, + ) + .await + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + } + Ok(CapsuleCommitOutcome::Committed { transaction_id }) + } + Err(error @ crab_write::WriteError::CapsuleCommitUncertain { .. }) => { + Ok(CapsuleCommitOutcome::Indeterminate { + transaction_id, + source: Box::new(error), + }) + } + Err(error) => Err(error.into()), + } + } +} + +async fn upload_capsule_content( + layout: &crab_storage::StoreLayout, + lookup: &PointerCatalogLookup, + mut base: PointerCatalog, + content: &PreparedContent, + cancel: &CancellationToken, +) -> Result<( + PointerCatalog, + HashMap, +)> { + let local = content.pointer_catalog(); + let mut delta = PointerCatalog::new(); + let mut origins = HashMap::new(); + let mut needed_shards = std::collections::BTreeSet::new(); + for (file_hash, file) in local.files() { + if let Some(existing) = base.files().get(file_hash) { + if existing.size() != file.size() { + return Err(Error::Content(format!( + "file {file_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + needed_shards.insert(file.shard_hash().to_owned()); + delta.insert_file(file_hash.clone(), file.clone())?; + } + if !needed_shards.is_empty() { + base.apply( + &lookup + .shards( + layout.store(), + &needed_shards.iter().cloned().collect::>(), + ) + .await?, + )?; + } + let mut needed_xorbs = std::collections::BTreeSet::new(); + for shard_hash in &needed_shards { + let shard = local + .shards() + .get(shard_hash) + .ok_or_else(|| Error::Content("prepared shard catalog entry is missing".to_owned()))?; + if let Some(existing) = base.shards().get(shard_hash) { + if existing != shard { + return Err(Error::Content(format!( + "shard {shard_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + needed_xorbs.extend(shard.xorb_hashes().iter().cloned()); + delta.insert_shard(shard_hash.clone(), shard.clone())?; + } + if !needed_xorbs.is_empty() { + base.apply( + &lookup + .xorbs( + layout.store(), + &needed_xorbs.iter().cloned().collect::>(), + ) + .await?, + )?; + } + let xorb_artifacts = content + .xorbs + .iter() + .map(|artifact| { + ( + crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), + artifact, + ) + }) + .collect::>(); + for xorb_hash in needed_xorbs { + let local_entry = local + .xorbs() + .get(&xorb_hash) + .ok_or_else(|| Error::Content("prepared xorb catalog entry is missing".to_owned()))?; + if let Some(existing) = base.xorbs().get(&xorb_hash) { + if existing.chunks() != local_entry.chunks() { + return Err(Error::Content(format!( + "xorb {xorb_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let artifact = xorb_artifacts + .get(&xorb_hash) + .ok_or_else(|| Error::Content("prepared xorb artifact is missing".to_owned()))?; + let bytes = artifact + .read_verified(crab_xet::xorb::format::MAX_XORB_SIZE as u64, cancel) + .await?; + let hash = crab_xet::hash::MerkleHash::from(artifact.protocol_hash); + let path = layout.xorb_path(&hash); + let payload_digest = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()) + .map_err(Error::ContentFormat)? + .payload_digest(); + let (entry, payload_digest, identity) = match layout + .store() + .create_or_read_immutable(&path, bytes, crab_xet::xorb::format::MAX_XORB_SIZE as u64) + .await? + { + crab_storage::ImmutableCreateOutcome::Created(identity) => { + (local_entry.clone(), payload_digest, identity) + } + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, + identity, + } => { + let expected = local_entry.clone(); + let expected_hash = xorb_hash.clone(); + let (entry, payload_digest) = tokio::task::spawn_blocking(move || { + catalog_xorb_from_existing(&expected_hash, &expected, existing) + }) + .await??; + (entry, payload_digest, identity) + } + }; + origins.insert( + hash, + crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + hash.into(), + payload_digest, + entry.encoded_size(), + identity.e_tag, + identity.version, + ), + ); + delta.insert_xorb(xorb_hash, entry)?; + } + let shard_artifacts = content + .shards + .iter() + .map(|artifact| { + ( + crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), + artifact, + ) + }) + .collect::>(); + for shard_hash in needed_shards { + if base.shards().contains_key(&shard_hash) { + continue; + } + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let artifact = shard_artifacts + .get(&shard_hash) + .ok_or_else(|| Error::Content("prepared shard artifact is missing".to_owned()))?; + let bytes = artifact + .read_verified(crab_xet::shard::MAX_PUSH_SHARD_SIZE_BYTES, cancel) + .await?; + layout + .store() + .put_if_absent_verified( + &layout.shard_path(&crab_xet::hash::MerkleHash::from(artifact.protocol_hash)), + bytes, + ) + .await?; + } + if !delta.shards().is_empty() { + crab_metadata::ref_registry::union_register_repo_shards( + layout.store(), + layout, + delta.shards().keys().cloned().collect(), + ) + .await?; + } + base.apply(&delta)?; + Ok((delta, origins)) +} + +fn catalog_xorb_from_existing( + xorb_hash: &str, + local: &XorbCatalogEntry, + bytes: Bytes, +) -> Result<(XorbCatalogEntry, [u8; 32])> { + let parser = + crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).map_err(Error::ContentFormat)?; + parser + .verify_payload_digest() + .map_err(Error::ContentFormat)?; + parser.verify_all_chunks().map_err(Error::ContentFormat)?; + if parser.hash().hex() != xorb_hash { + return Err(Error::Content( + "existing xorb has the wrong logical identity".to_owned(), + )); + } + let chunks = (0..parser.num_chunks()) + .map(|index| { + parser + .chunk_meta(index) + .map(|chunk| XorbChunkEntry::new(chunk.hash.hex(), chunk.uncompressed_len)) + }) + .collect::, _>>() + .map_err(Error::ContentFormat)?; + if chunks != local.chunks() { + return Err(Error::Content( + "existing xorb has a conflicting chunk layout".to_owned(), + )); + } + Ok(( + XorbCatalogEntry::new( + bytes.len() as u64, + blake3::hash(&bytes).to_hex().to_string(), + chunks, + ), + parser.payload_digest(), + )) +} diff --git a/crates/crab-remote/tests/capsule_preparation.rs b/crates/crab-remote/tests/capsule_preparation.rs new file mode 100644 index 000000000..52d9db6fe --- /dev/null +++ b/crates/crab-remote/tests/capsule_preparation.rs @@ -0,0 +1,621 @@ +#![cfg(feature = "publication")] + +use std::sync::Arc; + +use bytes::Bytes; +use crab_remote::prepare::{ContentArtifact, ContentFile, PreparedContent}; +use crab_xet::hash::{MerkleHash, compute_data_hash}; +use crab_xet::shard::{PushShardSession, file_info_from_placements, xorb_info_from_placements}; +use crab_xet::xorb::builder::{RunId, XorbBuilder}; +use crab_xet::xorb::format::Chunk; +use tokio_util::sync::CancellationToken; + +const LIMIT: u64 = 8 * 1024 * 1024; + +async fn content_fixture() -> (tempfile::TempDir, PreparedContent) { + let directory = tempfile::tempdir().unwrap(); + let data = Bytes::from_static(b"prepared capsule content must remain reconstructable"); + let chunk_hash = compute_data_hash(&data); + let file_hash = MerkleHash::from(*blake3::hash(&data).as_bytes()); + let mut builder = XorbBuilder::new(); + builder + .push( + &Chunk { + hash: chunk_hash, + data: data.clone(), + }, + RunId(0), + ) + .unwrap(); + let xorb = builder.finalize().unwrap().pop().unwrap(); + let placements = xorb + .placements + .iter() + .map(|placement| (placement.chunk_hash, placement.clone())) + .collect(); + let file = file_info_from_placements(file_hash, &[chunk_hash], &placements).unwrap(); + let xorb_info = Arc::new(xorb_info_from_placements(xorb.hash, &xorb.placements).unwrap()); + let mut shards = PushShardSession::new(); + shards.add_file_bundle(file, &[xorb_info]).unwrap(); + let (shard, shard_hash) = shards.finalize().unwrap().pop().unwrap(); + let xorb_path = directory.path().join("content.xorb"); + let shard_path = directory.path().join("content.shard"); + std::fs::write(&xorb_path, &xorb.bytes).unwrap(); + std::fs::write(&shard_path, &shard).unwrap(); + let content = PreparedContent::new( + vec![ContentArtifact::new( + xorb.hash.into(), + *blake3::hash(&xorb.bytes).as_bytes(), + xorb.bytes.len() as u64, + xorb_path, + )], + vec![ContentArtifact::new( + shard_hash.into(), + *blake3::hash(&shard).as_bytes(), + shard.len() as u64, + shard_path, + )], + vec![ContentFile::new( + file_hash.into(), + data.len() as u64, + shard_hash.into(), + )], + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + (directory, content) +} + +#[tokio::test] +async fn prepared_catalog_preserves_raw_blake3_body_identity() { + let (directory, content) = content_fixture().await; + let expected = blake3::hash(&std::fs::read(directory.path().join("content.xorb")).unwrap()) + .to_hex() + .to_string(); + let actual = content + .pointer_catalog() + .xorbs() + .values() + .next() + .unwrap() + .body_digest(); + assert_eq!( + actual, expected, + "body digests use raw Blake3 hex; only protocol identities use Xet word order" + ); +} + +#[derive(Default)] +struct Reads(std::sync::atomic::AtomicUsize); +impl crab_storage::StorageObserver for Reads { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, observation: crab_storage::StorageObservation) { + if matches!( + observation.operation, + crab_storage::StorageOperation::Get | crab_storage::StorageOperation::Range + ) { + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + } +} + +fn git(directory: &std::path::Path, args: &[&str]) -> Vec { + git_input(directory, args, b"") +} + +fn git_input(directory: &std::path::Path, args: &[&str], input: &[u8]) -> Vec { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgSign=false", + "-c", + "core.hooksPath=/dev/null", + ]) + .args(args) + .current_dir(directory) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(input).unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + +fn source_commit(directory: &std::path::Path, content: &PreparedContent) { + git(directory, &["init", "-b", "main"]); + let file = &content.files()[0]; + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\n", + blake3::Hash::from_bytes(*file.file_hash()).to_hex(), + file.size() + ); + std::fs::write(directory.join("model.bin"), pointer).unwrap(); + git(directory, &["add", "model.bin"]); + git(directory, &["commit", "-m", "publish pointer"]); +} + +async fn layout() -> (crab_storage::StoreLayout, Arc) { + let reads = Arc::new(Reads::default()); + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(reads.clone()); + let layout = crab_storage::StoreLayout::new(store, "repositories/prepared".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + (layout, reads) +} + +fn proof_limits() -> crab_read::dependency_proof::DependencyProofLimits { + crab_read::dependency_proof::DependencyProofLimits { + max_dependencies: 16, + max_total_file_bytes: LIMIT, + max_duration: std::time::Duration::from_secs(60), + lookup: crab_metadata::file_index_lookup::FileIndexLookupLimits { + max_files: 16, + max_shard_visits: 16, + max_shard_bytes: LIMIT, + max_recipe_entries: 1024, + }, + content: crab_read::pointer_proof::PointerProofLimits { + max_file_bytes: LIMIT, + max_shard_bytes: LIMIT, + max_xorb_bytes: LIMIT, + max_read_bytes: LIMIT, + max_chunks: 1024, + max_duration: std::time::Duration::from_secs(60), + }, + } +} + +async fn prepare( + layout: &crab_storage::StoreLayout, + directory: &std::path::Path, + content: Option, +) -> ( + crab_remote::prepare::Prepared, + crab_read::capsule_protocol::CapsuleRepositoryView, + Arc, + crab_remote::publication::PublicationLeases, +) { + let cancel = CancellationToken::new(); + let leases = crab_remote::publication::acquire_leases( + layout.store(), + layout, + ["refs/heads/main".to_owned()], + crab_remote::publication::LeaseOptions::renewing(std::time::Duration::from_secs(60)), + &cancel, + ) + .await + .unwrap(); + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + let view = crab_read::capsule_protocol::open_git_view_from_root( + layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let repository = view + .git_repository_from_store( + layout.clone(), + crab_remote_git::RepositoryIdentity::new("memory", "prepared", 1).unwrap(), + runtime.clone(), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap(); + let oid = gix_hash::ObjectId::from_hex( + String::from_utf8(git(directory, &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .as_bytes(), + ) + .unwrap(); + let mut revisions = String::from("HEAD\n"); + if let Some(old) = view.refs().get("refs/heads/main") { + revisions.push_str(&format!("^{old}\n")); + } + let pack = git_input( + directory, + &["pack-objects", "--stdout", "--revs"], + revisions.as_bytes(), + ); + let incoming = directory.join("incoming.pack"); + std::fs::write(&incoming, pack).unwrap(); + let limits = crab_git::incoming_pack::ReceiveLimits { + max_pack_bytes: LIMIT, + max_objects: 1024, + max_object_bytes: LIMIT as usize, + max_inflated_bytes: LIMIT, + max_delta_depth: 128, + }; + let prepared = crab_remote::prepare::prepare_capsule( + repository, + view.git_visibility_index().await.unwrap(), + crab_git::incoming_pack::ScratchSpace::new(directory, limits), + Some(std::io::BufReader::new( + std::fs::File::open(incoming).unwrap(), + )), + vec![crab_git::receive_plan::RefUpdate { + name: "refs/heads/main".to_owned(), + old: view + .refs() + .get("refs/heads/main") + .map(|oid| gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap()), + new: Some(oid), + }], + std::collections::BTreeMap::new(), + &cancel, + crab_remote::prepare::Options { + layout: layout.clone(), + graph: crab_git::receive_plan::GraphLimits { + max_ref_updates: 16, + max_graph_steps: 1024, + max_object_bytes: LIMIT as usize, + max_read_bytes: LIMIT, + }, + pack: limits, + policy: |_: &str| crab_git::receive_plan::RefPolicy { + allow_delete: true, + allow_non_fast_forward: false, + }, + }, + ) + .await + .unwrap(); + let prepared = match content { + Some(content) => prepared.attach_content(content).unwrap(), + None => prepared, + }; + (prepared, view, runtime, leases) +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_upload_rejects_changed_prepared_bytes() { + let mut rejected = Vec::new(); + for artifact in ["content.xorb", "content.shard"] { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + std::fs::write(directory.path().join(artifact), b"changed after inspection").unwrap(); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + leases.release().await; + runtime.shutdown().await; + rejected.push((artifact, result.is_err())); + } + assert!( + rejected.iter().all(|(_, failed)| *failed), + "changed artifacts must fail before ref publication: {rejected:?}" + ); +} + +async fn verify_indexed_publication( + layout: crab_storage::StoreLayout, + occupied: bool, +) { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let chunk = content + .pointer_catalog() + .xorbs() + .values() + .next() + .unwrap() + .chunks()[0] + .hash() + .to_owned(); + if occupied { + let artifact = &content.xorbs()[0]; + layout + .store() + .put( + &layout.xorb_path(&MerkleHash::from(*artifact.protocol_hash())), + Bytes::from(std::fs::read(directory.path().join("content.xorb")).unwrap()), + ) + .await + .unwrap(); + } + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let result = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!(matches!( + result.unwrap(), + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + let reader = + slatedb::DbReader::builder(config.chunk_index_path, Arc::clone(layout.store().inner())) + .build() + .await + .expect("shared publication must populate cold native-add candidates"); + let hash = MerkleHash::from_hex(&chunk).unwrap(); + let candidate = reader + .get(crab_metadata::key_codec::capsule_chunk_head_key( + &hash.into(), + )) + .await + .unwrap() + .expect("published payload candidate"); + let placement = crab_metadata::receipts::CommittedChunkPlacement::decode(&candidate).unwrap(); + let anchor = reader + .get(crab_metadata::key_codec::capsule_chunk_anchor_key( + &placement.source_anchor_id, + )) + .await + .unwrap() + .unwrap(); + let anchor = crab_metadata::capsule_protocol::CapsuleChunkAnchor::decode( + &placement.source_anchor_id, + &anchor, + ) + .unwrap(); + reader.close().await.unwrap(); + let path = layout.xorb_path(&MerkleHash::from(placement.xorb_hash)); + let (body, identity) = layout.store().get_with_etag(&path).await.unwrap(); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let current = crab_read::capsule_protocol::open_git_view_from_root( + &layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); + let catalog = current.pointer_catalog().await.unwrap(); + anchor + .validate_placement(&placement, &catalog, path.as_ref()) + .unwrap(); + assert_eq!( + ( + anchor.origin.etag, + anchor.origin.object_version, + anchor.body_digest + ), + ( + identity.e_tag, + identity.version, + blake3::hash(&body).to_hex().to_string() + ) + ); + let crab_git::pointer_detect::PointerKind::Crab(pointer) = crab_git::pointer_detect::classify( + &std::fs::read(directory.path().join("model.bin")).unwrap(), + ) else { + panic!("fixture pointer"); + }; + crab_read::pointer_proof::verify_crab_pointer( + &layout, + &pointer, + MerkleHash::from_hex(catalog.files().values().next().unwrap().shard_hash()).unwrap(), + proof_limits().content, + &CancellationToken::new(), + ) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_commit_indexes_verified_native_payload() { + for occupied in [false, true] { + let (layout, _) = layout().await; + verify_indexed_publication(layout, occupied).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] +async fn live_capsule_preparation_publishes_and_reconstructs_native_content() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").expect("explicit scratch bucket required"); + let store = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + for occupied in [false, true] { + let prefix = format!("shared-preparation/{nonce}/{occupied}"); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.clone(), + format!("{prefix}/repo"), + format!("{prefix}/global"), + ); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + verify_indexed_publication(layout, occupied).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn git_only_capsule_upload_does_not_read_historical_catalog() { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, reads) = layout().await; + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let committed = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + committed, + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + leases.release_after_commit().await; + runtime.shutdown().await; + git(directory.path(), &["rm", "model.bin"]); + std::fs::write(directory.path().join("README.md"), b"ordinary Git update").unwrap(); + git(directory.path(), &["add", "README.md"]); + git( + directory.path(), + &["commit", "-m", "replace pointer with ordinary Git file"], + ); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), None).await; + assert!(prepared.plan().pointers().is_empty()); + reads.0.store(0, std::sync::atomic::Ordering::Relaxed); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + let count = reads.0.load(std::sync::atomic::Ordering::Relaxed); + leases.release().await; + runtime.shutdown().await; + result.unwrap(); + assert_eq!( + count, 0, + "ordinary Git publication must not read unrelated Xet history" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn committed_capsule_survives_candidate_index_failure() { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + let db = slatedb::Db::open(config.chunk_index_path, Arc::clone(layout.store().inner())) + .await + .unwrap(); + db.put( + crab_metadata::key_codec::encode_system_key(crab_metadata::key_codec::SYS_FORMAT_VERSION), + 2u32.to_le_bytes(), + ) + .await + .unwrap(); + db.close().await.unwrap(); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let result = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!( + matches!( + result.unwrap(), + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + ), + "repairable hint-index failure must not reject committed refs" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn historical_pointer_lookup_preserves_declared_size_checks() { + for size_delta in [0, 1] { + let (directory, content) = content_fixture().await; + let file = content.files()[0].clone(); + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let committed = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + committed, + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + leases.release_after_commit().await; + runtime.shutdown().await; + // A hint changes the Git blob identity while retaining the existing file. + // Native negotiation therefore sends a real historical-pointer dependency. + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\nshard-hint {}\n", + blake3::Hash::from_bytes(*file.file_hash()).to_hex(), + file.size() + size_delta, + blake3::Hash::from_bytes(*file.shard_hash()).to_hex() + ); + std::fs::write(directory.path().join("model.bin"), pointer).unwrap(); + git(directory.path(), &["add", "model.bin"]); + git( + directory.path(), + &["commit", "-m", "update pointer binding"], + ); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), None).await; + assert_eq!(prepared.plan().pointers().len(), 1); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + leases.release().await; + runtime.shutdown().await; + assert_eq!( + result.is_ok(), + size_delta == 0, + "historical pointer size delta {size_delta}" + ); + } +} diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index 6291ec7c5..f82e41126 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -45,6 +45,10 @@ writes, bounded reads, byte/request observers, and provider-neutral `StorageError` values. `put_if_absent` distinguishes a new immutable object from an identical retry without a preceding HEAD, and streamed size/hash verification uses the GET response metadata instead of a separate HEAD. +`create_or_read_immutable` retains both ETag and version from the successful +PUT, verified readback, or occupied-key GET alongside its outcome. Origin proof +consumers must carry that identity with the verified bytes; a later HEAD may +observe another object version. Occupied-key bytes still require caller validation. `Store::verify_bounded` combines that content-hash contract with pre-body size admission and streamed length checks. It uses one GET for a successful read; the limit applies to buffered encoded bytes, not the caller's decoded objects. diff --git a/crates/crab-storage/src/store.rs b/crates/crab-storage/src/store.rs index 77583e604..edfbb3d96 100644 --- a/crates/crab-storage/src/store.rs +++ b/crates/crab-storage/src/store.rs @@ -60,10 +60,10 @@ pub enum ImmutableWriteVerification { /// Result of a create-only immutable write whose occupied key is returned to the caller. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ImmutableCreateOutcome { - /// This call created and verified the requested bytes. - Created, - /// The key was already occupied; these bounded bytes remain untrusted. - Existing(Bytes), + /// This call created and verified the bytes; identity belongs to that PUT or verified readback. + Created(ETag), + /// The key was occupied; identity belongs to this bounded GET, whose bytes remain untrusted. + Existing { bytes: Bytes, identity: ETag }, } /// Bounded-memory byte stream returned by object reads. @@ -727,11 +727,11 @@ impl Store { }; let payload = upload_payload(bytes); match self.inner.put_opts(&path, payload, options).await { - Ok(_) => Ok(true), + Ok(result) => Ok(Some(ETag::from(result))), Err(error) => { let mapped = map_object_store_error(error, path.as_ref()); if matches!(mapped, StorageError::StateConflict { .. }) { - Ok(false) + Ok(None) } else { Err(mapped) } @@ -740,13 +740,13 @@ impl Store { } }) .await?; - if !created { - let (existing, _) = self.get_with_etag_bounded(path, max_existing_bytes).await?; - return Ok(ImmutableCreateOutcome::Existing(existing)); - } + let Some(mut identity) = created else { + let (bytes, identity) = self.get_with_etag_bounded(path, max_existing_bytes).await?; + return Ok(ImmutableCreateOutcome::Existing { bytes, identity }); + }; if self.immutable_write_verification == ImmutableWriteVerification::ReadbackRequired { let maximum = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - let (stored, _) = self.get_with_etag_bounded(path, maximum).await?; + let (stored, observed_identity) = self.get_with_etag_bounded(path, maximum).await?; let actual_hash = *blake3::hash(&stored).as_bytes(); if actual_hash != expected_hash { return Err(StorageError::CorruptObject { @@ -758,8 +758,11 @@ impl Store { ), }); } + // Bind proof to the object version whose bytes were verified. A + // later HEAD may observe a replacement with a different encoding. + identity = observed_identity; } - Ok(ImmutableCreateOutcome::Created) + Ok(ImmutableCreateOutcome::Created(identity)) } /// Writes `bytes` at `path` iff nothing exists there yet. @@ -3763,6 +3766,7 @@ mod tests { Body(Vec), BodyError, Range(Range), + DistinctIdentities, } #[derive(Debug)] @@ -3785,7 +3789,12 @@ mod tests { payload: PutPayload, options: PutOptions, ) -> object_store::Result { - self.inner.put_opts(location, payload, options).await + let mut result = self.inner.put_opts(location, payload, options).await?; + if matches!(self.fault, ReadFault::DistinctIdentities) { + result.e_tag = Some("put-etag".to_owned()); + result.version = Some("put-version".to_owned()); + } + Ok(result) } async fn put_multipart_opts( @@ -3840,6 +3849,10 @@ mod tests { } ReadFault::Range(range) => response.range = range.clone(), ReadFault::Transient(_) => {} + ReadFault::DistinctIdentities => { + response.meta.e_tag = Some("get-etag".to_owned()); + response.meta.version = Some("get-version".to_owned()); + } } } Ok(response) @@ -4194,7 +4207,78 @@ mod tests { .await .unwrap(); - assert_eq!(outcome, ImmutableCreateOutcome::Existing(existing)); + let observed = store.head(&path).await.unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Existing { + bytes: existing, + identity: ETag { + e_tag: observed.e_tag, + version: observed.version + }, + } + ); + } + + #[tokio::test] + async fn logical_immutable_identity_belongs_to_the_verified_operation() { + for (existing, verification) in [ + (false, ImmutableWriteVerification::Sha256Checksum), + (false, ImmutableWriteVerification::ReadbackRequired), + (true, ImmutableWriteVerification::ReadbackRequired), + ] { + let inner = Arc::new(InMemory::new()); + let path = Path::from("blobs/operation-identity"); + let bytes = Bytes::from_static(b"candidate"); + if existing { + inner.put(&path, bytes.clone().into()).await.unwrap(); + } + let store = memory_store_with_inner(Arc::new(ReadFaultStore { + inner, + fault: ReadFault::DistinctIdentities, + })) + .with_immutable_write_verification(verification); + let outcome = store + .create_or_read_immutable(&path, bytes.clone(), 1024) + .await + .unwrap(); + let observed = + if existing || verification == ImmutableWriteVerification::ReadbackRequired { + "get" + } else { + "put" + }; + let identity = ETag { + e_tag: Some(format!("{observed}-etag")), + version: Some(format!("{observed}-version")), + }; + let expected = if existing { + ImmutableCreateOutcome::Existing { bytes, identity } + } else { + ImmutableCreateOutcome::Created(identity) + }; + assert_eq!( + outcome, expected, + "existing={existing}, verification={verification:?}" + ); + } + } + + #[tokio::test] + async fn logical_immutable_create_withholds_identity_when_readback_is_corrupt() { + let store = memory_store_with_inner(Arc::new(ReadFaultStore { + inner: Arc::new(InMemory::new()), + fault: ReadFault::Body(vec![Bytes::from_static(b"changed!!")]), + })); + let error = store + .create_or_read_immutable( + &Path::from("blobs/corrupt-readback"), + Bytes::from_static(b"candidate"), + 1024, + ) + .await + .expect_err("invalid bytes cannot acquire a verified identity"); + assert!(matches!(error, StorageError::CorruptObject { .. })); } #[tokio::test] @@ -4232,8 +4316,14 @@ mod tests { .await .unwrap(); - assert_eq!(outcome, ImmutableCreateOutcome::Created); let result = inner.get(&path).await.unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Created(ETag { + e_tag: result.meta.e_tag.clone(), + version: result.meta.version.clone(), + }) + ); assert_eq!( result.attributes.get(&Attribute::StorageClass), Some(&"STANDARD_IA".to_owned().into()) @@ -4269,7 +4359,7 @@ mod tests { .await .unwrap(); - assert!(matches!(outcome, ImmutableCreateOutcome::Existing(_))); + assert!(matches!(outcome, ImmutableCreateOutcome::Existing { .. })); let result = inner.get(&path).await.unwrap(); assert_eq!( result.attributes.get(&Attribute::StorageClass),