From 0aad061552b2d7ebe7137ca99a5254af7c7a5532 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:18:07 -0700 Subject: [PATCH 01/17] fix(fsck): verify retained browse dependencies --- crab/src/cmd/fsck_store.rs | 29 ++ crates/crab-metadata/README.md | 5 + .../src/capsule_protocol/browse_retention.rs | 88 +++++- .../browse_retention/tests.rs | 293 ++++++++++++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 2 + .../src/git_object_locator/paged/reader.rs | 30 ++ 6 files changed, 446 insertions(+), 1 deletion(-) diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 17c2cabd0..8dc5aae48 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -151,6 +151,18 @@ impl StoreChecker { } async fn check_capsule_data_chain(&self, state: &CapsuleFsckState) -> Result> { + let layout = crab_storage::StoreLayout::with_global_prefix( + self.store.as_storage().clone(), + self.router.repo_prefix().to_owned(), + self.router.global_prefix().to_owned(), + ); + // Current catalog health cannot prove cold captures still have their + // original sources. Full fsck checks lifetime roots; mirror recipe probes stay narrow. + crab_metadata::capsule_protocol::verify_retained_browse_indexes( + &layout, + MAX_FSCK_FRONTIER_BYTES, + ) + .await?; let mut issues = Vec::new(); for (hash, entry) in state.catalog.xorbs() { @@ -2060,6 +2072,23 @@ mod tests { assert!(shard_issues.is_empty()); } + #[tokio::test] + async fn capsule_data_check_rejects_corrupt_browse_lifetime() { + let (store, prefix, root, _, _) = capsule_checker_fixture().await; + let checker = StoreChecker::for_capsule_repository(store.clone(), prefix, root) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + store + .put( + &checker.router.repo_path("v2/browse-retention"), + bytes::Bytes::from_static(b"corrupt"), + ) + .await + .unwrap(); + assert!(checker.check_data_chain().await.is_err()); + } + #[tokio::test] async fn capsule_checker_validates_repository_without_legacy_layout() { let (store, prefix, root, _, _) = capsule_checker_fixture().await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 62c676373..69c34c83d 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -130,6 +130,11 @@ foreground reads still load only the current bounded readiness record. An unchanged capture does not append a node. Missing/corrupt lifetime dependencies stop GC before deletion. A valid readiness record without lifetime ownership requires background browse maintenance before GC; there is no scan fallback. +Full capsule fsck also authenticates the retained chain, decodes complete +graph/path-state indexes, and verifies physical-index and original source bytes, +including superseded ref-only runs. Shared captures stream each source body once +during this phase. Lifetime/readiness movement during verification requires a retry; +mirror pointer-recipe probes do not acquire this full maintenance workload. Explicit history pruning trims superseded captures below the oldest retained checkpoint generation only after the authoritative history CAS succeeds. Current diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 89a06c4b9..48b26afb3 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -11,7 +11,7 @@ use super::{ use crate::error::{MetadataError, Result}; use crate::validation::{corrupt_object, validate_content_hash}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Head { version: u32, @@ -285,6 +285,92 @@ pub async fn load_retained_browse_indexes( } } +/// Verify retained index/source integrity against stable lifetime and readiness heads. +/// +/// Maintenance decodes complete graph/path indexes and streams original source +/// and physical-index bodies. This proves retained bytes, never ref visibility. +#[cfg(feature = "remote-index")] +pub async fn verify_retained_browse_indexes( + layout: &StoreLayout, + maximum_bytes: u64, +) -> Result<()> { + let head = load_head(layout).await?; + let current = super::load_browse_indexes(layout).await?; + let records = match &head { + Some((head, _)) => load_chain(layout, head).await?, + None => Vec::new(), + }; + if current + .as_ref() + .is_some_and(|current| !records.iter().any(|record| record.indexes() == current)) + { + return Err(corrupt("published browse indexes have no retained capture")); + } + let mut sources = BTreeSet::new(); + for record in records { + let graph = crate::split_commit_graph::load_split_commit_graph( + layout.store(), + layout, + record.indexes().commit_graph_hash(), + maximum_bytes.min(crate::split_commit_graph::DEFAULT_MAX_SPLIT_COMMIT_GRAPH_BYTES), + ) + .await?; + if graph.descriptor.generation != record.generation() { + return Err(corrupt( + "retained graph generation differs from its capture", + )); + } + if let Some(hash) = record.indexes().path_state_hash() { + crate::path_state::load_path_state( + layout.store(), + layout, + hash, + &graph, + maximum_bytes.min(crate::path_state::DEFAULT_MAX_PATH_STATE_BYTES), + ) + .await?; + } + if let Some(hash) = record.indexes().object_index_hash() { + crate::git_object_locator::PagedGitObjectIndex::verify_retained( + layout.store(), + layout, + hash, + record.indexes().state_digest(), + &graph.descriptor, + maximum_bytes, + ) + .await?; + } + // Captures may share original multi-gigabyte sources. Authenticate each + // immutable body once, including ref-only sources absent from current history. + for path in record.source_paths(layout) { + if !sources.insert(path.clone()) { + continue; + } + let size = layout.store().head(&path).await?.size; + if size > maximum_bytes { + return Err(corrupt("retained source exceeds byte limit")); + } + let hash = path + .filename() + .and_then(|hash| blake3::Hash::from_hex(hash).ok()) + .ok_or_else(|| corrupt("retained source path has no content identity"))?; + layout + .store() + .verify_size_and_hash(&path, size, hash.as_bytes()) + .await?; + } + } + // Browse maintenance can change lifetime without moving a Git ref. A + // concurrent prune/rebuild must yield a retry, never a clean mixed capture. + if load_head(layout).await? != head || super::load_browse_indexes(layout).await? != current { + return Err(corrupt( + "browse retention changed during verification; retry fsck", + )); + } + Ok(()) +} + /// Trim captured generations after authoritative history pruning commits. /// /// The caller holds the exclusive repository GC lease. Failure leaves extra diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index 61412516f..e5fb4d855 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -15,6 +15,299 @@ fn indexes(byte: char) -> BrowseIndexes { BrowseIndexes::new(byte.to_string().repeat(64), "b".repeat(64), None).unwrap() } +#[cfg(feature = "remote-index")] +async fn integrity_fixture() -> (StoreLayout, Vec) { + use super::super::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleRun, CapsuleTransaction, PackLayer, + PointerCatalog, + }; + use crate::git_object_locator::{ + GitObjectLocation, GitObjectLocator, GitObjectMetadata, GitPackInventoryEntry, + PagedGitObjectIndexWrite, + }; + use crate::path_state::{PathStateInput, append_path_state, upload_path_state}; + use crate::split_commit_graph::{ + CommitGraphInput, append_paged_commit_graph, load_split_commit_graph, + upload_split_commit_graph, + }; + + let layout = fixture(); + let mut manifest = crate::manifests::Manifest::default_for_repo("refs/heads/main"); + manifest.generation = 1; + let pack_id = crab_xet::hash::MerkleHash::from([3; 32]); + let inventory = [GitPackInventoryEntry { + pack_id, + object_count: 1, + pack_size: 42, + }]; + manifest.pack_index_hash = crate::manifests::compact_pack_index( + 1, + &[crate::manifests::PackManifestEntry { + pack_id: pack_id.to_string(), + size: 42, + content_hash: pack_id.to_string(), + ref_tips: Vec::new(), + object_count: 1, + }], + ) + .unwrap() + .0; + manifest.git_validation_digest = "b".repeat(64); + let graph = append_paged_commit_graph( + None, + 1, + manifest.pack_index_hash.clone(), + manifest.git_validation_digest.clone(), + &[[1; 20]], + vec![CommitGraphInput { + oid: [1; 20], + tree_oid: [2; 20], + commit_time: 1, + parents: Vec::new(), + }], + ) + .unwrap() + .unwrap(); + upload_split_commit_graph(layout.store(), &layout, &graph) + .await + .unwrap(); + let graph_hash = graph.descriptor_hash.clone(); + let mut paths = vec![layout.repo_path(&graph.layers[0].reference.path)]; + let graph = load_split_commit_graph(layout.store(), &layout, &graph_hash, 1024 * 1024) + .await + .unwrap(); + let path_state = append_path_state( + None, + &graph, + vec![PathStateInput { + oid: [1; 20], + first_parent: None, + author: Vec::new(), + author_seconds: 1, + message: Vec::new(), + mutations: Vec::new(), + }], + ) + .unwrap(); + upload_path_state(layout.store(), &layout, &path_state, 1024 * 1024) + .await + .unwrap(); + paths.push(layout.repo_path(&path_state.layers[0].reference.path)); + let entries = [( + [1; 20], + GitObjectLocator { + ordinal: 0, + pack_id, + location: GitObjectLocation { + pack_offset: 12, + entry_len: 10, + crc32: 0, + }, + metadata: GitObjectMetadata::default(), + }, + )]; + let physical = + PagedGitObjectIndexWrite::build(&"a".repeat(64), &manifest, &inventory, &entries).unwrap(); + let hash = physical.upload(layout.store(), &layout).await.unwrap(); + paths.push( + crate::git_object_locator::PagedGitObjectIndex::retained_paths( + layout.store(), + &layout, + &hash, + &"a".repeat(64), + 1024 * 1024, + ) + .await + .unwrap()[1] + .clone(), + ); + let indexes = BrowseIndexes::new( + "a".repeat(64), + graph_hash.clone(), + Some(path_state.descriptor_hash), + ) + .unwrap() + .with_object_index(hash) + .unwrap(); + let transaction = CapsuleTransaction::new( + &"c".repeat(64), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("1".repeat(40)), + None, + )], + ) + .unwrap(); + let run = + CapsuleRun::leaf(Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap()).unwrap(); + let run_path = layout.capsule_path(run.hash()); + layout + .store() + .put(&run_path, run.bytes().clone()) + .await + .unwrap(); + paths.push(run_path); + let pointer = CapsulePointer::new( + run.hash(), + run.bytes().len() as u64, + run.control_offset(), + run.control_size(), + run.footer_hash(), + run.level(), + run.transaction_ids(), + run.newest_base_root_digest(), + ) + .unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from(vec![0x42; 1024]), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "3".repeat(40), + 1, + ) + .unwrap(); + let layer = PackLayer::build(&pack).unwrap(); + let layer_path = layout.capsule_pack_layer_path(layer.hash()); + layout + .store() + .put(&layer_path, layer.bytes().clone()) + .await + .unwrap(); + paths.push(layer_path); + let checkpoint = LayeredCheckpoint::build( + 1, + &"c".repeat(64), + vec![layer.source_descriptor().unwrap()], + PointerCatalog::new(), + None, + ) + .unwrap(); + let checkpoint_path = layout.capsule_checkpoint_path(checkpoint.hash()); + layout + .store() + .put(&checkpoint_path, checkpoint.bytes().clone()) + .await + .unwrap(); + paths.push(checkpoint_path); + retain_browse_indexes(&layout, &indexes, 1, Some(&checkpoint), &[pointer]) + .await + .unwrap(); + // The old bodies now belong only to retained capture lifetime, not the + // latest readiness or any authoritative checkpoint-history chain. + let latest = BrowseIndexes::new("d".repeat(64), graph_hash, None).unwrap(); + retain_browse_indexes(&layout, &latest, 1, None, &[]) + .await + .unwrap(); + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + latest.encode().unwrap(), + ) + .await + .unwrap(); + (layout, paths) +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn integrity_checks_every_retained_body_after_readiness_replacement() { + let (layout, paths) = integrity_fixture().await; + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); + for path in paths { + let original = layout.store().get_with_etag(&path).await.unwrap().0; + for missing in [false, true] { + if missing { + layout.store().delete(&path).await.unwrap(); + } else { + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + } + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err(), + "missing={missing}, path={path}" + ); + layout + .store() + .put_overwrite(&path, original.clone()) + .await + .unwrap(); + } + } + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn advertised_readiness_requires_authenticated_lifetime_ownership() { + let layout = fixture(); + layout + .store() + .create_strict_with_etag( + &layout.capsule_browse_indexes_path(), + indexes('a').encode().unwrap(), + ) + .await + .unwrap(); + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err() + ); +} + +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn retained_physical_index_must_match_its_graph_and_state() { + for field in [ + "state_digest", + "generation", + "pack_index_hash", + "git_validation_digest", + ] { + let (layout, _) = integrity_fixture().await; + let records = load_retained_browse_indexes(&layout).await.unwrap(); + let indexes = records[1].indexes(); + let path = + layout.bulk_manifest_path("git-object-index", indexes.object_index_hash().unwrap()); + let bytes = layout.store().get_with_etag(&path).await.unwrap().0; + let mut descriptor: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + descriptor[field] = if field == "generation" { + serde_json::json!(2) + } else { + serde_json::json!("e".repeat(64)) + }; + let bytes = Bytes::from(serde_json::to_vec(&descriptor).unwrap()); + let hash = blake3::hash(&bytes).to_hex().to_string(); + layout + .store() + .put(&layout.bulk_manifest_path("git-object-index", &hash), bytes) + .await + .unwrap(); + let indexes = indexes.clone().with_object_index(hash).unwrap(); + retain_browse_indexes(&layout, &indexes, 1, None, &[]) + .await + .unwrap(); + assert!( + verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .is_err(), + "field={field}" + ); + } +} + #[tokio::test] async fn unchanged_capture_does_not_grow_retention_or_rewrite_its_head() { let layout = fixture(); diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index d456fb982..17b51aaab 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -31,6 +31,8 @@ mod visibility; #[cfg(feature = "storage")] pub use browse::load_browse_indexes; pub use browse::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES}; +#[cfg(feature = "remote-index")] +pub use browse_retention::verify_retained_browse_indexes; #[cfg(feature = "storage")] pub use browse_retention::{ RetainedBrowseIndexes, load_retained_browse_indexes, prune_retained_browse_indexes, diff --git a/crates/crab-metadata/src/git_object_locator/paged/reader.rs b/crates/crab-metadata/src/git_object_locator/paged/reader.rs index f26164933..2326d40e4 100644 --- a/crates/crab-metadata/src/git_object_locator/paged/reader.rs +++ b/crates/crab-metadata/src/git_object_locator/paged/reader.rs @@ -53,6 +53,36 @@ impl PagedGitObjectIndex { ]) } + /// Authenticate a retained body's bytes and graph binding without granting visibility. + /// + /// The caller supplies an authenticated graph descriptor from the same capture. + pub async fn verify_retained( + store: &Store, + layout: &StoreLayout, + hash: &str, + state_digest: &str, + graph: &crate::split_commit_graph::CommitGraphDescriptor, + maximum: u64, + ) -> Result<()> { + let (descriptor, _) = Self::descriptor(store, layout, hash, maximum).await?; + if descriptor.state_digest != state_digest + || descriptor.generation != graph.generation + || descriptor.pack_index_hash != graph.pack_index_hash + || descriptor.git_validation_digest != graph.git_validation_digest + || descriptor.body_bytes > maximum + { + return bad("retained object index does not match graph or byte limit"); + } + store + .verify_size_and_hash( + &body_path(layout, &descriptor.body_hash), + descriptor.body_bytes, + &hash_bytes(&descriptor.body_hash)?, + ) + .await?; + Ok(()) + } + async fn descriptor( store: &Store, layout: &StoreLayout, From b435f43f96b9b44774dd38d3493015ee652f638f Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 03:48:42 -0700 Subject: [PATCH 02/17] perf(add): skip absent-index cache writes An unopened global index cannot supply per-chunk absence facts. Keep that state in the existing OnceCell instead of persisting a negative row for every chunk in a new file. Preserve initialized-index cache policy, cancellation, close ownership and push-side proof validation. --- crab/docs/design/add.md | 5 ++ crab/src/git/push.rs | 103 ++++++++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 20 deletions(-) diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index eda232407..d706ec4b7 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -224,6 +224,11 @@ repacking; push still revalidates every positive placement and origin proof. Cache failures, stale entries, and evictions are advisory misses; they cannot change the bytes selected for a push. +An uninitialized global chunk index is remembered only for the current add +session. It does not establish per-chunk absence, so it does not populate the +persistent negative cache. Existing index snapshots keep their normal cache +behavior; unavailable remote authority still leaves payload ownership local. + Push reads remote candidates in bounded pages under a shared deadline. It retains completed pages and local cache hits if the deadline expires or a later remote page fails; every retained placement still crosses normal proof diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index d9e7b97fe..c0a87fb0e 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -4494,17 +4494,23 @@ impl crab_staging::push_plan::ExistingChunkLookup for AddRemoteChunkClassifier { "remote add classifier failed: {error}" )), })?; + // Keep index absence in the session's OnceCell: persisting one miss + // per chunk makes cache writes scale with the entire new file. + let cache = self + .candidate_cache + .as_ref() + .filter(|_| matches!(self.remote_chunk_index.get(), Some(Some(_)))); let mut updates = Vec::with_capacity(misses.len()); for chunk_hash in misses { let candidate = fetched.get(&chunk_hash).copied(); - if self.candidate_cache.is_some() { + if cache.is_some() { updates.push((chunk_hash, candidate)); } if let Some(candidate) = candidate { candidates.insert(chunk_hash, candidate); } } - if let Some(cache) = &self.candidate_cache { + if let Some(cache) = cache { if let Err(error) = cache.memory_insert_batch(&updates) { warn!(error = %error, "add remote candidate memory cache update failed"); } @@ -22045,16 +22051,22 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn add_classifier_does_not_repeat_an_uninitialized_index_probe() { - let inner = Arc::new(object_store::memory::InMemory::new()); - let reads = Arc::new(Mutex::new(Vec::new())); - let recording_store: Arc = Arc::new(RecordingReadStore { - inner, - reads: Arc::clone(&reads), - }); - let store = Store::new(recording_store); - let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); + fn add_classifier_with_private_cache( + store: &Store, + router: &StoreLayout, + cache_root: &std::path::Path, + ) -> ( + AddRemoteChunkClassifier, + Arc, + ) { + let cache_root = cache_root.join("cache"); + crab_cache::ensure_private_cache_directory(&cache_root).expect("private cache root"); + let candidate_cache = Arc::new( + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( + &cache_root.join("candidates.sqlite"), + ) + .expect("open candidate cache"), + ); let pipeline = PushPipeline::new( PushConfig::default(), Vec::new(), @@ -22067,20 +22079,32 @@ mod tests { CancellationToken::new(), None, ); - pipeline.install_metadb(build_push_metadb_guard( - &store, - &router, - None, - &crate::core::config::MetaDbTomlConfig::default(), - true, - )); + let mut config = crate::core::config::MetaDbTomlConfig::default(); + config.chunk_index.local_path = Some(cache_root.join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard(store, router, None, &config, true)); let classifier = AddRemoteChunkClassifier { pipeline, remote_chunk_index: tokio::sync::OnceCell::new(), - candidate_cache: None, + candidate_cache: Some(Arc::clone(&candidate_cache)), candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), }; + (classifier, candidate_cache) + } + + #[tokio::test(flavor = "multi_thread")] + async fn add_classifier_keeps_uninitialized_index_absence_session_scoped() { + let cache_root = tempfile::tempdir().expect("private candidate cache"); + let inner = Arc::new(object_store::memory::InMemory::new()); + let reads = Arc::new(Mutex::new(Vec::new())); + let recording_store: Arc = Arc::new(RecordingReadStore { + inner, + reads: Arc::clone(&reads), + }); + let store = Store::new(recording_store); + let router = StoreLayout::new(store.clone(), "repo-add-uninitialized-index".to_owned()); + let (classifier, candidate_cache) = + add_classifier_with_private_cache(&store, &router, cache_root.path()); let first = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( &classifier, &[(MerkleHash::from([1; 32]), 4)], @@ -22118,6 +22142,45 @@ mod tests { assert!(matches!(cancelled, crab_staging::StagingError::Cancelled)); assert_eq!(manifest_probes(), initial_probes); classifier.close().await; + assert!( + candidate_cache + .load_persistent(&[MerkleHash::from([1; 32]), MerkleHash::from([2; 32])]) + .expect("inspect persisted candidates") + .is_empty(), + "an absent index must not write a per-chunk negative cache" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn add_classifier_persists_misses_from_an_initialized_index() { + let cache_root = tempfile::tempdir().expect("private candidate cache"); + let (store, router) = test_store_router("repo-add-initialized-index"); + let seed = build_push_metadb_guard( + &store, + &router, + None, + &crate::core::config::MetaDbTomlConfig::default(), + false, + ); + seed.chunk_index_system_keys().await.expect("seed manifest"); + seed.close().await.expect("close seed writer"); + let (classifier, candidate_cache) = + add_classifier_with_private_cache(&store, &router, cache_root.path()); + let hash = MerkleHash::from([4; 32]); + let lookup = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( + &classifier, + &[(hash, 4)], + ) + .await; + classifier.close().await; + lookup.expect("initialized-index lookup"); + assert_eq!( + candidate_cache + .load_persistent(&[hash]) + .expect("inspect persisted miss") + .get(&hash), + Some(&None) + ); } #[tokio::test(flavor = "multi_thread")] From 36c7a3da4237108a7d8db8f6b6692e2df896ce27 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 00:43:38 -0700 Subject: [PATCH 03/17] test(read): qualify captured catalog lifecycle Prove cold catalog pages survive checkpoint replacement and a real orphan sweep. Repair the async visibility test caller and carry forward the qualified staging-version fixture. Record the remaining shallow-date adapter failure without weakening its CLI smoke assertion. --- .../docs/architecture/indexed-remote-reads.md | 86 +++++++- .../e2e/run_add_commit_push_rustfs_smoke.py | 13 +- crab/src/cmd/gc/capsule_cleanup_tests.rs | 194 ++++++++++++++++++ 3 files changed, 282 insertions(+), 11 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 1b57dd83b..a13bda5eb 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -69,7 +69,9 @@ counts contain no raw paths, OIDs, endpoints, content or credentials; separate oracle evidence contains the paths/OIDs needed to prove correctness. `/usr/bin/time` supplies process CPU and peak RSS. Those cover opening and both -queries together; they are not per-query RSS. Index construction uses an explicit +queries together; they are not per-query RSS. Catalog qualifier totals also +include the complete inspection oracle, so they cannot establish selected-query +memory bounds. Index construction uses an explicit `build-indexes` operation, outside measured reads, with its own requests, bytes, CPU and peak RSS. Read-only qualifications do not trigger maintenance. @@ -2188,11 +2190,79 @@ two path heads, two leases, 65 payload records and 4,194,304 retained segment bytes remain. These receipts establish the oracle and staging component; they do not qualify the new integrated binary or catalog format. -Combined-source CLI compilation and runtime qualification are still pending. -Remaining catalog consumers include shared/protected preparation, pinned -hydration/mount lookup and the current-state file lookup opener. Complete +The combined release binary compiled and the focused RustFS smoke passed +163 commands and 84 checks, including exact retirement, committed restaging +before first push, both cold versions, force/ref updates and invalid roots. +The integrated native-first entropy 10-GiB run passes 75 commands and 57 checks: +all four pushes, exact retirement, eight cold historical SHA-256 checks, Git +integrity and remote fsck with zero errors or repairs. +Those receipts bind the earlier integrated source, not subsequent reader changes. + +Pinned hydration/mount lookup uses captured catalog pages in PR #581. PR #584 +extends this to current-state file lookup and shares ref ordering and captured +visibility with Git readers. Metadata tests pass (458), as do all 47 read capsule-protocol tests after +preserving corruption classification at the read boundary. CLI, runtime and +retained-root lifecycle checks are required before readiness. +A reproduced independent-ref fixture also requires complete inspection to +validate dependencies after assembling the visible set, while publication-time +delta validation remains strict. + +`qualify_remote ... catalog-current` measures the actual +current-state lookup session. The argument uses Xet `MerkleHash::hex()` encoding, +not the pointer wire format’s raw-byte hex. Hydration converts parsed pointer +bytes with `MerkleHash::from(pointer.file_hash)` before lookup; the qualifier +accepts that catalog identity directly. Opening, cold queries and warm queries are separate; the full +inspection oracle runs after measured queries and explicit session close. This +keeps unrelated catalog decoding out of cold-query preparation. Use a quiescent, +dedicated fixture so the post-query full oracle has the same visible authority. +Compare these +receipts across fixed small edits and increasing distinct historical chunks, +ref counts and uncompacted history; checkpoint row bounds alone are insufficient. + +Shared/protected preparation remains a separate integration boundary. Complete inspection remains appropriate for fsck, compaction and GC; foreground lookups -must not inherit those scans. Carry forward applicable CLI range/cancellation -contracts before final qualification. The complete integrated 10-GiB matrix, -history scaling, lifecycle/migration, fuller blame and representative S3 gates -remain open. +must not inherit those scans. CLI snapshot reads still install all Git packs; +carry forward the range/cancellation and cleanup contracts when removing that +cost. The final combined 10-GiB matrix, history scaling, lifecycle/migration, +fuller blame and representative S3 gates remain open. + +The GC regression passes with both existing cleanup tests (three tests total). +It captures a 20,000-file checkpoint catalog without a file query, replaces it, +and sweeps past the grace period. Actual orphan deletion is required; the old +reader then fetches cold pages and resolves duplicated file requests, while a +new session observes the replacement. Both checkpoints have a valid physical +source descriptor/body. This is an in-memory metadata lifecycle proof, separate +from payload hydration and concurrent real-store qualification. + +Current-state qualification against the four-version, two-file 10-GiB entropy +history passes with the updated reader. Opening uses 28 backend operations and +149,860 bytes. The duplicate-file cold query uses 497 range reads and 7,434,846 +bytes; the warm query repeats all 497 reads. Both results match full inspection, +operations drain and no writes occur. These are one local debug-profile sample, +not cloud latency or independent history-scaling proof. Repeated recipe metadata +reads are a measured remaining performance gap, not a successful warm-cache gate. +The first attempted receipt remains failed: it supplied pointer-wire hex where +the qualifier requires the distinct catalog `MerkleHash::hex()` representation. + +A differential run of the same qualifier binary against a real 4-MiB file passes: +five cold ranges / 3,755 bytes and zero warm backend operations. Combined with +the unchanged captured lookup and the existing 256-page / 8-MiB FIFO page cache, +this supports recipe-page eviction as the cause of repeated large-file warm +reads. A bounded cache of already-verified file bindings is the next candidate; +it must preserve the initial complete dependency proof, immutable capture, +ordered duplicates/misses, error retry and shared close/cancellation contracts. + +The current reader's minimal CLI/NFS check and debug build pass. The force-push +policy/visibility regression also passes after awaiting the now-async visibility +API in its test caller; all six policy cases retain their assertions. Eleven +Python fixture tests pass. The previously qualified staging-version assertion +from the combined branch is carried forward with exact post-push retirement. + +The complete current-source RustFS CLI smoke remains failed (152 commands, +61 checks). Its shallow-since case supplies an ISO date; the helper treats the +remote-helper option as an integer wire timestamp and rejects it before clone. +The harness still expects the earlier unsupported-option diagnostic. Git's +remote-helper boundary passes date text through, while wire fetch uses a +normalized timestamp. Fixing that adapter and qualifying dated boundaries +against native Git are required; changing the expected message alone would +hide the missing behavior. No full CLI E2E pass is claimed from this receipt. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index b79cbe9b9..79d217fea 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -2100,11 +2100,18 @@ def run_committed_restage_before_first_push_case(self) -> None: ["staging", "stats", "--json"], name=f"{case_name} staging ownership before push", ) - before_push = json.loads(self.read_stdout(before_push_record))["data"]["lifecycle"] + before_push_data = json.loads(self.read_stdout(before_push_record))["data"] + before_push = before_push_data["lifecycle"] + expected_versions = {first_pointer.file_hash, second_pointer.file_hash} + retained_versions = {entry["file_hash"] for entry in before_push_data["files"]} + # Native preparation and Git-clean retain independent handoff leases. + # Version coverage, followed by exact retirement, proves their lifetime; + # a fixed pre-push lease count cannot identify the preserved versions. self.check( f"{case_name}-retains-current-and-committed-history-owners", - before_push["path_heads"] == 2 - and before_push["path_leases"] == 2 + retained_versions == expected_versions + and before_push["recipes"] == len(expected_versions) + and before_push["path_leases"] >= len(expected_versions) and before_push["open_batches_without_publication"] == 0 and before_push["reclaimable_superseded_leases"] == 0 and before_push["reclaimable_files"] == 0, diff --git a/crab/src/cmd/gc/capsule_cleanup_tests.rs b/crab/src/cmd/gc/capsule_cleanup_tests.rs index a501550aa..9a3a7f618 100644 --- a/crab/src/cmd/gc/capsule_cleanup_tests.rs +++ b/crab/src/cmd/gc/capsule_cleanup_tests.rs @@ -19,6 +19,200 @@ use crate::core::error::CrabError; use crate::storage::StoreLayout; use crate::storage::store::Store; +#[tokio::test] +async fn captured_catalog_pages_survive_checkpoint_replacement_and_gc() { + use crab_metadata::capsule_protocol::{ + CapsuleGitPack, FileCatalogEntry, LayeredCheckpoint, PackLayer, PointerCatalog, + ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + }; + use crab_metadata::file_index_lookup::FileIndexLookupSession; + use crab_xet::hash::MerkleHash; + + let store = Store::new(Arc::new(InMemory::new())); + let router = StoreLayout::new(store.clone(), "org/catalog-gc".to_owned()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let base = + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let mut catalog = PointerCatalog::new(); + let shard = "a".repeat(64); + let xorb = "b".repeat(64); + catalog + .insert_xorb( + xorb.clone(), + XorbCatalogEntry::new( + 80, + "c".repeat(64), + vec![XorbChunkEntry::new("d".repeat(64), 16)], + ), + ) + .unwrap(); + catalog + .insert_shard(shard.clone(), ShardCatalogEntry::new(80, vec![xorb])) + .unwrap(); + for index in 0..20_000 { + catalog + .insert_file( + format!("{index:064x}"), + FileCatalogEntry::new(16, shard.clone()), + ) + .unwrap(); + } + let transaction = CapsuleTransaction::new( + base.record().digest(), + vec![CapsuleRefEdit::new( + "refs/heads/main", + None, + Some("2".repeat(40)), + None, + )], + ) + .unwrap(); + let capsule = Capsule::build(&transaction, Vec::new(), Vec::new()).unwrap(); + crab_write::capsule_protocol::publish(&layout, base, &transaction, &capsule) + .await + .unwrap(); + let view = crab_read::capsule_protocol::open_view( + &layout, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 1024 * 1024, + max_frontier_bytes: 8 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let layer = PackLayer::build( + &CapsuleGitPack::new( + Bytes::from_static(b"PACK"), + Bytes::from_static(b"index"), + Bytes::from_static(b"reverse"), + Bytes::from_static(b"locator"), + "3".repeat(40), + 1, + ) + .unwrap(), + ) + .unwrap(); + let source = layer.source_descriptor().unwrap(); + store + .put( + &layout.capsule_pack_layer_path(layer.hash()), + layer.bytes().clone(), + ) + .await + .unwrap(); + let checkpoint = LayeredCheckpoint::build( + view.root().root().generation(), + view.root().digest(), + vec![source.clone()], + catalog, + None, + ) + .unwrap(); + let first = crab_write::capsule_protocol::publish_ref_layered_checkpoint( + &layout, + view.root_snapshot().clone(), + &checkpoint, + view.refs().clone(), + view.peeled_refs().clone(), + view.visible_ref_transactions().clone(), + view.capsule_run_pointers().to_vec(), + ) + .await + .unwrap(); + let reads = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed_reads = reads.clone(); + let observed = layout + .store() + .clone() + .with_read_request_observer(Arc::new(move |_| { + observed_reads.fetch_add(1, Ordering::SeqCst); + })); + let captured = FileIndexLookupSession::open_for_storage(&observed, layout.repo_prefix()) + .await + .unwrap(); + // No file query has run: the old reader must fetch its catalog leaf after + // compaction and a real sweep, rather than succeed from a warmed page cache. + let next = LayeredCheckpoint::build( + first.record().root().generation(), + first.record().digest(), + vec![source], + PointerCatalog::new(), + None, + ) + .unwrap(); + let root = crab_write::capsule_protocol::publish_ref_layered_checkpoint( + &layout, + first.clone(), + &next, + first.record().root().refs().clone(), + first.record().root().peeled_refs().clone(), + first.record().root().compacted_ref_transactions().clone(), + Vec::new(), + ) + .await + .unwrap(); + let file = MerkleHash::from_hex(&format!("{:064x}", 19_999)).unwrap(); + let latest = FileIndexLookupSession::open_for_storage(layout.store(), layout.repo_prefix()) + .await + .unwrap(); + let latest_result = latest.lookup(&file).await; + latest.close().await.unwrap(); + assert_eq!(latest_result.unwrap(), None); + + let orphan = layout.capsule_checkpoint_path(&"e".repeat(64)); + store + .put(&orphan, Bytes::from_static(b"orphan")) + .await + .unwrap(); + let current = crab_read::capsule_protocol::open_view_from_root_with_control( + &layout, + root.clone(), + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: 8 * 1024 * 1024, + max_frontier_bytes: 8 * 1024 * 1024, + }, + ) + .await + .unwrap(); + let snapshot = current.git_snapshot().unwrap(); + let outcome = sweep_capsule_objects( + &GcArgs::default(), + &store, + &layout, + root.record().root(), + current.capsule_run_pointers(), + ¤t.state_digest(), + &snapshot.manifest, + &HashSet::new(), + &CancellationToken::new(), + SystemTime::now() + Duration::from_secs(2 * 3600), + Duration::from_secs(3600), + Instant::now(), + ) + .await + .unwrap(); + assert_eq!(outcome.packs_deleted, 1); + assert!(matches!( + store.head(&orphan).await, + Err(CrabError::NotFound { .. }) + )); + reads.store(0, Ordering::SeqCst); + let selected = captured.lookup_batch(&[file, file]).await; + captured.close().await.unwrap(); + assert!( + reads.load(Ordering::SeqCst) > 0, + "captured lookup must fetch cold pages after GC" + ); + let expected = MerkleHash::from_hex(&shard).unwrap(); + assert_eq!(selected.unwrap(), vec![Some(expected), Some(expected)]); +} + #[tokio::test] async fn failed_capsule_view_releases_root_and_sweep_fences() { let store = Store::new(Arc::new(InMemory::new())); From 9d1aefcbf684bfcda07ee99fb5d2bc06bde36a4e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 01:25:54 -0700 Subject: [PATCH 04/17] fix(git): honor dated shallow selections Remote helpers receive Git date expressions before wire normalization. Apply shallow selectors before requested-tip selection so an empty result cannot become a depth-one checkout. --- crab/docs/architecture/git-protocol-v2.md | 15 ++- .../e2e/run_add_commit_push_rustfs_smoke.py | 111 ++++++++------- crab/src/git/remote_helper.rs | 126 +++++++++++++++--- crates/crab-read/src/upload_pack.rs | 45 +++++-- crates/crab-remote-git/src/error.rs | 5 + 5 files changed, 217 insertions(+), 85 deletions(-) diff --git a/crab/docs/architecture/git-protocol-v2.md b/crab/docs/architecture/git-protocol-v2.md index e188bda4c..ad1d1efc2 100644 --- a/crab/docs/architecture/git-protocol-v2.md +++ b/crab/docs/architecture/git-protocol-v2.md @@ -55,9 +55,8 @@ the invocation rather than allowing every protocol globally: git -c protocol.crab.allow=always clone --recurse-submodules crab://bucket/team/repo ``` -"All commands" is not a support claim. Date/ref-exclusion shallow selectors -remain unimplemented; signed push certificates, push options intended for -server-side hooks, and remote archive service need separate protocol contracts. +"All commands" is not a support claim. Signed push certificates, push options +intended for server-side hooks, and remote archive service need separate protocol contracts. Signed commits/tags are Git objects and are distinct from signed push requests. PRs, CI execution, and mandatory server-side hooks cannot execute inside a passive bucket. Clients with unrestricted bucket write credentials can bypass @@ -76,7 +75,7 @@ select a v2 authority. The session supports: - `ls-refs` with ref prefixes, symrefs, peeled tags, unborn HEAD, and hidden refs; - `fetch` with wants, haves, done, tags, sideband/progress, shallow/deepen, - and `deepen-relative`; + `deepen-relative`, `deepen-since`, and `deepen-not`; - the bounded filter matrix: `blob:none`; `blob:limit=[kmg]`; `tree:`; `object:type={tag,commit,tree,blob}`; `sparse:oid=`; and repeated filters or percent-encoded `combine:` intersections; @@ -85,6 +84,12 @@ select a v2 authority. The session supports: required or assumed; - standard local Git pack installation on the v2 path. +The line helper normalizes `--shallow-since` date expressions through Git's +date parser before passing epoch seconds to the shared planner. Date and +ref-exclusion selectors apply to requested tips as well as their parents. +If no commits qualify, fetching fails before pack installation or shallow-file +publication; it does not substitute a depth-one checkout. + An empty manifest is a valid ref snapshot without a catalog. Empty clones and `ls-remote` complete through v2; clients requesting `unborn` receive the configured HEAD target unless hidden. Object wants against that empty snapshot @@ -229,7 +234,7 @@ takeover. ## Explicitly unsupported The first profile rejects stateful `connect`, `packfile-uris`, `object-info`, -`ref-in-want`, date/ref-exclusion shallow selectors, and other capabilities not +`ref-in-want`, and other capabilities not listed above. A v2 session cannot fall back to the line protocol after the terminal handoff; it reports the protocol failure instead. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 79d217fea..6af60724b 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -1097,53 +1097,70 @@ def run_atomic_rejection_case(self) -> None: def run_shallow_case(self) -> None: case_name = "shallow-history" repo, remote_url, _repo_prefix = self.prepare_git_repo(case_name) - first = self.commit_text(repo, "history.txt", "one\n", "one") - self.commit_text(repo, "history.txt", "two\n", "two") - self.commit_text(repo, "history.txt", "three\n", "three") - self.run_git(repo, ["push", "-u", "origin", "main"]) - - selector_clone = self.run_root / case_name / "shallow-since-clone" - selector_result = self.clone_git( - remote_url, - selector_clone, - name=f"{case_name} unsupported shallow-since clone", - extra_args=["--shallow-since", "2100-01-01T00:00:00Z"], - check=False, - ) - selector_stderr = Path(selector_result.stderr_log).read_text( - encoding="utf-8", errors="replace" - ) - self.check( - f"{case_name}-shallow-since-fails-explicitly", - selector_result.exit_code != 0 - and "deepen-since is not supported" in selector_stderr - and not (selector_clone / "history.txt").exists(), - { - "exit_code": selector_result.exit_code, - "stderr_log": selector_result.stderr_log, - }, - ) - exclude_clone = self.run_root / case_name / "shallow-exclude-clone" - exclude_result = self.clone_git( - remote_url, - exclude_clone, - name=f"{case_name} unsupported shallow-exclude clone", - extra_args=["--shallow-exclude", "main"], - check=False, - ) - exclude_stderr = Path(exclude_result.stderr_log).read_text( - encoding="utf-8", errors="replace" - ) - self.check( - f"{case_name}-shallow-exclude-fails-explicitly", - exclude_result.exit_code != 0 - and "deepen-not is not supported" in exclude_stderr - and not (exclude_clone / "history.txt").exists(), - { - "exit_code": exclude_result.exit_code, - "stderr_log": exclude_result.stderr_log, - }, - ) + commits = [] + for index, text in enumerate(("one", "two", "three"), start=1): + (repo / "history.txt").write_text(text + "\n", encoding="utf-8") + self.run_git(repo, ["add", "history.txt"]) + date = f"2023-11-{14 + index:02d}T22:13:20Z" + self.run_cmd( + f"{case_name} dated commit {text}", + ["git", "commit", "-m", text], + repo, + extra_env={"GIT_AUTHOR_DATE": date, "GIT_COMMITTER_DATE": date}, + ) + commits.append(self.rev_parse(repo, "HEAD")) + first = commits[0] + self.run_git(repo, ["tag", "archive", first]) + self.run_git(repo, ["push", "-u", "origin", "main", "--tags"]) + + # Match native upload-pack before asserting Crab's selector behavior. + # Fixed dates separate date parsing from commit-boundary selection. + selectors = ( + ("since", ["--shallow-since", "2023-11-16T22:13:20Z"]), + ("exclude", ["--shallow-exclude", "archive"]), + ("future", ["--shallow-since", "2030-01-01T00:00:00Z"]), + ("exclude-all", ["--shallow-exclude", "main"]), + ) + for selector, arguments in selectors: + expected_success = selector in ("since", "exclude") + transports = ( + ("native", repo.resolve().as_uri(), []), + ("crab-v2", remote_url, ["--config", "protocol.version=2"]), + ("crab-classic", remote_url, ["--config", "protocol.version=0"]), + ) + for transport, url, protocol in transports: + clone = self.run_root / case_name / f"{transport}-{selector}-clone" + result = self.clone_git( + url, clone, + name=f"{case_name} {transport} {selector} clone", + extra_args=[*protocol, *arguments], check=False, + ) + detail = {"exit_code": result.exit_code, "stderr_log": result.stderr_log} + if not expected_success: + diagnostic = Path(result.stderr_log).read_text( + encoding="utf-8", errors="replace" + ) + self.check( + f"{case_name}-{transport}-{selector}-rejects-empty-selection", + result.exit_code != 0 + and "no commits selected for shallow requests" in diagnostic + and not (clone / "history.txt").exists(), + detail, + ) + continue + self.check( + f"{case_name}-{transport}-{selector}-clone-succeeds", + result.exit_code == 0, detail, + ) + selected = self.read_stdout(self.run_git(clone, ["rev-list", "HEAD"])) + boundary = (clone / ".git" / "shallow").read_text().splitlines() + self.check( + f"{case_name}-{transport}-{selector}-exact-history-and-boundary", + selected.splitlines() == list(reversed(commits[1:])) + and boundary == [commits[1]] + and (clone / "history.txt").read_text() == "three\n", + {"commits": selected.splitlines(), "boundary": boundary}, + ) raw_fetch = self.run_root / case_name / "raw-fetch" raw_fetch.mkdir() diff --git a/crab/src/git/remote_helper.rs b/crab/src/git/remote_helper.rs index b95344ac3..107c4967a 100644 --- a/crab/src/git/remote_helper.rs +++ b/crab/src/git/remote_helper.rs @@ -1105,7 +1105,11 @@ where match cmd { HelperCommand::Option { key, value } => { - handle_option(&key, &value, options, writer).await?; + tokio::select! { + biased; + () = cancellation.cancelled() => return Err(CrabError::Cancelled), + result = handle_option(&key, &value, options, writer) => result?, + } } HelperCommand::Capabilities => { return Ok(Some(Batch::Capabilities)); @@ -1223,22 +1227,39 @@ async fn handle_option( } }, "deepen-since" => { - let timestamp = value - .parse::() - .map_err(|_| CrabError::Protocol(format!("invalid deepen-since value: {value}")))?; - if options - .fetch_options - .deepen_since - .replace(timestamp) - .is_some() - { + if options.fetch_options.deepen_since.is_some() { return Err(CrabError::Protocol( "duplicate deepen-since option".to_owned(), )); } + // Helpers receive Git date expressions; only upload-pack wire requests + // contain epoch seconds. Use Git's approxidate parser for the same grammar. + let date = decode_option_value(value)?; + let output = tokio::process::Command::new("git") + .arg("rev-parse") + .arg(format!("--since={date}")) + .stdin(std::process::Stdio::null()) + .kill_on_drop(true) + .output() + .await?; + if !output.status.success() { + return Err(CrabError::Protocol(format!( + "cannot parse deepen-since date: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + let timestamp = std::str::from_utf8(&output.stdout) + .ok() + .and_then(|line| line.trim().strip_prefix("--max-age=")) + .and_then(|seconds| seconds.parse::().ok()) + .ok_or_else(|| { + CrabError::Protocol("invalid Git deepen-since timestamp".to_owned()) + })?; + options.fetch_options.deepen_since = Some(timestamp); writer.write_all(b"ok\n").await?; } "deepen-not" => { + let value = decode_option_value(value)?; if value.is_empty() || value.bytes().any(|byte| byte.is_ascii_whitespace()) { return Err(CrabError::Protocol( "deepen-not requires one non-empty reference".to_owned(), @@ -1248,9 +1269,9 @@ async fn handle_option( .fetch_options .deepen_not .iter() - .any(|name| name == value) + .any(|name| name == &value) { - options.fetch_options.deepen_not.push(value.to_owned()); + options.fetch_options.deepen_not.push(value); } writer.write_all(b"ok\n").await?; } @@ -1356,19 +1377,25 @@ async fn handle_option( Ok(()) } -fn parse_ref_lease(value: &str) -> Result<(String, Option)> { +fn decode_option_value(value: &str) -> Result { if value.starts_with('"') && !value.ends_with('"') { return Err(CrabError::Protocol( - "unterminated quoted ref lease".to_owned(), + "unterminated quoted helper option".to_owned(), )); } let (decoded, consumed) = gix_quote::ansi_c::undo(value.as_bytes().into()) - .map_err(|error| CrabError::Protocol(format!("invalid quoted ref lease: {error}")))?; + .map_err(|error| CrabError::Protocol(format!("invalid quoted helper option: {error}")))?; if consumed != value.len() { - return Err(CrabError::Protocol("trailing data in ref lease".to_owned())); + return Err(CrabError::Protocol( + "trailing data in helper option".to_owned(), + )); } - let decoded = std::str::from_utf8(decoded.as_ref()) - .map_err(|error| CrabError::Protocol(format!("ref lease is not UTF-8: {error}")))?; + String::from_utf8(decoded.to_vec()) + .map_err(|error| CrabError::Protocol(format!("helper option is not UTF-8: {error}"))) +} + +fn parse_ref_lease(value: &str) -> Result<(String, Option)> { + let decoded = decode_option_value(value)?; let (name, oid) = decoded .rsplit_once(':') .ok_or_else(|| CrabError::Protocol("ref lease requires ref:oid".to_owned()))?; @@ -5732,7 +5759,7 @@ mod tests { let (source, store, router, commits, _tag) = capsule_history_fixture().await; let cutoff = String::from_utf8(run_git( source.path(), - &["show", "-s", "--format=%ct", &commits[1]], + &["show", "-s", "--format=%cI", &commits[1]], )) .expect("commit timestamp is utf8") .trim() @@ -5764,6 +5791,40 @@ mod tests { assert!(!git_object_exists(target.path(), &commits[0])); } + #[tokio::test] + async fn classic_capsule_empty_shallow_selectors_install_nothing() { + let (_source, store, router, commits, _tag) = capsule_history_fixture().await; + for selector in ["deepen-since 1893456000", "deepen-not main"] { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let git_dir = target.path().join(".git"); + let _guard = GitEnvCwdGuard::set(target.path(), &git_dir, target.path()); + let tip = commits.last().expect("history tip"); + let input = format!("option {selector}\nfetch {tip} refs/heads/main\n\n"); + let context = test_context( + store.clone(), + router.repo_prefix(), + target.path().join("push-state"), + ); + let (_, result) = + run_with_context(&input, context, tokio_util::sync::CancellationToken::new()).await; + let error = result.expect_err("empty shallow selection must fail"); + assert!( + error + .to_string() + .contains("no commits selected for shallow requests") + ); + assert_eq!( + ( + git_object_exists(target.path(), tip), + git_dir.join("shallow").exists() + ), + (false, false), + "{selector} must fail before pack installation or shallow publication" + ); + } + } + #[tokio::test] async fn capsule_promisor_fetch_installs_the_authorized_object() { let (store, router, blob) = capsule_promisor_fixture().await; @@ -7386,6 +7447,9 @@ mod tests { #[tokio::test] async fn shallow_selectors_are_accepted_without_degrading_to_full_fetch() { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let _guard = GitEnvCwdGuard::set(target.path(), &target.path().join(".git"), target.path()); let mut options = HelperOptions::default(); let mut output = Vec::new(); handle_option("deepen-since", "1700000000", &mut options, &mut output) @@ -7404,6 +7468,30 @@ mod tests { assert_eq!(String::from_utf8(output).unwrap(), "ok\nok\n"); } + #[tokio::test] + async fn option_shallow_since_normalizes_git_dates_and_quoting() { + let target = tempfile::tempdir().expect("target repository"); + run_git(target.path(), &["init", "-q"]); + let _guard = GitEnvCwdGuard::set(target.path(), &target.path().join(".git"), target.path()); + for date in [ + "1700000000", + "2023-11-14T22:13:20Z", + "2023-11-14T23:13:20+01:00", + "\"2023-11-14 22:13:20 +0000\"", + ] { + let mut options = HelperOptions::default(); + let mut output = Vec::new(); + handle_option("deepen-since", date, &mut options, &mut output) + .await + .unwrap(); + assert_eq!( + (options.fetch_options.deepen_since, output), + (Some(1_700_000_000), b"ok\n".to_vec()), + "{date}" + ); + } + } + #[tokio::test] async fn option_update_shallow_accepts_booleans_and_rejects_other_values() { assert_eq!(run("option update-shallow true\n").await, "ok\n"); diff --git a/crates/crab-read/src/upload_pack.rs b/crates/crab-read/src/upload_pack.rs index 5fb59eeb3..3bed3dc62 100644 --- a/crates/crab-read/src/upload_pack.rs +++ b/crates/crab-read/src/upload_pack.rs @@ -1128,6 +1128,7 @@ async fn plan_with_operation( } let mut selected = HashSet::new(); + let mut selected_commit = false; let mut object_ids = Vec::new(); let mut shallow = HashSet::new(); let mut unshallow = HashSet::new(); @@ -1141,7 +1142,11 @@ async fn plan_with_operation( let Some(item) = queue.pop_front() else { break; }; - if common_haves.contains(&item.oid) && request.deepen.is_none() { + if common_haves.contains(&item.oid) + && request.deepen.is_none() + && request.deepen_since.is_none() + && request.deepen_not.is_empty() + { continue; } // Tree entries prove that blobs are leaves. Rejecting a known omitted blob here @@ -1241,9 +1246,29 @@ async fn plan_with_operation( invariant: "batched upload-pack read is missing an object", })? }; - if excluded_commits.contains(&item.oid) && !roots.contains(&item.oid) { + // Wants authorize access but do not override shallow selectors. Applying + // these before selection also prevents an excluded tip's tree leaking in. + if excluded_commits.contains(&item.oid) { continue; } + if object.kind == gix_object::Kind::Commit + && let Some(since) = request.deepen_since + { + let commit = gix_object::CommitRef::from_bytes(&object.data, gix_hash::Kind::Sha1) + .map_err(|_| RemoteGitError::Corrupt { + stage: CorruptionStage::Commit, + })?; + let timestamp = commit + .time() + .map_err(|_| RemoteGitError::Corrupt { + stage: CorruptionStage::Commit, + })? + .seconds; + if timestamp < since { + continue; + } + } + selected_commit |= object.kind == gix_object::Kind::Commit; let encountered_client_have = matches!(visibility, VisibilitySource::TipBound { .. }) && client_haves.contains(&item.oid) && object.kind == gix_object::Kind::Commit; @@ -1278,6 +1303,10 @@ async fn plan_with_operation( } } + if (request.deepen_since.is_some() || !request.deepen_not.is_empty()) && !selected_commit { + return Err(RemoteGitError::EmptyShallowSelection); + } + if request.include_tags { for reference in &repository.refs().entries { if !reference.name.starts_with("refs/tags/") @@ -2396,18 +2425,6 @@ async fn enqueue_children( maximum_objects, deduplicate_by_oid, )?; - if let Some(since) = request.deepen_since { - let timestamp = commit - .time() - .map_err(|_| RemoteGitError::Corrupt { - stage: CorruptionStage::Commit, - })? - .seconds; - if timestamp < since { - shallow.insert(item.oid); - return Ok(()); - } - } match item.depth { TraversalDepth::RelativeBoundary => { if existing_shallow.contains(&item.oid) { diff --git a/crates/crab-remote-git/src/error.rs b/crates/crab-remote-git/src/error.rs index 15bcdeb54..0e40a9f61 100644 --- a/crates/crab-remote-git/src/error.rs +++ b/crates/crab-remote-git/src/error.rs @@ -237,6 +237,10 @@ pub enum Error { #[error("repository is empty")] EmptyRepository, + /// Date and ref exclusions selected no commits for a shallow fetch. + #[error("no commits selected for shallow requests")] + EmptyShallowSelection, + /// Locator publication has not reached the required manifest generation. #[error( "repository object locator is not ready: observed generation {observed:?}, required {required}" @@ -516,6 +520,7 @@ impl Error { | Self::InvalidPath { .. } | Self::InvalidCursor { .. } | Self::InvalidLimit { .. } + | Self::EmptyShallowSelection | Self::Revision { reason: RevisionError::InvalidReference From d3d749a8e9f7bb41734d45add8de35cd93e7e661 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 01:37:48 -0700 Subject: [PATCH 05/17] test(git): keep raw SHA probe unadvertised Advertise an archive side tip instead of the ancestor under raw-fetch admission test. Its exclusion closure still contains that ancestor, preserving both selector and admission assertions. --- .../scripts/e2e/run_add_commit_push_rustfs_smoke.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 6af60724b..ff9097936 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -1110,7 +1110,18 @@ def run_shallow_case(self) -> None: ) commits.append(self.rev_parse(repo, "HEAD")) first = commits[0] - self.run_git(repo, ["tag", "archive", first]) + # Exclude the first commit through a side tip, keeping that ancestor + # unadvertised for the raw-SHA policy check below. + archive = self.read_stdout(self.run_cmd( + f"{case_name} archive side tip", + ["git", "commit-tree", f"{first}^{{tree}}", "-p", first, "-m", "archive"], + repo, + extra_env={ + "GIT_AUTHOR_DATE": "2023-11-15T23:13:20Z", + "GIT_COMMITTER_DATE": "2023-11-15T23:13:20Z", + }, + )) + self.run_git(repo, ["tag", "archive", archive]) self.run_git(repo, ["push", "-u", "origin", "main", "--tags"]) # Match native upload-pack before asserting Crab's selector behavior. From 7df03bac97a6995e02b15e7a37de4bad8597257e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 02:00:04 -0700 Subject: [PATCH 06/17] test(git): fault current capsule payloads V2 stores Git packs and indexes inside capsule runs. Target those ranges and compare authority bytes directly: damaged discovery controls cannot prove unchanged refs through ls-remote. --- crab/docs/architecture/git-protocol-v2.md | 7 + .../e2e/run_add_commit_push_rustfs_smoke.py | 190 ++++++++---------- .../e2e/test_add_commit_push_source.py | 31 +++ 3 files changed, 121 insertions(+), 107 deletions(-) diff --git a/crab/docs/architecture/git-protocol-v2.md b/crab/docs/architecture/git-protocol-v2.md index ad1d1efc2..24d6f8bcb 100644 --- a/crab/docs/architecture/git-protocol-v2.md +++ b/crab/docs/architecture/git-protocol-v2.md @@ -254,6 +254,13 @@ filter reads must leave canonical object counts and bytes unchanged; generated response-pack cache and coordination objects are accounted for separately. This is not a claim that first-read repair works with read-only object-store credentials. +The add/commit/push RustFS smoke exercises the current physical fault boundary: +loss of a published capsule and corruption confined to its embedded Git pack +or index range. Each rejection probe uses an empty cache and requires no local +HEAD or checked-out payload. It compares the remote root and ref-head bytes +before and after the fault; discovery can itself reject a damaged capsule, +so successful `ls-remote` is not the mutation oracle. + Direct pushes and protected/service publication paths publish visibility before the corresponding ref or manifest commit. Protected receive extracts the closure from the already-verified materialization ODB before that workspace diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index ff9097936..5e9f70b77 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -10,7 +10,7 @@ clone, hydration, and byte identity. A separate ordinary-Git matrix proves missing/corrupt metadata-root handling, update/fetch, deletion, force and non-fast-forward behavior, atomic rejection, tag following, shallow history, -immutable-pack failures, and concurrent push serialization. +capsule loss, embedded pack/index corruption, and concurrent push serialization. With ``--source``, run only the two managed-file workflows on disposable clones of a read-only real repository. Use a run root on the workspace volume @@ -205,6 +205,31 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def capsule_member_range(body: bytes, kind: str) -> tuple[int, int]: + """Locate one pack/index fault inside the synthetic single-member run.""" + if len(body) < 48 or body[-8:] != b"CRBRUN01" or kind not in ("pack", "index"): + raise SmokeError("fault fixture is not a capsule run with a pack/index target") + footer_length = int.from_bytes(body[-48:-40], "big") + footer_start = len(body) - 48 - footer_length + if not 0 < footer_length <= 8 * 1024 * 1024 or footer_start < 0: + raise SmokeError("fault fixture footer is outside its container") + try: + footer = json.loads(body[footer_start:-48]) + members = footer["git_packs"] + if footer["version"] != 4 or len(members) != 1: + raise SmokeError("fault fixture requires one current-format pack member") + offset, length = members[0][kind]["offset"], members[0][kind]["length"] + except (KeyError, TypeError, ValueError) as exc: + raise SmokeError("fault fixture has no usable pack member") from exc + if (type(offset) is not int or type(length) is not int + or offset < 0 or length < 4 or offset + length > footer_start): + raise SmokeError("fault fixture range overlaps control bytes or escapes its container") + magic = b"PACK" if kind == "pack" else b"\xfftOc" + if body[offset:offset + 4] != magic: + raise SmokeError("fault fixture member does not contain the selected Git format") + return offset, length + + def parse_pointer(blob: str) -> PointerRecord: raw = blob.encode("utf-8") if len(raw) > 256: @@ -1314,122 +1339,74 @@ def run_corrupt_root_case(self) -> None: {"stderr_log": clone.stderr_log}, ) - def run_immutable_object_failure_case(self, object_kind: str) -> None: - case_name = f"missing-{object_kind}" - repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) - remote_tip = self.commit_text(repo, "payload.txt", "immutable\n", "immutable") - self.run_git(repo, ["push", "-u", "origin", "main"]) - keys = self.list_keys(repo_prefix) - suffix = f".{object_kind}" - candidates = sorted(key for key in keys if key.endswith(suffix)) - self.check( - f"{case_name}-fixture-object-found", - bool(candidates), - {"suffix": suffix, "keys": sorted(keys)}, - ) - target_key = candidates[0] - status, _, body = self.signed_s3_request("DELETE", target_key) - self.check( - f"{case_name}-fixture-object-deleted", - status in (200, 204), - { - "status": status, - "key": target_key, - "body": body.decode("utf-8", errors="replace")[:200], - }, - ) - - clone_dir = self.run_root / case_name / "clone" - clone = self.clone_git( - remote_url, clone_dir, name=f"{case_name} git clone", check=False - ) - head = self.run_git( - self.run_root, - ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], - name=f"{case_name} verify no local ref update", - check=False, - ) - self.check( - f"{case_name}-clone-fails-before-checkout", - clone.exit_code != 0 - and head.exit_code != 0 - and not (clone_dir / "payload.txt").exists(), - { - "clone_exit_code": clone.exit_code, - "head_exit_code": head.exit_code, - "target_key": target_key, - }, - ) - refs = self.ls_remote(remote_url, name=f"{case_name} refs remain coherent") - self.check( - f"{case_name}-remote-ref-remains-original", - refs.get("refs/heads/main") == remote_tip, - {"expected": remote_tip, "actual": refs.get("refs/heads/main")}, - ) - - def run_corrupt_index_failure_case(self) -> None: - case_name = "corrupt-idx" + def run_capsule_failure_case(self, fault: str) -> None: + case_name = "missing-capsule" if fault == "missing" else f"corrupt-{fault}" repo, remote_url, repo_prefix = self.prepare_git_repo(case_name) remote_tip = self.commit_text(repo, "payload.txt", "immutable\n", "immutable") self.run_git(repo, ["push", "-u", "origin", "main"]) - candidates = sorted( - key for key in self.list_keys(repo_prefix) if key.endswith(".idx") - ) self.check( - f"{case_name}-fixture-object-found", - bool(candidates), - {"suffix": ".idx", "keys": candidates}, + f"{case_name}-healthy-tip-before-fault", + self.ls_remote(remote_url, name=f"{case_name} healthy refs").get("refs/heads/main") == remote_tip, ) + keys = self.list_keys(repo_prefix) + candidates = sorted(key for key in keys if key.startswith(f"{repo_prefix}/v2/capsules/")) + self.check(f"{case_name}-fixture-object-found", len(candidates) == 1, {"keys": candidates}) target_key = candidates[0] - status, _, body = self.signed_s3_request( - "PUT", target_key, body=b"not a valid git pack index\n" - ) - self.check( - f"{case_name}-fixture-object-corrupted", - status in (200, 201), - { - "status": status, - "key": target_key, - "body": body.decode("utf-8", errors="replace")[:200], - }, - ) - + authority = {} + for key in sorted(key for key in keys if key == f"{repo_prefix}/v2/root" or key.startswith(f"{repo_prefix}/v2/refs/")): + status, _, value = self.signed_s3_request("GET", key) + self.check(f"{case_name}-authority-readable-before-fault", status == 200, {"key": key}) + authority[key] = value + status, headers, original = self.signed_s3_request("GET", target_key) + self.check(f"{case_name}-fixture-body-readable", status == 200 and len(original) < 1024 * 1024) + if fault == "missing": + status, _, _ = self.signed_s3_request("DELETE", target_key) + absent, _, _ = self.signed_s3_request("GET", target_key) + self.check(f"{case_name}-fixture-object-deleted", status in (200, 204) and absent == 404) + else: + offset, length = capsule_member_range(original, fault) + damaged = bytearray(original) + damaged[offset] ^= 1 + status, _, _ = self.signed_s3_request( + "PUT", target_key, body=bytes(damaged), extra_headers={"if-match": headers["etag"]}, + ) + read_status, _, stored = self.signed_s3_request("GET", target_key) + self.check( + f"{case_name}-fixture-member-corrupted", + status in (200, 201) and read_status == 200 and stored == bytes(damaged), + {"key": target_key, "member_offset": offset, "member_length": length}, + ) clone_dir = self.run_root / case_name / "clone" - clone = self.clone_git( - remote_url, clone_dir, name=f"{case_name} git clone", check=False + clone_cache = self.run_root / case_name / "cold-cache" + self.check(f"{case_name}-clone-cache-starts-empty", not clone_cache.exists()) + clone = self.run_cmd( + f"{case_name} git clone", ["git", "clone", remote_url, str(clone_dir)], self.run_root, + check=False, extra_env={"CRAB_CACHE_DIR": str(clone_cache)}, ) head = self.run_git( - self.run_root, - ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], - name=f"{case_name} verify no local ref update", - check=False, - ) - clone_stderr = Path(clone.stderr_log).read_text( - encoding="utf-8", errors="replace" + self.run_root, ["-C", str(clone_dir), "rev-parse", "--verify", "HEAD"], + name=f"{case_name} verify no local ref update", check=False, ) self.check( f"{case_name}-clone-fails-before-checkout", - clone.exit_code != 0 - and head.exit_code != 0 - and not (clone_dir / "payload.txt").exists(), - { - "clone_exit_code": clone.exit_code, - "head_exit_code": head.exit_code, - "target_key": target_key, - }, - ) - self.check( - f"{case_name}-clone-surfaces-integrity-error", - "CRAB-E0020" in clone_stderr - or "corrupt object" in clone_stderr.lower(), - {"stderr_log": clone.stderr_log}, - ) - refs = self.ls_remote(remote_url, name=f"{case_name} refs remain coherent") - self.check( - f"{case_name}-remote-ref-remains-original", - refs.get("refs/heads/main") == remote_tip, - {"expected": remote_tip, "actual": refs.get("refs/heads/main")}, + clone.exit_code != 0 and head.exit_code != 0 and not (clone_dir / "payload.txt").exists(), + {"clone_exit_code": clone.exit_code, "head_exit_code": head.exit_code, "target_key": target_key}, ) + if fault != "missing": + diagnostic = Path(clone.stderr_log).read_text(encoding="utf-8", errors="replace") + self.check( + f"{case_name}-clone-surfaces-integrity-error", + "CRAB-E0020" in diagnostic or "corrupt object" in diagnostic.lower(), + {"stderr_log": clone.stderr_log}, + ) + # Payloads share their physical object with discovery controls. Compare + # authority bytes directly; failed discovery cannot prove unchanged refs. + after = {} + for key in authority: + status, _, value = self.signed_s3_request("GET", key) + self.check(f"{case_name}-authority-readable-after-fault", status == 200, {"key": key}) + after[key] = value + self.check(f"{case_name}-remote-authority-unchanged", after == authority) def run_concurrent_push_case(self) -> None: case_name = "concurrent-cas" @@ -2312,9 +2289,8 @@ def run(self) -> None: self.run_ref_update_delete_force_and_tag_case() self.run_atomic_rejection_case() self.run_shallow_case() - self.run_immutable_object_failure_case("pack") - self.run_immutable_object_failure_case("idx") - self.run_corrupt_index_failure_case() + for fault in ("missing", "pack", "index"): + self.run_capsule_failure_case(fault) self.run_concurrent_push_case() self.run_committed_restage_before_first_push_case() self.run_partial_overlap_case() diff --git a/crab/scripts/e2e/test_add_commit_push_source.py b/crab/scripts/e2e/test_add_commit_push_source.py index 5b9ef5052..4f2c42e1a 100644 --- a/crab/scripts/e2e/test_add_commit_push_source.py +++ b/crab/scripts/e2e/test_add_commit_push_source.py @@ -214,5 +214,36 @@ def failed_run(smoke: SMOKE.AddCommitPushSmoke) -> None: self.assertNotIn(secret, report_text + stderr.getvalue()) +class CapsuleFaultRangeTests(unittest.TestCase): + @staticmethod + def fixture(*, offset: int = 8, length: int = 9, version: int = 4) -> bytes: + body = b"PACKbody\xfftOcindex" + footer = json.dumps({ + "version": version, + "git_packs": [{ + "pack": {"offset": 0, "length": 8}, + "index": {"offset": offset, "length": length}, + }], + }).encode() + return body + footer + len(footer).to_bytes(8, "big") + bytes(32) + b"CRBRUN01" + + def test_targets_the_absolute_pack_and_index_ranges(self) -> None: + body = self.fixture() + for kind, expected in (("pack", (0, 8)), ("index", (8, 9))): + with self.subTest(kind=kind): + self.assertEqual(SMOKE.capsule_member_range(body, kind), expected) + + def test_refuses_ranges_that_could_damage_unselected_controls(self) -> None: + for arguments in ({"offset": -1}, {"offset": 8, "length": 10}, {"offset": True}, {"length": 3}): + with self.subTest(arguments=arguments), self.assertRaises(SMOKE.SmokeError): + SMOKE.capsule_member_range(self.fixture(**arguments), "index") + + def test_refuses_other_container_formats_and_malformed_trailers(self) -> None: + body = self.fixture() + for malformed in (body[:-1], body[:20], self.fixture(version=3), body[:-48] + bytes(48)): + with self.subTest(size=len(malformed)), self.assertRaises(SMOKE.SmokeError): + SMOKE.capsule_member_range(malformed, "pack") + + if __name__ == "__main__": unittest.main() From 7009fc4e52015d8ef50397c2c604d980c65e74d6 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 02:32:53 -0700 Subject: [PATCH 07/17] fix(storage): retain immutable object identity Keep the provider token beside the bytes it identifies. Preserve both ETag and version from the PUT or verified GET so origin proofs need no later HEAD. Update all consumers of this unreleased Rust outcome shape; stored receipt formats remain unchanged. --- crab/src/git/xet_publication.rs | 6 +- crab/src/optimize/xorbs/executor.rs | 6 +- crates/crab-remote/src/prepare.rs | 8 +- crates/crab-storage/README.md | 4 + crates/crab-storage/src/store.rs | 122 ++++++++++++++++++++++++---- 5 files changed, 122 insertions(+), 24 deletions(-) diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 3e13c2392..4029b8c41 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -719,7 +719,7 @@ async fn publish_xorb_candidate( .create_or_read_immutable(&path, bytes, MAX_XORB_SIZE as u64) .await? { - crab_storage::ImmutableCreateOutcome::Created => { + crab_storage::ImmutableCreateOutcome::Created(_) => { if let (Some(cache), Some(bytes)) = (caching_store, cache_bytes) { warm_published_cache_object( cache, @@ -731,7 +731,9 @@ async fn publish_xorb_candidate( } Ok((local_entry, local_placements, true)) } - crab_storage::ImmutableCreateOutcome::Existing(existing) => { + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => { let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? .ok_or_else(|| CrabError::CorruptObject { path: path.to_string(), diff --git a/crab/src/optimize/xorbs/executor.rs b/crab/src/optimize/xorbs/executor.rs index cd51c621c..3c7115ccc 100644 --- a/crab/src/optimize/xorbs/executor.rs +++ b/crab/src/optimize/xorbs/executor.rs @@ -476,8 +476,10 @@ async fn upload_destination( () = cancel.cancelled() => return Err(CrabError::Cancelled), }; match publication { - crab_storage::ImmutableCreateOutcome::Created => {} - crab_storage::ImmutableCreateOutcome::Existing(existing) => { + crab_storage::ImmutableCreateOutcome::Created(_) => {} + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => { let parser = XorbParser::parse(existing).map_err(CrabError::from)?; if parser.hash() != destination_hash { return Err(CrabError::CorruptObject { diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 3c526c14e..4d8ba63bf 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -1147,10 +1147,10 @@ async fn upload_capsule_content( ) .await? { - crab_storage::ImmutableCreateOutcome::Created => local_entry.clone(), - crab_storage::ImmutableCreateOutcome::Existing(existing) => { - catalog_xorb_from_existing(&xorb_hash, local_entry, existing)? - } + crab_storage::ImmutableCreateOutcome::Created(_) => local_entry.clone(), + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, .. + } => catalog_xorb_from_existing(&xorb_hash, local_entry, existing)?, }; delta.insert_xorb(xorb_hash, entry)?; } diff --git a/crates/crab-storage/README.md b/crates/crab-storage/README.md index 6291ec7c5..f82e41126 100644 --- a/crates/crab-storage/README.md +++ b/crates/crab-storage/README.md @@ -45,6 +45,10 @@ writes, bounded reads, byte/request observers, and provider-neutral `StorageError` values. `put_if_absent` distinguishes a new immutable object from an identical retry without a preceding HEAD, and streamed size/hash verification uses the GET response metadata instead of a separate HEAD. +`create_or_read_immutable` retains both ETag and version from the successful +PUT, verified readback, or occupied-key GET alongside its outcome. Origin proof +consumers must carry that identity with the verified bytes; a later HEAD may +observe another object version. Occupied-key bytes still require caller validation. `Store::verify_bounded` combines that content-hash contract with pre-body size admission and streamed length checks. It uses one GET for a successful read; the limit applies to buffered encoded bytes, not the caller's decoded objects. diff --git a/crates/crab-storage/src/store.rs b/crates/crab-storage/src/store.rs index 77583e604..edfbb3d96 100644 --- a/crates/crab-storage/src/store.rs +++ b/crates/crab-storage/src/store.rs @@ -60,10 +60,10 @@ pub enum ImmutableWriteVerification { /// Result of a create-only immutable write whose occupied key is returned to the caller. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ImmutableCreateOutcome { - /// This call created and verified the requested bytes. - Created, - /// The key was already occupied; these bounded bytes remain untrusted. - Existing(Bytes), + /// This call created and verified the bytes; identity belongs to that PUT or verified readback. + Created(ETag), + /// The key was occupied; identity belongs to this bounded GET, whose bytes remain untrusted. + Existing { bytes: Bytes, identity: ETag }, } /// Bounded-memory byte stream returned by object reads. @@ -727,11 +727,11 @@ impl Store { }; let payload = upload_payload(bytes); match self.inner.put_opts(&path, payload, options).await { - Ok(_) => Ok(true), + Ok(result) => Ok(Some(ETag::from(result))), Err(error) => { let mapped = map_object_store_error(error, path.as_ref()); if matches!(mapped, StorageError::StateConflict { .. }) { - Ok(false) + Ok(None) } else { Err(mapped) } @@ -740,13 +740,13 @@ impl Store { } }) .await?; - if !created { - let (existing, _) = self.get_with_etag_bounded(path, max_existing_bytes).await?; - return Ok(ImmutableCreateOutcome::Existing(existing)); - } + let Some(mut identity) = created else { + let (bytes, identity) = self.get_with_etag_bounded(path, max_existing_bytes).await?; + return Ok(ImmutableCreateOutcome::Existing { bytes, identity }); + }; if self.immutable_write_verification == ImmutableWriteVerification::ReadbackRequired { let maximum = u64::try_from(bytes.len()).unwrap_or(u64::MAX); - let (stored, _) = self.get_with_etag_bounded(path, maximum).await?; + let (stored, observed_identity) = self.get_with_etag_bounded(path, maximum).await?; let actual_hash = *blake3::hash(&stored).as_bytes(); if actual_hash != expected_hash { return Err(StorageError::CorruptObject { @@ -758,8 +758,11 @@ impl Store { ), }); } + // Bind proof to the object version whose bytes were verified. A + // later HEAD may observe a replacement with a different encoding. + identity = observed_identity; } - Ok(ImmutableCreateOutcome::Created) + Ok(ImmutableCreateOutcome::Created(identity)) } /// Writes `bytes` at `path` iff nothing exists there yet. @@ -3763,6 +3766,7 @@ mod tests { Body(Vec), BodyError, Range(Range), + DistinctIdentities, } #[derive(Debug)] @@ -3785,7 +3789,12 @@ mod tests { payload: PutPayload, options: PutOptions, ) -> object_store::Result { - self.inner.put_opts(location, payload, options).await + let mut result = self.inner.put_opts(location, payload, options).await?; + if matches!(self.fault, ReadFault::DistinctIdentities) { + result.e_tag = Some("put-etag".to_owned()); + result.version = Some("put-version".to_owned()); + } + Ok(result) } async fn put_multipart_opts( @@ -3840,6 +3849,10 @@ mod tests { } ReadFault::Range(range) => response.range = range.clone(), ReadFault::Transient(_) => {} + ReadFault::DistinctIdentities => { + response.meta.e_tag = Some("get-etag".to_owned()); + response.meta.version = Some("get-version".to_owned()); + } } } Ok(response) @@ -4194,7 +4207,78 @@ mod tests { .await .unwrap(); - assert_eq!(outcome, ImmutableCreateOutcome::Existing(existing)); + let observed = store.head(&path).await.unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Existing { + bytes: existing, + identity: ETag { + e_tag: observed.e_tag, + version: observed.version + }, + } + ); + } + + #[tokio::test] + async fn logical_immutable_identity_belongs_to_the_verified_operation() { + for (existing, verification) in [ + (false, ImmutableWriteVerification::Sha256Checksum), + (false, ImmutableWriteVerification::ReadbackRequired), + (true, ImmutableWriteVerification::ReadbackRequired), + ] { + let inner = Arc::new(InMemory::new()); + let path = Path::from("blobs/operation-identity"); + let bytes = Bytes::from_static(b"candidate"); + if existing { + inner.put(&path, bytes.clone().into()).await.unwrap(); + } + let store = memory_store_with_inner(Arc::new(ReadFaultStore { + inner, + fault: ReadFault::DistinctIdentities, + })) + .with_immutable_write_verification(verification); + let outcome = store + .create_or_read_immutable(&path, bytes.clone(), 1024) + .await + .unwrap(); + let observed = + if existing || verification == ImmutableWriteVerification::ReadbackRequired { + "get" + } else { + "put" + }; + let identity = ETag { + e_tag: Some(format!("{observed}-etag")), + version: Some(format!("{observed}-version")), + }; + let expected = if existing { + ImmutableCreateOutcome::Existing { bytes, identity } + } else { + ImmutableCreateOutcome::Created(identity) + }; + assert_eq!( + outcome, expected, + "existing={existing}, verification={verification:?}" + ); + } + } + + #[tokio::test] + async fn logical_immutable_create_withholds_identity_when_readback_is_corrupt() { + let store = memory_store_with_inner(Arc::new(ReadFaultStore { + inner: Arc::new(InMemory::new()), + fault: ReadFault::Body(vec![Bytes::from_static(b"changed!!")]), + })); + let error = store + .create_or_read_immutable( + &Path::from("blobs/corrupt-readback"), + Bytes::from_static(b"candidate"), + 1024, + ) + .await + .expect_err("invalid bytes cannot acquire a verified identity"); + assert!(matches!(error, StorageError::CorruptObject { .. })); } #[tokio::test] @@ -4232,8 +4316,14 @@ mod tests { .await .unwrap(); - assert_eq!(outcome, ImmutableCreateOutcome::Created); let result = inner.get(&path).await.unwrap(); + assert_eq!( + outcome, + ImmutableCreateOutcome::Created(ETag { + e_tag: result.meta.e_tag.clone(), + version: result.meta.version.clone(), + }) + ); assert_eq!( result.attributes.get(&Attribute::StorageClass), Some(&"STANDARD_IA".to_owned().into()) @@ -4269,7 +4359,7 @@ mod tests { .await .unwrap(); - assert!(matches!(outcome, ImmutableCreateOutcome::Existing(_))); + assert!(matches!(outcome, ImmutableCreateOutcome::Existing { .. })); let result = inner.get(&path).await.unwrap(); assert_eq!( result.attributes.get(&Attribute::StorageClass), From cc66a7f6f0bfddb423731453e0643cdb705f4e81 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 03:56:49 -0700 Subject: [PATCH 08/17] feat(index): publish capsule chunk candidates Bind shared chunk hints to authenticated source shards and exact immutable object identities. Keep capsule provenance separate from released manifest receipt keys. Draft: exact staged-proof revalidation and current-source real-store qualification remain incomplete. --- crab/docs/design/add.md | 16 + crab/src/git/capsule_push.rs | 82 ++++- crab/src/git/push.rs | 282 ++++++++++++++++-- crab/src/git/xet_publication.rs | 120 +++++--- crab/src/metadata/metadb/mod.rs | 3 +- .../src/metadata/metadb/stores/chunk_index.rs | 54 ++++ .../crab-auth-server/src/receive/capsule.rs | 93 ++++-- .../src/receive/capsule_dependencies.rs | 108 +++++-- .../src/capsule_protocol/chunk_candidates.rs | 171 +++++++++++ .../chunk_candidates/tests.rs | 147 +++++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 7 + .../capsule_protocol/pointer/paged/reader.rs | 54 +++- crates/crab-metadata/src/key_codec.rs | 22 ++ crates/crab-metadata/src/remote_index.rs | 183 +++++++++++- .../src/remote_index/capsule_tests.rs | 86 ++++++ 15 files changed, 1297 insertions(+), 131 deletions(-) create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs create mode 100644 crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs create mode 100644 crates/crab-metadata/src/remote_index/capsule_tests.rs diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index d706ec4b7..94426f02c 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -203,6 +203,22 @@ SlateDB contains only committed, origin-bound receipts and is updated after a successful push CAS; it never stores add claims, local paths, or pending uploads. +Capsule pushes publish chunk hints into the same shared database after ref +visibility. Their head/source keys are separate from the released manifest +receipt keys. A source assertion carries the whole-xorb catalog digest and an +origin receipt captured from the verified PUT or GET; the receipt's payload +digest remains distinct. Readers batch requested chunks, authenticate the +source repository's current paged catalog, check each exact chunk position and +size, then verify canonical origin identity. An absent source or changed +physical descriptor cannot turn an index hint into authority. + +Direct publication and protected receive use the same bounded candidate +encoder/writer. Protected promotion carries the canonical operation identity; +client staging-prefix writes do not confer canonical origin proof. Every index +writer is explicitly closed, including batch failures. A post-commit index +failure leaves a successful ref publication visible and reports repair work; +local packing remains available for future lookup misses. + Concurrent add lookups own their receipt snapshot and validation result across remote awaits. They share keyed advisory caches, not the push pipeline's mutable per-push proof accumulator; otherwise one file can erase another's diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index 5d6c9e128..e0fbb8cc6 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -396,6 +396,7 @@ async fn run_inner( .collect::>() }; let mut xet_stats = super::xet_publication::XetPublicationStats::default(); + let mut pointer_delta = crab_metadata::capsule_protocol::PointerCatalog::new(); let publication: Result>> = async { // LFS bytes share the ref visibility boundary with Git and Xet data. @@ -409,19 +410,21 @@ async fn run_inner( cancel, ) .await?; - let (pointer_delta, prepared_xet_stats) = super::xet_publication::prepare_delta( - &layout, - &base, - &prepared.pointers, - staging, - caching_store, - metrics, - config.upload_concurrency, - config.protected_push.is_none(), - cancel, - ) - .await?; + let (prepared_pointer_delta, prepared_xet_stats) = + super::xet_publication::prepare_delta( + &layout, + &base, + &prepared.pointers, + staging, + caching_store, + metrics, + config.upload_concurrency, + config.protected_push.is_none(), + cancel, + ) + .await?; xet_stats = prepared_xet_stats; + pointer_delta = prepared_pointer_delta; if config.protected_push.is_none() && let Some(replication) = config.active_active_replication.as_ref() { @@ -477,6 +480,7 @@ async fn run_inner( } let result = if changes_namespace { let commit_layout = layout.clone(); + let commit_pointer_delta = &pointer_delta; let commit_push_ref_head_bases = push_ref_head_bases.clone(); crab_write::with_ref_namespaces_wait( layout.store(), @@ -501,7 +505,7 @@ async fn run_inner( base, &transaction, &capsule, - &pointer_delta, + commit_pointer_delta, &lfs_objects, &commit_push_ref_head_bases, ) @@ -545,6 +549,58 @@ async fn run_inner( } }; if let Some(active_active_commit) = committed { + if layout.store().staging_write_prefix().is_none() { + let mut index_config = + crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + if let Some(path) = config.metadb.chunk_index.db.path.as_ref() { + index_config.chunk_index_path.clone_from(path); + } + // The ref is already committed. Index failures only remove acceleration; + // reporting an uncommitted push here would invite a duplicate publication. + match crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &index_config, + layout.repo_prefix(), + &pointer_delta, + &xet_stats.origins, + ) + .await + { + Ok(()) => { + let cache_path = crate::cache::add_remote_candidate_cache_path( + &crate::cache::default_cache_root(), + &store.bucket_identity(), + layout.global_prefix(), + ); + if cache_path.is_file() { + let hashes = pointer_delta + .xorbs() + .values() + .flat_map(|entry| entry.chunks()) + .filter_map(|chunk| MerkleHash::from_hex(chunk.hash()).ok()) + .collect::>(); + let invalidation = tokio::task::spawn_blocking(move || { + crate::cache::add_remote_candidates::AddRemoteCandidateCache::open( + &cache_path, + ) + .and_then(|cache| cache.forget_negatives(&hashes)) + }) + .await; + if !matches!(invalidation, Ok(Ok(()))) { + tracing::warn!( + "capsule committed; candidate cache invalidation failed" + ); + } + } + } + Err(error) => { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair") + } + } + } if let Some(snapshot) = staging_snapshot && let Err(error) = snapshot.commit() { diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index c0a87fb0e..571615cfa 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -15015,6 +15015,103 @@ impl PushPipeline { Ok(Some((hits, skipped_remote))) } + async fn lookup_capsule_chunks_for_add( + &self, + hashes: &[MerkleHash], + chunk_store: &crate::metadata::ChunkIndexStore, + ) -> Result> { + let candidates = chunk_store.capsule_candidates(hashes).await?; + let Some(store) = &self.store else { + return Ok(HashMap::new()); + }; + #[derive(Default)] + struct SourceCandidates<'a> { + shards: HashSet<&'a str>, + placements: Vec<&'a crab_metadata::receipts::CommittedChunkPlacement>, + } + let mut by_repo: BTreeMap<&str, SourceCandidates<'_>> = BTreeMap::new(); + for placement in candidates.placements.values() { + if let Some(anchor) = candidates.anchors.get(&placement.source_anchor_id) { + let source = by_repo.entry(&anchor.source_repo_prefix).or_default(); + source.shards.insert(&anchor.source_shard_hash); + source.placements.push(placement); + } + } + let mut refs = HashMap::new(); + let mut origins = HashMap::new(); + let mut proofs = HashMap::new(); + for (repo, source) in by_repo { + check_cancelled(&self.cancel)?; + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + repo.to_owned(), + self.router.global_prefix().to_owned(), + ); + let root = match crab_metadata::capsule_protocol::load_root(&layout).await { + Ok(root) => root, + Err(crab_metadata::error::MetadataError::Storage { + source: crab_storage::StorageError::NotFound { .. }, + }) => continue, + Err(error) => return Err(error.into()), + }; + let lookup = crab_metadata::capsule_protocol::load_pointer_catalog_lookup_from_root( + &layout, &root, + ) + .await?; + let hashes = source + .shards + .into_iter() + .map(str::to_owned) + .collect::>(); + // Validate and release one source closure before opening the next; + // otherwise each source could retain its entire format byte budget. + let catalog = lookup.shards(layout.store(), &hashes).await?; + for placement in source.placements { + let Some(anchor) = candidates.anchors.get(&placement.source_anchor_id) else { + continue; + }; + let xorb_hash = MerkleHash::from(placement.xorb_hash); + let path = self.router.xorb_path(&xorb_hash); + if anchor + .validate_placement(placement, &catalog, path.as_ref()) + .is_err() + { + continue; + } + let hash = MerkleHash::from(placement.chunk_hash); + refs.insert( + hash, + XorbRef { + xorb_hash, + chunk_index: placement.chunk_index, + uncompressed_size: placement.uncompressed_size, + }, + ); + origins.insert(xorb_hash, anchor.origin.clone()); + proofs.insert(hash, (placement.placement_id(), placement.origin_proof_id)); + } + } + // Source catalogs establish commitment; the existing origin verifier + // binds payload/index checks to the exact object version in the receipt. + let verified = self + .verify_xorb_refs_with_committed_receipts(&refs, &origins) + .await?; + Ok(verified + .into_iter() + .filter_map(|(hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proofs.get(&hash).copied()?; + Some(( + hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + }) + .collect()) + } + async fn lookup_proven_remote_chunks_for_add( &self, chunk_hashes: &[MerkleHash], @@ -15024,12 +15121,8 @@ impl PushPipeline { return Ok(HashMap::new()); } check_cancelled(&self.cancel)?; - let remote_candidates = tokio::time::timeout( - GLOBAL_CHUNK_LOOKUP_BUDGET, - chunk_store.get_committed_candidates_batch(chunk_hashes), - ) - .await - .map_err(|_| { + let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; + let timeout_error = || { CrabError::NetworkTransient(object_store::Error::Generic { store: "chunk_index_db", source: Box::new(std::io::Error::new( @@ -15037,7 +15130,27 @@ impl PushPipeline { "add remote classification proof budget exhausted", )), }) - })??; + }; + let mut capsule = tokio::time::timeout_at( + deadline, + self.lookup_capsule_chunks_for_add(chunk_hashes, &chunk_store), + ) + .await + .map_err(|_| timeout_error())??; + let remaining = chunk_hashes + .iter() + .copied() + .filter(|hash| !capsule.contains_key(hash)) + .collect::>(); + if remaining.is_empty() { + return Ok(capsule); + } + let remote_candidates = tokio::time::timeout_at( + deadline, + chunk_store.get_committed_candidates_batch(&remaining), + ) + .await + .map_err(|_| timeout_error())??; let mut refs = HashMap::with_capacity(remote_candidates.placements.len()); let mut proof_ids = HashMap::with_capacity(remote_candidates.placements.len()); @@ -15066,20 +15179,18 @@ impl PushPipeline { let verified = self .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins) .await?; - Ok(verified - .into_iter() - .filter_map(|(chunk_hash, xorb_ref)| { - let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; - (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( - chunk_hash, - crab_staging::push_plan::ExistingChunkCandidate { - xorb_ref, - placement_id, - origin_proof_id, - }, - )) - }) - .collect()) + capsule.extend(verified.into_iter().filter_map(|(chunk_hash, xorb_ref)| { + let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; + (validated.placements.get(&chunk_hash) == Some(&placement_id)).then_some(( + chunk_hash, + crab_staging::push_plan::ExistingChunkCandidate { + xorb_ref, + placement_id, + origin_proof_id, + }, + )) + })); + Ok(capsule) } async fn lookup_cache_service_chunk_refs( @@ -22051,6 +22162,135 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { + use crab_staging::push_plan::ExistingChunkLookup; + use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; + + let _git_env = CleanGitEnvGuard::new(); + let source = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .args(args) + .current_dir(source.path()) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + git(&["init", "-b", "main"]); + let staging_root = source.path().join("staging"); + let staging = crab_staging::StagingArea::open(staging_root.clone()) + .await + .unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(b"shared capsule payload"), + std::path::Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let chunks = staging + .recipe_page(&staged.recipe, 0) + .unwrap() + .chunks + .into_iter() + .map(|chunk| (chunk.chunk_hash, chunk.len)) + .collect::>(); + staging + .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + let pointer = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); + git(&["add", "model.bin"]); + git(&["commit", "-m", "shared payload"]); + staging.close().await.unwrap(); + let staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(staging_root) + .await + .unwrap(), + ); + let (store, source_router) = test_store_router("repos/source"); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..Default::default() + }; + let (result, _) = super::super::capsule_push::run( + &config, + &[crate::git::remote_helper::PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &source_router, + None, + &[], + Some(&staging), + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(result.all_ok(), "source publication failed: {result:?}"); + drop(staging); + + let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); + let pipeline = PushPipeline::new( + PushConfig::default(), + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + CancellationToken::new(), + None, + ); + let cache_root = tempfile::tempdir().unwrap(); + let mut metadata = crate::core::config::MetaDbTomlConfig::default(); + metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard( + &store, &router, None, &metadata, true, + )); + // No advisory caches or source-repository state can manufacture a hit. + // This exercises the real publisher-to-classifier authority boundary. + let classifier = AddRemoteChunkClassifier { + pipeline, + remote_chunk_index: tokio::sync::OnceCell::new(), + candidate_cache: None, + candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), + candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), + }; + let candidates = classifier.lookup_existing_candidates(&chunks).await; + classifier.close().await; + let candidates = candidates.unwrap(); + assert!( + !candidates.is_empty() && candidates.iter().all(Option::is_some), + "committed source chunks must be reusable from a cold consumer: {candidates:?}" + ); + } + fn add_classifier_with_private_cache( store: &Store, router: &StoreLayout, diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index 4029b8c41..b9b2c57d1 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -23,11 +23,12 @@ use crate::core::metrics::Metrics; // existing payload bound even when a caller requests more network concurrency. const MAX_IN_FLIGHT_XORBS: usize = 4; -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub(crate) struct XetPublicationStats { pub xorbs_uploaded: u64, pub shards_uploaded: u64, pub xorb_bytes_uploaded: u64, + pub origins: HashMap, } #[expect( @@ -133,6 +134,7 @@ pub(crate) async fn prepare_delta( let mut shard_session = PushShardSession::new(); let mut pending_files = Vec::with_capacity(unresolved.len()); let mut uploaded_xorbs = HashSet::new(); + let mut origins = HashMap::new(); let mut xorb_bytes_uploaded = 0_u64; let mut prepared_files = Vec::with_capacity(unresolved.len()); let mut prepared_candidates = Vec::::new(); @@ -278,18 +280,19 @@ pub(crate) async fn prepare_delta( |(ordinal, (expected_hash, expected_refs))| async move { check_cancelled(cancel)?; let path = layout.xorb_path(&expected_hash); - let bytes = match layout + let (bytes, identity) = match layout .store() .get_with_etag_bounded(&path, MAX_XORB_SIZE as u64) .await { - Ok((bytes, _)) => bytes, + Ok(result) => result, Err(crab_storage::StorageError::NotFound { .. }) => { return Ok::<_, CrabError>((ordinal, None)); } Err(error) => return Err(error.into()), }; - let verified = verify_external_xorb(&path, expected_hash, &expected_refs, bytes)?; + let verified = + verify_external_xorb(&path, expected_hash, &expected_refs, bytes, identity)?; Ok((ordinal, verified)) }, )) @@ -302,10 +305,11 @@ pub(crate) async fn prepare_delta( verified_external.sort_by_key(|(ordinal, _)| *ordinal); let mut reused_xorbs = 0_usize; for (_, verified) in verified_external { - let Some((xorb_hash, entry, xorb_placements)) = verified else { + let Some((xorb_hash, entry, xorb_placements, origin)) = verified else { continue; }; reused_xorbs += 1; + origins.insert(xorb_hash, origin); for placement in xorb_placements { placements.entry(placement.chunk_hash).or_insert(placement); } @@ -375,7 +379,7 @@ pub(crate) async fn prepare_delta( let bytes = Bytes::from(bytes); let (entry, xorb_placements) = verify_prepared_xorb(&path, planned_hash, &planned, bytes.clone())?; - let (entry, xorb_placements, created) = publish_xorb_candidate( + let (entry, xorb_placements, created, origin) = publish_xorb_candidate( layout, caching_store, planned_hash, @@ -391,6 +395,7 @@ pub(crate) async fn prepare_delta( entry, xorb_placements, uploaded_bytes, + origin, )) }, )) @@ -400,8 +405,11 @@ pub(crate) async fn prepare_delta( while let Some(result) = uploads.next().await { verified_candidates.push(result?); } - verified_candidates.sort_by_key(|(ordinal, _, _, _, _)| *ordinal); - for (_, planned_hash, entry, xorb_placements, uploaded_bytes) in verified_candidates { + verified_candidates.sort_by_key(|(ordinal, _, _, _, _, _)| *ordinal); + for (_, planned_hash, entry, xorb_placements, uploaded_bytes, origin) in verified_candidates { + if let Some(origin) = origin { + origins.insert(planned_hash, origin); + } if uploaded_bytes > 0 { uploaded_xorbs.insert(planned_hash); xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -449,6 +457,7 @@ pub(crate) async fn prepare_delta( &mut placements, &mut xorb_entries, &mut uploaded_xorbs, + &mut origins, ) .await?; xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -466,6 +475,7 @@ pub(crate) async fn prepare_delta( &mut placements, &mut xorb_entries, &mut uploaded_xorbs, + &mut origins, ) .await?; xorb_bytes_uploaded = xorb_bytes_uploaded.saturating_add(uploaded_bytes); @@ -608,6 +618,7 @@ pub(crate) async fn prepare_delta( xorbs_uploaded: uploaded_xorbs.len() as u64, shards_uploaded, xorb_bytes_uploaded, + origins, }, )) } @@ -619,11 +630,12 @@ async fn publish_built_xorb( placements: &mut ChunkPlacementMap, xorb_entries: &mut HashMap, uploaded_xorbs: &mut HashSet, + origins: &mut HashMap, ) -> Result { let encoded_size = result.bytes.len() as u64; let body_digest = blake3::hash(&result.bytes).to_hex().to_string(); let entry = catalog_xorb_entry(result.bytes.len() as u64, body_digest, &result.placements)?; - let (entry, xorb_placements, created) = publish_xorb_candidate( + let (entry, xorb_placements, created, origin) = publish_xorb_candidate( layout, caching_store, result.hash, @@ -632,6 +644,9 @@ async fn publish_built_xorb( result.placements, ) .await?; + if let Some(origin) = origin { + origins.insert(result.hash, origin); + } if created { uploaded_xorbs.insert(result.hash); } @@ -653,6 +668,7 @@ async fn publish_xorb_candidate( crab_metadata::capsule_protocol::XorbCatalogEntry, Vec, bool, + Option, )> { let path = layout.xorb_path(&xorb_hash); let staged = layout.store().staging_write_prefix().is_some(); @@ -691,16 +707,22 @@ async fn publish_xorb_candidate( .get_with_etag_bounded(&path, MAX_XORB_SIZE as u64) .await { - Ok((existing, _)) => { - let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? - .ok_or_else(|| CrabError::CorruptObject { - path: path.to_string(), - reason: format!( - "existing xorb {} does not authenticate as the requested logical content", - xorb_hash.hex() - ), - })?; - return Ok((verified.1, verified.2, false)); + Ok((existing, identity)) => { + let verified = verify_external_xorb( + &path, + xorb_hash, + &expected_refs, + existing, + identity, + )? + .ok_or_else(|| CrabError::CorruptObject { + path: path.to_string(), + reason: format!( + "existing xorb {} does not authenticate as the requested logical content", + xorb_hash.hex() + ), + })?; + return Ok((verified.1, verified.2, false, Some(verified.3))); } Err(crab_storage::StorageError::NotFound { .. }) => {} Err(error) => return Err(error.into()), @@ -711,15 +733,18 @@ async fn publish_xorb_candidate( // canonical encoding above when present; otherwise record verified // staged bytes for service-owned promotion before ref publication. let created = layout.store().put_if_absent_verified(&path, bytes).await?; - return Ok((local_entry, local_placements, created)); + return Ok((local_entry, local_placements, created, None)); } let cache_bytes = caching_store.map(|_| bytes.clone()); + // Local builder/prepared verification already authenticated these bytes. + // Carry the payload digest with the exact immutable PUT/readback identity. + let payload_digest = XorbParser::parse(bytes.clone())?.payload_digest(); match layout .store() .create_or_read_immutable(&path, bytes, MAX_XORB_SIZE as u64) .await? { - crab_storage::ImmutableCreateOutcome::Created(_) => { + crab_storage::ImmutableCreateOutcome::Created(identity) => { if let (Some(cache), Some(bytes)) = (caching_store, cache_bytes) { warm_published_cache_object( cache, @@ -729,20 +754,36 @@ async fn publish_xorb_candidate( ) .await; } - Ok((local_entry, local_placements, true)) + let origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + xorb_hash.into(), + payload_digest, + local_entry.encoded_size(), + identity.e_tag, + identity.version, + ); + Ok((local_entry, local_placements, true, Some(origin))) } crab_storage::ImmutableCreateOutcome::Existing { - bytes: existing, .. + bytes: existing, + identity, } => { - let verified = verify_external_xorb(&path, xorb_hash, &expected_refs, existing)? - .ok_or_else(|| CrabError::CorruptObject { - path: path.to_string(), - reason: format!( - "existing xorb {} does not authenticate as the requested logical content", - xorb_hash.hex() - ), - })?; - Ok((verified.1, verified.2, false)) + let verified = verify_external_xorb( + &path, + xorb_hash, + &expected_refs, + existing, + identity, + )? + .ok_or_else(|| CrabError::CorruptObject { + path: path.to_string(), + reason: format!( + "existing xorb {} does not authenticate as the requested logical content", + xorb_hash.hex() + ), + })?; + Ok((verified.1, verified.2, false, Some(verified.3))) } } } @@ -844,11 +885,13 @@ fn verify_external_xorb( expected_hash: MerkleHash, expected_refs: &HashMap, bytes: Bytes, + identity: crab_storage::ETag, ) -> Result< Option<( MerkleHash, crab_metadata::capsule_protocol::XorbCatalogEntry, Vec, + crab_metadata::receipts::OriginReceipt, )>, > { let encoded_size = bytes.len() as u64; @@ -921,7 +964,16 @@ fn verify_external_xorb( } } let entry = catalog_xorb_entry(encoded_size, body_digest, &xorb_placements)?; - Ok(Some((expected_hash, entry, xorb_placements))) + let origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + expected_hash.into(), + parser.payload_digest(), + encoded_size, + identity.e_tag, + identity.version, + ); + Ok(Some((expected_hash, entry, xorb_placements, origin))) } fn verify_prepared_xorb( @@ -1685,7 +1737,7 @@ mod tests { layout.store().clone() }; let layout = crab_storage::StoreLayout::new(store, "repo".to_owned()); - let (entry, placements, created) = publish_xorb_candidate( + let (entry, placements, created, _origin) = publish_xorb_candidate( &layout, None, candidate.hash, diff --git a/crab/src/metadata/metadb/mod.rs b/crab/src/metadata/metadb/mod.rs index a4e4db9a0..7d5344963 100644 --- a/crab/src/metadata/metadb/mod.rs +++ b/crab/src/metadata/metadb/mod.rs @@ -27,6 +27,7 @@ use tracing::{debug, info, warn}; use crate::core::error::{CrabError, MetaDbError, Result}; use crate::core::metrics::Metrics; use crab_metadata::chunk_index::ChunkIndex; +use crab_metadata::key_codec::METADB_FORMAT_VERSION; use crab_metadata::persistent_chunk_index::PersistentChunkIndex; use crab_metadata::{key_codec, value_codec}; @@ -66,8 +67,6 @@ const DEFAULT_WAL_FLUSH_SIZE: u64 = 4 * 1024 * 1024; /// lookups. const DEFAULT_BLOOM_BITS_PER_KEY: u32 = 10; -const METADB_FORMAT_VERSION: u32 = 1; - /// Aggregated snapshot of `sys:*` keys in one database. /// /// Returned by [`MetaDb::file_index_system_keys`] and diff --git a/crab/src/metadata/metadb/stores/chunk_index.rs b/crab/src/metadata/metadb/stores/chunk_index.rs index 341d53898..89dd9597a 100644 --- a/crab/src/metadata/metadb/stores/chunk_index.rs +++ b/crab/src/metadata/metadb/stores/chunk_index.rs @@ -104,6 +104,60 @@ impl ChunkIndexStore { self } + /// Load capsule hints without interpreting them as released manifest receipts. + pub(crate) async fn capsule_candidates( + &self, + hashes: &[MerkleHash], + ) -> Result { + use crab_metadata::capsule_protocol::{ + CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, + capsule_chunk_head_key, + }; + let mut output = CapsuleChunkCandidates::default(); + // Stay below Db's dense range-scan threshold: unrelated chunk history + // must not turn an add-time point batch into a scan of the global index. + for hashes in hashes.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { + let keys = hashes + .iter() + .map(|hash| Bytes::copy_from_slice(&capsule_chunk_head_key(&(*hash).into()))) + .collect::>(); + let values = self.db.get_batch(&keys).await?; + let mut anchor_ids = HashSet::new(); + for (hash, value) in hashes.iter().zip(values) { + let Some(value) = value else { + continue; + }; + let placement = CommittedChunkPlacement::decode(&value)?; + if placement.chunk_hash != <[u8; 32]>::from(*hash) { + return Err(CrabError::StagingCorrupt( + "capsule candidate hash differs from index key".to_owned(), + )); + } + anchor_ids.insert(placement.source_anchor_id); + output.placements.insert(placement.chunk_hash, placement); + } + let ids = anchor_ids + .into_iter() + .filter(|id| !output.anchors.contains_key(id)) + .collect::>(); + let keys = ids + .iter() + .map(|id| Bytes::copy_from_slice(&capsule_chunk_anchor_key(id))) + .collect::>(); + let values = self.db.get_batch(&keys).await?; + for (id, value) in ids.into_iter().zip(values) { + let value = value.ok_or_else(|| { + CrabError::StagingCorrupt( + "capsule candidate source anchor is absent".to_owned(), + ) + })?; + let anchor = CapsuleChunkAnchor::decode(&id, &value)?; + output.anchors.insert(id, anchor); + } + } + Ok(output) + } + /// Look up a single chunk hash through all three tiers. /// /// Cache-first: in-memory → persistent, when available → remote. diff --git a/crates/crab-auth-server/src/receive/capsule.rs b/crates/crab-auth-server/src/receive/capsule.rs index 3a6475ac3..1a6a613eb 100644 --- a/crates/crab-auth-server/src/receive/capsule.rs +++ b/crates/crab-auth-server/src/receive/capsule.rs @@ -1,6 +1,6 @@ //! Protocol-v2 protected-push candidate verification. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use crab_auth::PushRefUpdate; use crab_coordination::write_coordinator::{CommitOutcome, CoordinatedRefUpdate}; @@ -27,6 +27,7 @@ pub(super) struct VerifiedCapsuleCandidate { pub replication_objects: Vec, pub publication: Capsule, dependency_copies: Vec, + origins: HashMap, } pub(super) async fn commit_capsule_candidate( @@ -102,9 +103,29 @@ pub(super) async fn commit_capsule_candidate( .map(Some); } - promote_staged_writes(ctx.store(), &plan.staged_objects).await?; - promote_dependency_copies(ctx, &verified.dependency_copies).await?; - if let Some(delta) = capsule.pointer_catalog_delta()? { + let mut origins = verified.origins; + let mut other_objects = Vec::new(); + for object in &plan.staged_objects { + if crab_storage::content_hash_from_path(&object.canonical_key, "xorbs").is_some() { + let bytes = read_verified_staged_object(ctx.store(), object).await?; + let origin = super::capsule_dependencies::promote_xorb_origin( + ctx.store(), + &object.canonical_key, + bytes, + ) + .await?; + origins.insert( + crab_xet::hash::MerkleHash::from(origin.content_hash), + origin, + ); + } else { + other_objects.push(object.clone()); + } + } + promote_staged_writes(ctx.store(), &other_objects).await?; + origins.extend(promote_dependency_copies(ctx, &verified.dependency_copies).await?); + let delta = capsule.pointer_catalog_delta()?.unwrap_or_default(); + if !delta.is_empty() { crab_metadata::ref_registry::union_register_repo_shards( ctx.store(), ctx.router(), @@ -122,12 +143,12 @@ pub(super) async fn commit_capsule_candidate( .iter() .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); let base = view.root_snapshot().clone(); - if changes_namespace { + let result = if changes_namespace { let layout = ctx.router().clone(); let repo_prefix = ctx.repo_prefix().to_owned(); let active_active = active_active.cloned(); let replication_objects = replication_objects.to_vec(); - return crab_write::with_ref_namespaces( + crab_write::with_ref_namespaces( ctx.store(), ctx.router(), &ref_names, @@ -158,19 +179,38 @@ pub(super) async fn commit_capsule_candidate( .await }, ) - .await; + .await + } else { + publish_verified_candidate( + ctx.store(), + ctx.router(), + ctx.repo_prefix(), + base, + &transaction, + &capsule, + active_active, + replication_objects, + ) + .await + }; + if result.is_ok() { + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + ctx.repo_prefix(), + ctx.router().global_prefix(), + ); + if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( + std::sync::Arc::clone(ctx.store().inner()), + &config, + ctx.repo_prefix(), + &delta, + &origins, + ) + .await + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + } } - publish_verified_candidate( - ctx.store(), - ctx.router(), - ctx.repo_prefix(), - base, - &transaction, - &capsule, - active_active, - replication_objects, - ) - .await + result } async fn publish_verified_candidate( @@ -358,7 +398,7 @@ pub(super) async fn verify_capsule_candidate( validate_capsule_transaction(plan, candidate)?; let candidate_delta = candidate.pointer_catalog_delta()?.unwrap_or_default(); - let (changed_paths, publication, replication_objects, dependency_copies) = + let (changed_paths, publication, replication_objects, dependency_copies, origins) = match prepare.view_scope.as_ref() { None => { if prepared_root != plan.base_root_digest { @@ -382,9 +422,9 @@ pub(super) async fn verify_capsule_candidate( ) .await?; catalog.apply(&candidate_delta)?; - let (objects, copies) = + let (objects, copies, origins) = verify_pointer_dependencies(ctx, plan, &catalog, &pointers, None).await?; - (changed_paths, candidate.clone(), objects, copies) + (changed_paths, candidate.clone(), objects, copies, origins) } Some(scope) => { let filtered_router = crab_storage::StoreLayout::with_global_prefix( @@ -445,7 +485,7 @@ pub(super) async fn verify_capsule_candidate( )?; let mut final_catalog = source_catalog; final_catalog.apply(&delta)?; - let (objects, copies) = verify_pointer_dependencies( + let (objects, copies, origins) = verify_pointer_dependencies( ctx, plan, &final_catalog, @@ -465,7 +505,13 @@ pub(super) async fn verify_capsule_candidate( } let publication = Capsule::build(&materialized.transaction, materialized.git_packs, sections)?; - (materialized.changed_paths, publication, objects, copies) + ( + materialized.changed_paths, + publication, + objects, + copies, + origins, + ) } }; let publication_transaction = publication.transaction()?; @@ -496,6 +542,7 @@ pub(super) async fn verify_capsule_candidate( replication_objects: replication_objects.into_iter().collect(), publication, dependency_copies, + origins, }) } diff --git a/crates/crab-auth-server/src/receive/capsule_dependencies.rs b/crates/crab-auth-server/src/receive/capsule_dependencies.rs index a3d692b3e..059be9987 100644 --- a/crates/crab-auth-server/src/receive/capsule_dependencies.rs +++ b/crates/crab-auth-server/src/receive/capsule_dependencies.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use bytes::Bytes; use crab_metadata::capsule_protocol::PointerCatalog; @@ -130,7 +130,11 @@ pub(super) async fn verify_pointer_dependencies( catalog: &PointerCatalog, pointers: &ReachablePointerScan, fallback: Option<&crab_storage::StoreLayout>, -) -> Result<(BTreeSet, Vec)> { +) -> Result<( + BTreeSet, + Vec, + HashMap, +)> { let mut referenced = BTreeSet::from([ctx .router() .capsule_path(&plan.run_hash) @@ -139,6 +143,7 @@ pub(super) async fn verify_pointer_dependencies( let mut verified_shards = BTreeMap::::new(); let mut verified_xorbs = BTreeSet::new(); let mut copies = Vec::new(); + let mut origins = HashMap::new(); for pointer in &pointers.crab_pointers { let file_hash = MerkleHash::from(pointer.file_hash).hex(); let file = catalog @@ -163,7 +168,7 @@ pub(super) async fn verify_pointer_dependencies( Some(bytes) => bytes.clone(), None => { let fallback_key = fallback.map(|layout| layout.shard_path(&shard_hash)); - let bytes = read_candidate_object( + let (bytes, _) = read_candidate_object( ctx, plan, &shard_key, @@ -217,7 +222,7 @@ pub(super) async fn verify_pointer_dependencies( referenced.insert(key.clone()); if verified_xorbs.insert(hash.to_owned()) { let fallback_key = fallback.map(|layout| layout.xorb_path(&xorb_hash)); - let bytes = read_candidate_object( + let (bytes, identity) = read_candidate_object( ctx, plan, &key, @@ -232,6 +237,12 @@ pub(super) async fn verify_pointer_dependencies( )); } validate_staged_xorb(&key, &bytes, &chunks)?; + if let Some(identity) = identity { + origins.insert( + xorb_hash, + verified_xorb_origin(&key, bytes.clone(), identity)?, + ); + } } } } @@ -252,7 +263,8 @@ pub(super) async fn verify_pointer_dependencies( ) .to_string(); referenced.insert(key.clone()); - let bytes = read_candidate_object(ctx, plan, &key, None, pointer.size, &mut copies).await?; + let (bytes, _) = + read_candidate_object(ctx, plan, &key, None, pointer.size, &mut copies).await?; if <[u8; 32]>::from(Sha256::digest(&bytes)) != pointer.oid { return Err(invalid( "candidate LFS object digest differs from its pointer", @@ -278,7 +290,7 @@ pub(super) async fn verify_pointer_dependencies( } } copies.dedup_by(|left, right| left.target_key == right.target_key); - Ok((referenced, copies)) + Ok((referenced, copies, origins)) } async fn read_candidate_object( @@ -288,8 +300,8 @@ async fn read_candidate_object( fallback_key: Option<&object_store::path::Path>, expected_size: u64, copies: &mut Vec, -) -> Result { - let bytes = match plan +) -> Result<(Bytes, Option)> { + let (bytes, identity) = match plan .staged_objects .iter() .find(|object| object.canonical_key == canonical_key) @@ -300,7 +312,10 @@ async fn read_candidate_object( "staged dependency size differs from the candidate catalog", )); } - read_verified_staged_object(ctx.store(), object).await? + ( + read_verified_staged_object(ctx.store(), object).await?, + None, + ) } None => match ctx .store() @@ -310,7 +325,7 @@ async fn read_candidate_object( ) .await { - Ok((bytes, _)) => bytes, + Ok((bytes, identity)) => (bytes, Some(identity)), Err(crab_storage::StorageError::NotFound { .. }) => { let fallback_key = fallback_key.ok_or_else(|| AuthServerError::NotFound { path: canonical_key.to_owned(), @@ -325,7 +340,7 @@ async fn read_candidate_object( size: expected_size, digest: blake3::hash(&bytes).to_hex().to_string(), }); - bytes + (bytes, None) } Err(error) => return Err(error.into()), }, @@ -335,13 +350,14 @@ async fn read_candidate_object( "candidate dependency size differs from the candidate catalog", )); } - Ok(bytes) + Ok((bytes, identity)) } pub(super) async fn promote_dependency_copies( ctx: &ReceiveContext, copies: &[DependencyCopy], -) -> Result<()> { +) -> Result> { + let mut origins = HashMap::new(); for copy in copies { let (bytes, _) = ctx .store() @@ -356,12 +372,64 @@ pub(super) async fn promote_dependency_copies( "filtered view dependency changed after protected verification", )); } - ctx.store() - .put_if_absent_verified( - &object_store::path::Path::from(copy.target_key.clone()), - bytes, - ) - .await?; + if content_hash_from_path(©.target_key, "xorbs").is_some() { + let origin = promote_xorb_origin(ctx.store(), ©.target_key, bytes).await?; + origins.insert(MerkleHash::from(origin.content_hash), origin); + } else { + ctx.store() + .put_if_absent_verified( + &object_store::path::Path::from(copy.target_key.clone()), + bytes, + ) + .await?; + } } - Ok(()) + Ok(origins) +} + +// Verification checked the complete body before promotion. Keep the immutable +// operation identity with that body so a later HEAD cannot certify a replacement. +pub(super) async fn promote_xorb_origin( + store: &crab_storage::Store, + key: &str, + bytes: Bytes, +) -> Result { + let expected = *blake3::hash(&bytes).as_bytes(); + let maximum = bytes.len() as u64; + let outcome = store + .create_or_read_immutable(&object_store::path::Path::from(key), bytes.clone(), maximum) + .await?; + let identity = match outcome { + crab_storage::ImmutableCreateOutcome::Created(identity) => identity, + crab_storage::ImmutableCreateOutcome::Existing { + bytes: stored, + identity, + } => { + if *blake3::hash(&stored).as_bytes() != expected { + return Err(conflict( + "canonical xorb differs from the verified protected body", + )); + } + identity + } + }; + verified_xorb_origin(key, bytes, identity) +} + +fn verified_xorb_origin( + key: &str, + bytes: Bytes, + identity: crab_storage::ETag, +) -> Result { + let size = bytes.len() as u64; + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes)?; + Ok(crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + key.to_owned(), + parser.hash().into(), + parser.payload_digest(), + size, + identity.e_tag, + identity.version, + )) } diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs new file mode 100644 index 000000000..9553a0c83 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs @@ -0,0 +1,171 @@ +//! Rebuildable chunk hints whose source authority is a capsule catalog. +use std::collections::{BTreeMap, HashMap}; + +use serde::{Deserialize, Serialize}; + +use super::XorbCatalogEntry; +use crate::error::{MetadataError, Result}; +use crate::receipts::{CommittedChunkPlacement, OriginReceipt, RECEIPT_SCHEMA_VERSION}; +use crab_xet::hash::MerkleHash; + +pub use crate::key_codec::{capsule_chunk_anchor_key, capsule_chunk_head_key}; + +/// Capsule source and verified physical origin for one xorb. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapsuleChunkAnchor { + pub source_repo_prefix: String, + pub source_shard_hash: String, + pub body_digest: String, + pub origin: OriginReceipt, +} + +impl CapsuleChunkAnchor { + /// Decode bounded provenance and verify its immutable index identity. + pub fn decode(id: &[u8; 32], bytes: &[u8]) -> Result { + if bytes.len() > 16 * 1024 { + return Err(corrupt("capsule source anchor exceeds its byte bound")); + } + let anchor: Self = serde_json::from_slice(bytes) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if anchor.id()? != *id { + return Err(corrupt("capsule source anchor digest differs")); + } + Ok(anchor) + } + + /// Identify this immutable source and origin assertion. + pub fn id(&self) -> Result<[u8; 32]> { + if self.source_repo_prefix.len() > 4096 + || self.source_shard_hash.len() != 64 + || self.body_digest.len() != 64 + { + return Err(corrupt("invalid capsule source identity")); + } + let bytes = serde_json::to_vec(self) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?; + if bytes.len() > 16 * 1024 { + return Err(corrupt("capsule source anchor exceeds its byte bound")); + } + let mut hash = blake3::Hasher::new_derive_key("crab capsule chunk source v1"); + hash.update(&bytes); + Ok(*hash.finalize().as_bytes()) + } + + /// Check an exact placement against its committed shard closure and origin assertion. + pub fn validate_placement( + &self, + placement: &CommittedChunkPlacement, + catalog: &super::PointerCatalog, + object_key: &str, + ) -> Result<()> { + placement.validate()?; + let xorb_hash = MerkleHash::from(placement.xorb_hash).hex(); + let rooted = catalog + .shards() + .get(&self.source_shard_hash) + .is_some_and(|shard| shard.xorb_hashes().binary_search(&xorb_hash).is_ok()); + let entry = catalog + .xorbs() + .get(&xorb_hash) + .ok_or_else(|| corrupt("capsule placement xorb is absent from source catalog"))?; + self.validate_catalog_xorb(object_key, entry)?; + let chunk = entry.chunks().get(placement.chunk_index as usize); + if !rooted + || placement.source_anchor_id != self.id()? + || placement.origin_proof_id != self.origin.proof_id() + || placement.xorb_hash != self.origin.content_hash + || !chunk.is_some_and(|chunk| { + chunk.hash() == MerkleHash::from(placement.chunk_hash).hex() + && chunk.uncompressed_size() == placement.uncompressed_size + }) + { + return Err(corrupt( + "capsule placement differs from its source shard or origin", + )); + } + Ok(()) + } + + /// Check the source catalog's physical descriptor before verifying origin availability. + pub fn validate_catalog_xorb(&self, object_key: &str, entry: &XorbCatalogEntry) -> Result<()> { + if self.source_repo_prefix.is_empty() || self.body_digest != entry.body_digest() { + return Err(corrupt( + "source catalog differs from the capsule chunk anchor", + )); + } + self.origin.validate( + "canonical-origin", + object_key, + self.origin.content_hash, + entry.encoded_size(), + ) + } +} + +/// Untrusted hints, requiring current source-catalog and canonical-origin checks. +#[derive(Debug, Default)] +pub struct CapsuleChunkCandidates { + pub placements: HashMap<[u8; 32], CommittedChunkPlacement>, + pub anchors: HashMap<[u8; 32], CapsuleChunkAnchor>, +} + +/// Encode one bounded xorb's candidates after full verification and ref publication. +pub(crate) fn capsule_chunk_candidate_rows( + repo_prefix: &str, + source_shard_hash: &str, + xorb_hash: MerkleHash, + entry: &XorbCatalogEntry, + origin: &OriginReceipt, +) -> Result, Vec)>> { + let anchor = CapsuleChunkAnchor { + source_repo_prefix: repo_prefix.to_owned(), + source_shard_hash: source_shard_hash.to_owned(), + body_digest: entry.body_digest().to_owned(), + origin: origin.clone(), + }; + anchor.validate_catalog_xorb(&origin.object_key, entry)?; + if origin.content_hash != <[u8; 32]>::from(xorb_hash) { + return Err(corrupt("origin names a different xorb")); + } + let anchor_id = anchor.id()?; + let mut rows = vec![( + capsule_chunk_anchor_key(&anchor_id).to_vec(), + serde_json::to_vec(&anchor) + .map_err(|source| MetadataError::PointerCatalogRecord { source })?, + )]; + let mut placements = BTreeMap::new(); + for (index, chunk) in entry.chunks().iter().enumerate() { + let chunk_hash = + MerkleHash::from_hex(chunk.hash()).map_err(|_| corrupt("invalid chunk hash"))?; + placements + .entry(<[u8; 32]>::from(chunk_hash)) + .or_insert(CommittedChunkPlacement { + schema_version: RECEIPT_SCHEMA_VERSION, + chunk_hash: chunk_hash.into(), + xorb_hash: xorb_hash.into(), + chunk_index: u32::try_from(index) + .map_err(|_| corrupt("chunk position overflows"))?, + uncompressed_size: chunk.uncompressed_size(), + origin_proof_id: origin.proof_id(), + source_anchor_id: anchor_id, + }); + } + for (hash, placement) in placements { + rows.push(( + capsule_chunk_head_key(&hash).to_vec(), + placement.encode()?.to_vec(), + )); + } + Ok(rows) +} + +fn corrupt(reason: &str) -> MetadataError { + MetadataError::CorruptObject { + path: "capsule chunk candidates".to_owned(), + reason: reason.to_owned(), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs new file mode 100644 index 000000000..aff1436c4 --- /dev/null +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs @@ -0,0 +1,147 @@ +use super::*; +use crate::capsule_protocol::XorbChunkEntry; + +fn fixture() -> (MerkleHash, XorbCatalogEntry, OriginReceipt) { + let hash = MerkleHash::from([3; 32]); + let entry = XorbCatalogEntry::new( + 100, + blake3::hash(b"whole body").to_hex().to_string(), + vec![ + XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9), + XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9), + ], + ); + let mut origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", hash.hex()), + hash.into(), + *blake3::hash(b"payload").as_bytes(), + 100, + Some("etag".to_owned()), + Some("version".to_owned()), + ); + origin.proven_at_unix_secs = 1; + (hash, entry, origin) +} + +#[test] +fn capsule_candidates_preserve_distinct_digests_and_first_occurrence() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + assert_eq!( + ( + rows.len(), + placement.chunk_index, + anchor.body_digest, + anchor.origin.payload_digest + ), + (2, 0, entry.body_digest().to_owned(), origin.payload_digest) + ); +} + +#[test] +fn capsule_anchor_rejects_changed_source_identity() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let mut anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + anchor.source_repo_prefix = "other".to_owned(); + assert!( + CapsuleChunkAnchor::decode( + &placement.source_anchor_id, + &serde_json::to_vec(&anchor).unwrap() + ) + .is_err() + ); +} + +#[test] +fn capsule_anchor_requires_the_authenticated_physical_descriptor() { + let (hash, entry, origin) = fixture(); + let rows = + capsule_chunk_candidate_rows("repo", &"2".repeat(64), hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + let other = XorbCatalogEntry::new(100, "4".repeat(64), entry.chunks().to_vec()); + assert!( + anchor + .validate_catalog_xorb(&origin.object_key, &other) + .is_err() + ); +} + +#[test] +fn capsule_anchor_rejects_oversized_records_before_decoding() { + assert!(CapsuleChunkAnchor::decode(&[1; 32], &vec![b' '; 16 * 1024 + 1]).is_err()); +} + +fn rooted_fixture() -> ( + CapsuleChunkAnchor, + CommittedChunkPlacement, + crate::capsule_protocol::PointerCatalog, +) { + let (hash, entry, origin) = fixture(); + let shard = "2".repeat(64); + let rows = capsule_chunk_candidate_rows("repo", &shard, hash, &entry, &origin).unwrap(); + let placement = CommittedChunkPlacement::decode(&rows[1].1).unwrap(); + let anchor = CapsuleChunkAnchor::decode(&placement.source_anchor_id, &rows[0].1).unwrap(); + let mut catalog = crate::capsule_protocol::PointerCatalog::new(); + catalog.insert_xorb(hash.hex(), entry).unwrap(); + catalog + .insert_shard( + shard, + crate::capsule_protocol::ShardCatalogEntry::new(20, vec![hash.hex()]), + ) + .unwrap(); + (anchor, placement, catalog) +} + +#[test] +fn capsule_placement_requires_an_authenticated_source_shard() { + let (anchor, placement, rooted) = rooted_fixture(); + let mut unrooted = crate::capsule_protocol::PointerCatalog::new(); + for (hash, entry) in rooted.xorbs() { + unrooted.insert_xorb(hash.clone(), entry.clone()).unwrap(); + } + assert!( + anchor + .validate_placement(&placement, &unrooted, &anchor.origin.object_key) + .is_err() + ); +} + +#[test] +fn capsule_placement_accepts_exact_rooted_content() { + let (anchor, placement, catalog) = rooted_fixture(); + assert!( + anchor + .validate_placement(&placement, &catalog, &anchor.origin.object_key) + .is_ok() + ); +} + +#[test] +fn capsule_placement_rejects_tampered_occurrences_and_proofs() { + let (anchor, placement, catalog) = rooted_fixture(); + for field in ["size", "position", "chunk", "origin", "source"] { + let mut changed = placement.clone(); + match field { + "size" => changed.uncompressed_size += 1, + "position" => changed.chunk_index = 2, + "chunk" => changed.chunk_hash = [9; 32], + "origin" => changed.origin_proof_id = [9; 32], + "source" => changed.source_anchor_id = [9; 32], + _ => unreachable!(), + } + assert!( + anchor + .validate_placement(&changed, &catalog, &anchor.origin.object_key) + .is_err(), + "accepted changed {field}" + ); + } +} diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 17b51aaab..07c1359c9 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -10,6 +10,7 @@ mod capsule; mod captured_visibility; #[cfg(feature = "storage")] mod catalog_capture; +mod chunk_candidates; mod history; mod layered; #[cfg(feature = "storage")] @@ -123,3 +124,9 @@ fn valid_ref_namespace<'a>(names: impl IntoIterator) -> bool { .all(|(index, _)| !names.contains(&name[..index])) }) } + +pub use chunk_candidates::{ + CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, capsule_chunk_head_key, +}; + +pub(crate) use chunk_candidates::capsule_chunk_candidate_rows; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs index f9e3bf451..3b38e2e0d 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged/reader.rs @@ -280,7 +280,7 @@ impl PointerCatalogLookup { )?; let hashes = requested(files, &mut work)?; let mut shards = requested(shards, &mut work)?; - let mut xorbs = requested(xorbs, &mut work)?; + let xorbs = requested(xorbs, &mut work)?; if hashes.is_empty() && shards.is_empty() && xorbs.is_empty() { return Ok(PointerCatalog::new()); } @@ -320,13 +320,43 @@ impl PointerCatalogLookup { catalog.insert_file(hash.clone(), entry)?; } } + let selected = self.selected_shards(store, &shards, &mut work).await?; + for (hash, entry) in selected.shards { + catalog.insert_shard(hash, entry)?; + } + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + for (hash, entry) in selected.xorbs { + catalog.insert_xorb(hash, entry)?; + } + catalog.validate(true, true)?; + Ok(catalog) + } + + /// Select requested shard roots and their complete authenticated xorb closures. + pub async fn shards(&self, store: &Store, hashes: &[String]) -> Result { + let mut work = self.work(); + let hashes = requested(hashes, &mut work)?; + self.selected_shards(store, &hashes, &mut work).await + } + + async fn selected_shards( + &self, + store: &Store, + shards: &BTreeSet, + work: &mut PageWork, + ) -> Result { + let mut catalog = PointerCatalog::new(); let root = self .checkpoint .as_ref() .and_then(|s| s.descriptor.shards.as_ref()); let rows = self - .rows(store, root, Kind::CatalogShards, &keys(&shards)?, &mut work) + .rows(store, root, Kind::CatalogShards, &keys(shards)?, work) .await?; + let mut xorbs = BTreeSet::new(); for (hash, row) in shards.iter().zip(rows) { for value in row { let row: ShardRow = from_json(&value.bytes)?; @@ -336,7 +366,7 @@ impl PointerCatalogLookup { &value.origin, row.xorbs.as_ref(), Kind::CatalogShardXorbs, - &mut work, + work, ) .await?; let closure = closure @@ -352,20 +382,12 @@ impl PointerCatalogLookup { verify_identity(&value, Kind::CatalogShards, &json(&entry)?)?; catalog.insert_shard(hash.clone(), entry)?; } + let Some(entry) = catalog.shards.get(hash) else { + continue; + }; + xorbs.extend(entry.xorb_hashes.iter().cloned()); } - for file in catalog.files.values() { - catalog - .shards - .get(&file.shard_hash) - .ok_or_else(|| corrupt("selected file references an absent shard"))?; - } - xorbs.extend( - catalog - .shards - .values() - .flat_map(|entry| entry.xorb_hashes.iter().cloned()), - ); - let selected = self.selected_xorbs(store, &xorbs, &mut work).await?; + let selected = self.selected_xorbs(store, &xorbs, work).await?; for (hash, entry) in selected.xorbs { catalog.insert_xorb(hash, entry)?; } diff --git a/crates/crab-metadata/src/key_codec.rs b/crates/crab-metadata/src/key_codec.rs index 19301a0a6..b0436ec5a 100644 --- a/crates/crab-metadata/src/key_codec.rs +++ b/crates/crab-metadata/src/key_codec.rs @@ -11,6 +11,8 @@ //! - `0x03 || hash[32]` — latest committed chunk-receipt candidate. //! - `0x04 || proof_id[32]` — immutable xorb origin proof. //! - `0x05 || anchor_id[32]` — immutable source-shard anchor. +//! - `0x06 || hash[32]` — point-readable capsule chunk candidate. +//! - `0x07 || anchor_id[32]` — immutable capsule source/origin anchor. //! - `0xFF || b"sys:" || name` — system key (format_version, epoch, //! created_at, gc_generation). //! @@ -22,6 +24,9 @@ use crab_xet::xorb::format::MerkleHash; use crate::error::{MetadataError, Result}; +/// Canonical metadata database format shared by all reader and writer adapters. +pub const METADB_FORMAT_VERSION: u32 = 1; + /// Prefix byte for content-addressed keys (file_hash or chunk_hash). pub const PREFIX_CONTENT: u8 = 0x01; @@ -41,6 +46,23 @@ pub const PREFIX_ORIGIN_PROOF: u8 = 0x04; /// Prefix byte for immutable repository-shard source anchors. pub const PREFIX_SOURCE_ANCHOR: u8 = 0x05; +/// Capsule candidates use distinct keys so released manifest readers cannot +/// mistake capsule provenance for a manifest-generation assertion. +const PREFIX_CAPSULE_CHUNK_HEAD: u8 = 0x06; +const PREFIX_CAPSULE_CHUNK_ANCHOR: u8 = 0x07; + +/// Encode a point-readable capsule candidate key. +#[must_use] +pub fn capsule_chunk_head_key(hash: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_HEAD, hash) +} + +/// Encode an immutable capsule provenance key. +#[must_use] +pub fn capsule_chunk_anchor_key(hash: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_ANCHOR, hash) +} + /// Prefix byte for the `sys:*` system-key namespace. Chosen as `0xFF` /// so it sorts after every content key and is trivial to filter out of /// content-only scans. diff --git a/crates/crab-metadata/src/remote_index.rs b/crates/crab-metadata/src/remote_index.rs index e33763806..4c94dc6a8 100644 --- a/crates/crab-metadata/src/remote_index.rs +++ b/crates/crab-metadata/src/remote_index.rs @@ -100,6 +100,73 @@ pub async fn write_index_entries( close_result } +/// Publish rebuildable capsule chunk hints using origin receipts captured during byte verification. +/// +/// Call after ref visibility. Every opened database is closed on success or error; +/// callers must await this future through completion. Readers still revalidate source +/// visibility and exact origin identity, so partial batches never confer authority. +pub async fn write_capsule_chunk_candidates( + store: Arc, + config: &RemoteIndexConfig, + repo_prefix: &str, + catalog: &crate::capsule_protocol::PointerCatalog, + origins: &HashMap, +) -> Result<()> { + let mut source_shards = HashMap::new(); + for (shard_hash, shard) in catalog.shards() { + for xorb in shard.xorb_hashes() { + source_shards + .entry(xorb.as_str()) + .or_insert(shard_hash.as_str()); + } + } + let mut selected = Vec::new(); + for (hash, entry) in catalog.xorbs().iter().rev() { + let parsed = MerkleHash::from_hex(hash).map_err(|_| MetadataError::CorruptObject { + path: CHUNK_INDEX_DB_LABEL.to_owned(), + reason: "invalid capsule xorb hash".to_owned(), + })?; + if let Some(origin) = origins.get(&parsed) { + selected.push((hash, parsed, entry, origin)); + } + } + if selected.is_empty() { + return Ok(()); + } + for (hash, _, _, _) in &selected { + if !source_shards.contains_key(hash.as_str()) { + return Err(MetadataError::CorruptObject { + path: CHUNK_INDEX_DB_LABEL.to_owned(), + reason: "capsule xorb candidate has no source shard root".to_owned(), + }); + } + } + let writer = RemoteIndexWriter::open(store, config, false, true).await?; + let result = async { + // Last-write heads choose the lowest canonical xorb and first occurrence. + for (hash, parsed, entry, origin) in selected { + let source_shard = source_shards.get(hash.as_str()).ok_or_else(|| { + MetadataError::Internal( + "admitted capsule candidate lost its source shard".to_owned(), + ) + })?; + let rows = crate::capsule_protocol::capsule_chunk_candidate_rows( + repo_prefix, + source_shard, + parsed, + entry, + origin, + )?; + writer.write_capsule_candidates(&rows).await?; + } + Ok::<_, MetadataError>(()) + } + .await; + let close = writer.close().await; + result?; + close +} + /// Long-lived writer for caller-bounded remote index batches. /// /// Writes buffer data without waiting for durability. Always await [`Self::close`] @@ -164,6 +231,33 @@ impl RemoteIndexWriter { .await } + /// Buffer a bounded set of capsule candidate rows; close remains the durability barrier. + async fn write_capsule_candidates(&self, rows: &[(Vec, Vec)]) -> Result<()> { + let db = self + .chunk_db + .as_ref() + .ok_or_else(|| MetadataError::Internal("chunk index was not opened".to_owned()))?; + for rows in rows.chunks(4096) { + let mut batch = slatedb::WriteBatch::new(); + for (key, value) in rows { + batch.put(key.as_slice(), value.as_slice()); + } + db.write_with_options( + batch, + &slatedb::config::WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .map_err(|source| MetadataError::SlateDbWrite { + db: CHUNK_INDEX_DB_LABEL.to_owned(), + source, + })?; + } + Ok(()) + } + /// Flush and close every database opened by this writer. /// /// Both closes are attempted. If both fail, return the file-index error. @@ -474,6 +568,34 @@ async fn open_writer( path: &str, db: &'static str, ) -> Result { + let fresh = match open_reader(Arc::clone(&store), path, db).await? { + None => true, + Some(reader) => { + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let validation = async { + let value = + reader + .get(&key) + .await + .map_err(|source| MetadataError::SlateDbRead { + db: db.to_owned(), + source, + })?; + validate_writer_format(value.as_deref(), db) + } + .await; + let close = reader + .close() + .await + .map_err(|source| MetadataError::SlateDbClose { + db: db.to_owned(), + source, + }); + validation?; + close?; + false + } + }; let l0_sst_size_bytes = usize::try_from(DEFAULT_CHUNK_INDEX_L0_SST_SIZE_BYTES) .map_err(|error| MetadataError::Internal(format!("L0 SST size unsupported: {error}")))?; let settings = slatedb::config::Settings { @@ -484,7 +606,7 @@ async fn open_writer( l0_sst_size_bytes, ..slatedb::config::Settings::default() }; - slatedb::Db::builder(ObjectPath::from(path), store) + let writer = slatedb::Db::builder(ObjectPath::from(path), store) .with_settings(settings) .build() .await @@ -492,7 +614,61 @@ async fn open_writer( db: db.to_owned(), path: path.to_owned(), source, - }) + })?; + if fresh { + let initialized = async { + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = writer + .get(&key) + .await + .map_err(|source| MetadataError::SlateDbRead { + db: db.to_owned(), + source, + })?; + if let Some(value) = value { + return validate_writer_format(Some(&value), db); + } + let mut batch = slatedb::WriteBatch::new(); + batch.put(key, crate::key_codec::METADB_FORMAT_VERSION.to_le_bytes()); + writer + .write_with_options( + batch, + &slatedb::config::WriteOptions { + await_durable: false, + ..Default::default() + }, + ) + .await + .map(|_| ()) + .map_err(|source| MetadataError::SlateDbWrite { + db: db.to_owned(), + source, + }) + } + .await; + if let Err(error) = initialized { + let _ = close_writer(Some(writer), db).await; + return Err(error); + } + } + Ok(writer) +} + +fn validate_writer_format(value: Option<&[u8]>, db: &str) -> Result<()> { + if value + != Some( + crate::key_codec::METADB_FORMAT_VERSION + .to_le_bytes() + .as_slice(), + ) + { + return Err(MetadataError::CorruptObject { + path: db.to_owned(), + reason: "metadata database is missing or has an unsupported canonical format marker" + .to_owned(), + }); + } + Ok(()) } async fn open_reader( @@ -790,3 +966,6 @@ mod tests { assert_eq!(config.chunk_index_path, ".crab/chunk_index_db/"); } } + +#[cfg(test)] +mod capsule_tests; diff --git a/crates/crab-metadata/src/remote_index/capsule_tests.rs b/crates/crab-metadata/src/remote_index/capsule_tests.rs new file mode 100644 index 000000000..a6e396a85 --- /dev/null +++ b/crates/crab-metadata/src/remote_index/capsule_tests.rs @@ -0,0 +1,86 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use crate::capsule_protocol::{PointerCatalog, XorbCatalogEntry, XorbChunkEntry}; +use crate::receipts::OriginReceipt; +use crab_xet::hash::MerkleHash; +use object_store::memory::InMemory; + +use super::{RemoteIndexConfig, write_capsule_chunk_candidates}; + +#[tokio::test] +async fn capsule_candidates_reject_xorbs_without_a_source_shard_root() { + let hash = MerkleHash::from([3; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + hash.hex(), + XorbCatalogEntry::new( + 100, + "4".repeat(64), + vec![XorbChunkEntry::new(MerkleHash::from([1; 32]).hex(), 9)], + ), + ) + .unwrap(); + let origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", hash.hex()), + hash.into(), + [5; 32], + 100, + Some("etag".to_owned()), + None, + ); + let result = write_capsule_chunk_candidates( + Arc::new(InMemory::new()), + &RemoteIndexConfig::for_repo("repo"), + "repo", + &catalog, + &HashMap::from([(hash, origin)]), + ) + .await; + assert!(result.is_err(), "an xorb descriptor alone is not a GC root"); +} + +#[tokio::test] +async fn capsule_candidates_writer_initializes_canonical_format() { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let writer = super::RemoteIndexWriter::open(Arc::clone(&store), &config, false, true) + .await + .unwrap(); + writer.close().await.unwrap(); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let key = crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION); + let value = reader.get(&key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!(value.as_deref(), Some(1u32.to_le_bytes().as_slice())); +} + +#[tokio::test] +async fn capsule_candidates_writer_rejects_noncanonical_existing_databases() { + for marker in [None, Some(2u32.to_le_bytes().to_vec()), Some(vec![1])] { + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("repo"); + let seed = slatedb::Db::open(config.chunk_index_path.clone(), Arc::clone(&store)) + .await + .unwrap(); + if let Some(marker) = marker { + seed.put( + crate::key_codec::encode_system_key(crate::key_codec::SYS_FORMAT_VERSION), + marker, + ) + .await + .unwrap(); + } + seed.close().await.unwrap(); + let result = super::RemoteIndexWriter::open(store, &config, false, true).await; + if let Ok(writer) = result { + writer.close().await.unwrap(); + panic!("invalid existing format was accepted"); + } + } +} From bda0e7ffd81ed9b8901f72a9b74beea4b86dd993 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:36:12 -0700 Subject: [PATCH 09/17] fix(push): bind capsule reuse to staged proofs --- crab/docs/design/add.md | 20 + crab/src/git/capsule_push.rs | 1 + crab/src/git/push.rs | 397 +++++++++------ .../src/git/push/tests/capsule_chunk_tests.rs | 480 ++++++++++++++++++ crab/src/git/xet_publication.rs | 101 +++- .../src/metadata/metadb/stores/chunk_index.rs | 43 +- .../src/capsule_protocol/chunk_candidates.rs | 17 +- .../chunk_candidates/tests.rs | 2 +- .../crab-metadata/src/capsule_protocol/mod.rs | 5 +- .../src/capsule_protocol/pointer.rs | 1 + .../src/capsule_protocol/pointer/paged.rs | 3 +- .../crab-metadata/src/capsule_protocol/run.rs | 4 + .../src/capsule_protocol/visibility/paged.rs | 3 + crates/crab-metadata/src/key_codec.rs | 8 + .../src/remote_index/capsule_tests.rs | 79 +++ 15 files changed, 973 insertions(+), 191 deletions(-) create mode 100644 crab/src/git/push/tests/capsule_chunk_tests.rs diff --git a/crab/docs/design/add.md b/crab/docs/design/add.md index 94426f02c..16fbe9e6e 100644 --- a/crab/docs/design/add.md +++ b/crab/docs/design/add.md @@ -212,6 +212,26 @@ source repository's current paged catalog, check each exact chunk position and size, then verify canonical origin identity. An absent source or changed physical descriptor cannot turn an index hint into authority. +Each capsule placement is also stored under its immutable placement digest. +Staged recipes retain that digest and the origin proof ID, so replacing a shared +chunk head does not erase an add's chosen assertion. Capsule push re-reads those +exact records, revalidates the source shard, and requires the recorded physical +origin identity before adopting remote-only chunks. Warm payload/index caches +cannot substitute for that identity check. The proof session is read-only and +closed before publication opens the shared index writer. + +The revalidator returns caller-owned source descriptors and original receipts +with the accepted candidates. Capsule preparation adopts those descriptors +without a full xorb GET or decompression. Retained metadata shares one catalog +byte allowance across source repositories; conflicting physical assertions fail +explicitly. Released manifest receipts still use full-body verification when +constructing a capsule descriptor that their older record does not contain. + +Canonical origin receipts belong to the trusted `global_prefix` of the same +authorization scope as the repository. Managed and ACL-scoped views use their +own supplied global root. Local staging and advisory caches only name candidates; +they cannot issue canonical receipts or certify an origin identity. + Direct publication and protected receive use the same bounded candidate encoder/writer. Protected promotion carries the canonical operation identity; client staging-prefix writes do not confer canonical origin proof. Every index diff --git a/crab/src/git/capsule_push.rs b/crab/src/git/capsule_push.rs index e0fbb8cc6..fc58d03c3 100644 --- a/crab/src/git/capsule_push.rs +++ b/crab/src/git/capsule_push.rs @@ -418,6 +418,7 @@ async fn run_inner( staging, caching_store, metrics, + &config.metadb, config.upload_concurrency, config.protected_push.is_none(), cancel, diff --git a/crab/src/git/push.rs b/crab/src/git/push.rs index 571615cfa..5e46f30aa 100644 --- a/crab/src/git/push.rs +++ b/crab/src/git/push.rs @@ -9970,7 +9970,7 @@ impl PushPipeline { &self, refs: &HashMap, ) -> Result> { - self.verify_xorb_refs_with_committed_receipts(refs, &HashMap::new()) + self.verify_xorb_refs_with_committed_receipts(refs, &HashMap::new(), false) .await } @@ -9978,6 +9978,7 @@ impl PushPipeline { &self, refs: &HashMap, committed_receipts: &HashMap, + require_committed_receipt: bool, ) -> Result> { use futures_util::StreamExt; @@ -10007,7 +10008,7 @@ impl PushPipeline { let cache = self.remote_full_xorb_ref_cache.lock().await; refs.values() .map(|xorb_ref| xorb_ref.xorb_hash) - .filter(|xorb_hash| cache.contains(xorb_hash)) + .filter(|xorb_hash| !require_committed_receipt && cache.contains(xorb_hash)) .collect::>() }; { @@ -10018,7 +10019,8 @@ impl PushPipeline { .entry(xorb_ref.xorb_hash) .or_default() .push((*chunk_hash, *xorb_ref)); - } else if chunk_ref_cache.contains(&RemoteChunkRefKey::new(*chunk_hash, *xorb_ref)) + } else if !require_committed_receipt + && chunk_ref_cache.contains(&RemoteChunkRefKey::new(*chunk_hash, *xorb_ref)) { verified.insert(*chunk_hash, *xorb_ref); cached_xorb_hashes.insert(xorb_ref.xorb_hash); @@ -10057,7 +10059,7 @@ impl PushPipeline { { let index_cache = self.remote_xorb_index_cache.lock().await; for xorb_hash in refs_by_xorb.keys().copied() { - if let Some(index) = index_cache.get(&xorb_hash) { + if !require_committed_receipt && let Some(index) = index_cache.get(&xorb_hash) { verified_xorbs.insert(xorb_hash, index.clone()); } else { planned_hashes.push(xorb_hash); @@ -10185,6 +10187,14 @@ impl PushPipeline { return None; } Some(async move { + // Capsule assertions bind the exact origin selected at add. + // Cached payload validity cannot certify a replaced object. + if require_committed_receipt && !committed_receipt.as_ref().zip(object_meta.as_ref()) + .is_some_and(|(receipt, meta)| committed_origin_receipt_matches_remote( + receipt, &xorb_hash, &path, &index, meta, + )) { + return Ok((Vec::new(), None, None, Vec::new())); + } if is_full_cached { let xorb_info = build_xorb_info_from_remote_index(&index)?; return Ok((valid_refs, Some((xorb_hash, xorb_info)), None, Vec::new())); @@ -10361,19 +10371,19 @@ impl PushPipeline { async fn revalidate_add_plan_existing_candidates( &self, planned: &HashMap, - ) -> Result> { + ) -> Result { if planned.is_empty() { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); } let chunk_store = { let guard = self.metadb.lock().await; let Some(guard) = guard.as_ref() else { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); }; match guard.chunk_index().await { Ok(store) => store, Err(error) if error.is_metadb_read_only_uninitialized() => { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); } Err(error) => return Err(error), } @@ -10382,8 +10392,47 @@ impl PushPipeline { .iter() .map(|(chunk_hash, candidate)| (*chunk_hash, candidate.placement_id)) .collect::>(); - let remote_candidates = tokio::time::timeout( - GLOBAL_CHUNK_LOOKUP_BUDGET, + let deadline = tokio::time::Instant::now() + GLOBAL_CHUNK_LOOKUP_BUDGET; + let (capsule, capsule_indexed, capsule_matched) = + tokio::time::timeout_at(deadline, async { + let mut candidates = chunk_store + .capsule_candidates_by_id(&exact_candidates) + .await?; + let indexed = candidates.placements.len(); + candidates.placements.retain(|hash, placement| { + planned + .get(&MerkleHash::from(*hash)) + .is_some_and(|candidate| { + candidate.placement_id == placement.placement_id() + && candidate.origin_proof_id == placement.origin_proof_id + && candidate.xorb_ref + == XorbRef { + xorb_hash: MerkleHash::from(placement.xorb_hash), + chunk_index: placement.chunk_index, + uncompressed_size: placement.uncompressed_size, + } + }) + }); + let matched = candidates.placements.len(); + let verified = self.verify_capsule_chunk_candidates(candidates).await?; + Ok::<_, CrabError>((verified, indexed, matched)) + }) + .await + .map_err(|_| { + CrabError::NetworkTransient(object_store::Error::Generic { + store: "chunk_index_db", + source: Box::new(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "staged capsule-proof revalidation budget exhausted", + )), + }) + })??; + let exact_candidates = exact_candidates + .into_iter() + .filter(|(hash, _)| !capsule.candidates.contains_key(hash)) + .collect::>(); + let remote_candidates = tokio::time::timeout_at( + deadline, chunk_store.get_committed_candidates_by_id_batch(&exact_candidates), ) .await @@ -10432,26 +10481,30 @@ impl PushPipeline { let committed_receipts = self.validate_committed_chunk_receipts(&refs).await; let validated = self.verified_committed_chunk_receipts.lock().await.clone(); let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &committed_receipts) + .verify_xorb_refs_with_committed_receipts(&refs, &committed_receipts, false) .await?; debug!( planned = planned.len(), - indexed = indexed_candidates, - matched = refs.len(), + indexed = indexed_candidates + capsule_indexed, + matched = refs.len() + capsule_matched, generation_verified = validated.len(), - payload_verified = verified.len(), + capsule_source_verified = capsule.candidates.len(), + payload_verified = verified.len() + capsule.candidates.len(), placement_mismatches, origin_proof_mismatches, "revalidated staged remote placement proofs" ); - Ok(verified - .into_iter() - .filter(|(chunk_hash, _)| { - placement_ids - .get(chunk_hash) - .is_some_and(|placement_id| validated.get(chunk_hash) == Some(placement_id)) - }) - .collect()) + let mut output = capsule; + output + .candidates + .extend(verified.into_iter().filter_map(|(hash, _)| { + let placement = placement_ids.get(&hash)?; + (validated.get(&hash) == Some(placement)) + .then(|| planned.get(&hash).copied()) + .flatten() + .map(|candidate| (hash, candidate)) + })); + Ok(output) } async fn prove_all_origin_xorbs_for_publish(&self) -> Result<()> { @@ -10960,7 +11013,7 @@ impl PushPipeline { // executor stack. This path owns several repository-sized maps; // heap-pinning the future avoids inflating the default test stack. match Box::pin(self.revalidate_add_plan_existing_candidates(&candidate_refs)).await { - Ok(refs) => refs, + Ok(verified) => verified.into_refs(), Err(CrabError::Cancelled) => return Err(CrabError::Cancelled), Err(error) => return Err(error), } @@ -15021,8 +15074,17 @@ impl PushPipeline { chunk_store: &crate::metadata::ChunkIndexStore, ) -> Result> { let candidates = chunk_store.capsule_candidates(hashes).await?; + self.verify_capsule_chunk_candidates(candidates) + .await + .map(|verified| verified.candidates) + } + + async fn verify_capsule_chunk_candidates( + &self, + candidates: crab_metadata::capsule_protocol::CapsuleChunkCandidates, + ) -> Result { let Some(store) = &self.store else { - return Ok(HashMap::new()); + return Ok(VerifiedRemoteCandidates::default()); }; #[derive(Default)] struct SourceCandidates<'a> { @@ -15040,6 +15102,14 @@ impl PushPipeline { let mut refs = HashMap::new(); let mut origins = HashMap::new(); let mut proofs = HashMap::new(); + let mut bindings: HashMap< + MerkleHash, + ( + crab_metadata::capsule_protocol::XorbCatalogEntry, + crab_metadata::receipts::OriginReceipt, + ), + > = HashMap::new(); + let mut retained_bytes = 0_u64; for (repo, source) in by_repo { check_cancelled(&self.cancel)?; let layout = crab_storage::StoreLayout::with_global_prefix( @@ -15078,6 +15148,49 @@ impl PushPipeline { { continue; } + let entry = catalog.xorbs().get(&xorb_hash.hex()).ok_or_else(|| { + CrabError::Internal( + "validated capsule placement lost its descriptor".to_owned(), + ) + })?; + if let Some((prior, origin)) = bindings.get(&xorb_hash) { + // Observation time may differ; physical assertions for the + // same immutable xorb cannot disagree or certify one another. + if prior != entry + || origin.etag != anchor.origin.etag + || origin.object_version != anchor.origin.object_version + || origin.payload_digest != anchor.origin.payload_digest + || origin.size != anchor.origin.size + { + return Err(CrabError::CorruptObject { + path: path.to_string(), + reason: "conflicting canonical xorb assertions".to_owned(), + }); + } + } else { + let metadata_bytes = std::mem::size_of_val(entry.chunks()) as u64 + + entry + .chunks() + .iter() + .map(|chunk| chunk.hash().len() as u64) + .sum::() + + entry.body_digest().len() as u64 + + serde_json::to_vec(&anchor.origin) + .map_err(|error| CrabError::Io(std::io::Error::other(error)))? + .len() as u64; + retained_bytes = + retained_bytes.checked_add(metadata_bytes).ok_or_else(|| { + CrabError::Internal("selected xorb metadata size overflows".to_owned()) + })?; + if retained_bytes > crab_metadata::capsule_protocol::MAX_CATALOG_BYTES { + return Err(CrabError::CorruptObject { + path: "capsule chunk candidates".to_owned(), + reason: "selected xorb metadata exceeds the catalog byte limit" + .to_owned(), + }); + } + bindings.insert(xorb_hash, (entry.clone(), anchor.origin.clone())); + } let hash = MerkleHash::from(placement.chunk_hash); refs.insert( hash, @@ -15091,12 +15204,18 @@ impl PushPipeline { proofs.insert(hash, (placement.placement_id(), placement.origin_proof_id)); } } - // Source catalogs establish commitment; the existing origin verifier - // binds payload/index checks to the exact object version in the receipt. + // Source catalogs establish commitment. Receipts belong to the trusted + // scope's canonical global index; fresh provider identity binds their + // verified bytes, while cache or staged IDs alone cannot certify reuse. let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &origins) + .verify_xorb_refs_with_committed_receipts(&refs, &origins, true) .await?; - Ok(verified + let accepted_xorbs = verified + .values() + .map(|reference| reference.xorb_hash) + .collect::>(); + bindings.retain(|hash, _| accepted_xorbs.contains(hash)); + let candidates = verified .into_iter() .filter_map(|(hash, xorb_ref)| { let (placement_id, origin_proof_id) = proofs.get(&hash).copied()?; @@ -15109,7 +15228,11 @@ impl PushPipeline { }, )) }) - .collect()) + .collect(); + Ok(VerifiedRemoteCandidates { + candidates, + xorbs: bindings, + }) } async fn lookup_proven_remote_chunks_for_add( @@ -15177,7 +15300,7 @@ impl PushPipeline { .validate_committed_chunk_candidates(&refs, candidates) .await; let verified = self - .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins) + .verify_xorb_refs_with_committed_receipts(&refs, &validated.origins, false) .await?; capsule.extend(verified.into_iter().filter_map(|(chunk_hash, xorb_ref)| { let (placement_id, origin_proof_id) = proof_ids.get(&chunk_hash).copied()?; @@ -15285,7 +15408,7 @@ impl PushPipeline { let committed_receipts = Box::pin(self.validate_committed_chunk_receipts(&hits)).await; let refs = self - .verify_xorb_refs_with_committed_receipts(&hits, &committed_receipts) + .verify_xorb_refs_with_committed_receipts(&hits, &committed_receipts, false) .await?; let stale_hashes = hits .keys() @@ -17057,6 +17180,77 @@ impl PushPipeline { }) } } +#[derive(Debug, Default)] +pub(crate) struct VerifiedRemoteCandidates { + pub candidates: HashMap, + pub xorbs: HashMap< + MerkleHash, + ( + crab_metadata::capsule_protocol::XorbCatalogEntry, + crab_metadata::receipts::OriginReceipt, + ), + >, +} + +impl VerifiedRemoteCandidates { + fn into_refs(self) -> HashMap { + self.candidates + .into_iter() + .map(|(hash, candidate)| (hash, candidate.xorb_ref)) + .collect() + } +} + +// Capsule preparation owns this read-only session through revalidation and close. +// Reusing the planner's verifier keeps released manifest and capsule proofs on +// the same origin checks without opening a writer during payload preparation. +pub(crate) async fn revalidate_staged_remote_chunks( + metadb: &crate::core::config::MetaDbTomlConfig, + layout: &crab_storage::StoreLayout, + planned: &HashMap, + cancel: &CancellationToken, +) -> Result { + if planned.is_empty() { + return Ok(VerifiedRemoteCandidates::default()); + } + check_cancelled(cancel)?; + let store = Store::from_storage(layout.store().clone()); + let router = StoreLayout::with_global_prefix( + store.clone(), + layout.repo_prefix().to_owned(), + layout.global_prefix().to_owned(), + ); + let config = PushConfig { + metadb: metadb.clone(), + ..Default::default() + }; + let pipeline = PushPipeline::new( + config, + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + cancel.clone(), + None, + ); + pipeline.install_metadb(build_push_metadb_guard_with_object_store( + &store, None, &router, None, metadb, true, + )); + let result = Box::pin(pipeline.revalidate_add_plan_existing_candidates(planned)).await; + let guard = { pipeline.metadb.lock().await.take() }; + let closed = match guard { + Some(guard) => guard.close().await, + None => Ok(()), + }; + let verified = result?; + closed?; + check_cancelled(cancel)?; + Ok(verified) +} + /// Run the push pipeline for a batch of push specs. /// /// This is the main entry point called by the remote helper when Git @@ -20705,6 +20899,7 @@ mod tests { struct RangeCountingStore { inner: object_store::memory::InMemory, range_gets: Arc, + full_xorb_gets: AtomicUsize, head_e_tag: Option>>, } @@ -20713,6 +20908,7 @@ mod tests { Self { inner: object_store::memory::InMemory::new(), range_gets, + full_xorb_gets: AtomicUsize::new(0), head_e_tag: None, } } @@ -20724,6 +20920,7 @@ mod tests { Self { inner: object_store::memory::InMemory::new(), range_gets, + full_xorb_gets: AtomicUsize::new(0), head_e_tag: Some(head_e_tag), } } @@ -20760,6 +20957,9 @@ mod tests { options: GetOptions, ) -> object_store::Result { let is_head = options.head; + if !is_head && options.range.is_none() && location.as_ref().contains("/xorbs/") { + self.full_xorb_gets.fetch_add(1, Ordering::Relaxed); + } if options.range.is_some() { self.range_gets.fetch_add(1, Ordering::Relaxed); } @@ -22162,134 +22362,7 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { - use crab_staging::push_plan::ExistingChunkLookup; - use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; - - let _git_env = CleanGitEnvGuard::new(); - let source = tempfile::tempdir().unwrap(); - let git = |args: &[&str]| { - let output = Command::new("git") - .args(args) - .current_dir(source.path()) - .env("GIT_AUTHOR_NAME", "Test") - .env("GIT_AUTHOR_EMAIL", "test@example.invalid") - .env("GIT_COMMITTER_NAME", "Test") - .env("GIT_COMMITTER_EMAIL", "test@example.invalid") - .output() - .unwrap(); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - }; - git(&["init", "-b", "main"]); - let staging_root = source.path().join("staging"); - let staging = crab_staging::StagingArea::open(staging_root.clone()) - .await - .unwrap(); - let staged = stage_reader_streaming( - &mut std::io::Cursor::new(b"shared capsule payload"), - std::path::Path::new("model.bin"), - &staging, - StreamStageProgress::default(), - &CancellationToken::new(), - ) - .await - .unwrap(); - let chunks = staging - .recipe_page(&staged.recipe, 0) - .unwrap() - .chunks - .into_iter() - .map(|chunk| (chunk.chunk_hash, chunk.len)) - .collect::>(); - staging - .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) - .unwrap(); - let pointer = crab_types::pointer::Pointer { - file_hash: staged.file_hash, - size: staged.size, - shard_hint: None, - }; - std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); - git(&["add", "model.bin"]); - git(&["commit", "-m", "shared payload"]); - staging.close().await.unwrap(); - let staging = Arc::new( - crab_staging::StagingAreaReadOnly::open(staging_root) - .await - .unwrap(), - ); - let (store, source_router) = test_store_router("repos/source"); - let layout = - crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); - crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") - .await - .unwrap(); - let config = PushConfig { - git_dir: Some(source.path().join(".git")), - ..Default::default() - }; - let (result, _) = super::super::capsule_push::run( - &config, - &[crate::git::remote_helper::PushSpec { - force: false, - src: "refs/heads/main".to_owned(), - dst: "refs/heads/main".to_owned(), - }], - &store, - &source_router, - None, - &[], - Some(&staging), - None, - None, - &CancellationToken::new(), - ) - .await - .unwrap(); - assert!(result.all_ok(), "source publication failed: {result:?}"); - drop(staging); - - let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); - let pipeline = PushPipeline::new( - PushConfig::default(), - Vec::new(), - Some(store.clone()), - None, - None, - router.repo_prefix().to_owned(), - router.clone(), - None, - CancellationToken::new(), - None, - ); - let cache_root = tempfile::tempdir().unwrap(); - let mut metadata = crate::core::config::MetaDbTomlConfig::default(); - metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); - pipeline.install_metadb(build_push_metadb_guard( - &store, &router, None, &metadata, true, - )); - // No advisory caches or source-repository state can manufacture a hit. - // This exercises the real publisher-to-classifier authority boundary. - let classifier = AddRemoteChunkClassifier { - pipeline, - remote_chunk_index: tokio::sync::OnceCell::new(), - candidate_cache: None, - candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), - candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), - }; - let candidates = classifier.lookup_existing_candidates(&chunks).await; - classifier.close().await; - let candidates = candidates.unwrap(); - assert!( - !candidates.is_empty() && candidates.iter().all(Option::is_some), - "committed source chunks must be reusable from a cold consumer: {candidates:?}" - ); - } + mod capsule_chunk_tests; fn add_classifier_with_private_cache( store: &Store, @@ -30830,7 +30903,11 @@ mod tests { range_gets.store(0, Ordering::Relaxed); let verified = pipeline - .verify_xorb_refs_with_committed_receipts(&refs, &HashMap::from([(xorb_hash, receipt)])) + .verify_xorb_refs_with_committed_receipts( + &refs, + &HashMap::from([(xorb_hash, receipt)]), + false, + ) .await .expect("verify refs from committed origin receipt"); diff --git a/crab/src/git/push/tests/capsule_chunk_tests.rs b/crab/src/git/push/tests/capsule_chunk_tests.rs new file mode 100644 index 000000000..0f60b530e --- /dev/null +++ b/crab/src/git/push/tests/capsule_chunk_tests.rs @@ -0,0 +1,480 @@ +use super::*; + +struct FirstCandidateOnly(Arc); + +#[async_trait::async_trait] +impl crab_staging::push_plan::ExistingChunkLookup for FirstCandidateOnly { + async fn lookup_existing_candidates( + &self, + chunks: &[(MerkleHash, u64)], + ) -> crab_staging::Result>> { + let candidates = crab_staging::push_plan::ExistingChunkLookup::lookup_existing_candidates( + self.0.as_ref(), + chunks, + ) + .await?; + Ok(candidates + .into_iter() + .enumerate() + .map( + |(index, candidate)| { + if index == 0 { candidate } else { None } + }, + ) + .collect()) + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_push_exposes_committed_chunks_to_cold_cross_repository_add() { + use crab_staging::push_plan::ExistingChunkLookup; + use crab_staging::stream::{StreamStageProgress, stage_reader_streaming}; + + let _git_env = CleanGitEnvGuard::new(); + let content = test_deterministic_bytes(0x4cab, 2 * 1024 * 1024); + let source = tempfile::tempdir().unwrap(); + let git = |args: &[&str]| { + let output = Command::new("git") + .args(args) + .current_dir(source.path()) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + git(&["init", "-b", "main"]); + let staging_root = source.path().join("staging"); + let staging = crab_staging::StagingArea::open(staging_root.clone()) + .await + .unwrap(); + let staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &staging, + StreamStageProgress::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + let chunks = staging + .recipe_page(&staged.recipe, 0) + .unwrap() + .chunks + .into_iter() + .map(|chunk| (chunk.chunk_hash, chunk.len)) + .collect::>(); + staging + .delegate_publication(&[(&staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + let pointer = crab_types::pointer::Pointer { + file_hash: staged.file_hash, + size: staged.size, + shard_hint: None, + }; + std::fs::write(source.path().join("model.bin"), pointer.serialize()).unwrap(); + git(&["add", "model.bin"]); + git(&["commit", "-m", "shared payload"]); + staging.close().await.unwrap(); + let staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(staging_root) + .await + .unwrap(), + ); + let reads = Arc::new(RangeCountingStore::new(Arc::new(AtomicUsize::new(0)))); + let store = Store::new(reads.clone()); + let source_router = StoreLayout::new(store.clone(), "repos/source".to_owned()); + let layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/source".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + let config = PushConfig { + git_dir: Some(source.path().join(".git")), + ..Default::default() + }; + let (result, _) = crate::git::capsule_push::run( + &config, + &[crate::git::remote_helper::PushSpec { + force: false, + src: "refs/heads/main".to_owned(), + dst: "refs/heads/main".to_owned(), + }], + &store, + &source_router, + None, + &[], + Some(&staging), + None, + None, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(result.all_ok(), "source publication failed: {result:?}"); + drop(staging); + + let router = StoreLayout::new(store.clone(), "repos/consumer".to_owned()); + let pipeline = PushPipeline::new( + PushConfig::default(), + Vec::new(), + Some(store.clone()), + None, + None, + router.repo_prefix().to_owned(), + router.clone(), + None, + CancellationToken::new(), + None, + ); + let cache_root = tempfile::tempdir().unwrap(); + let mut metadata = crate::core::config::MetaDbTomlConfig::default(); + metadata.chunk_index.local_path = Some(cache_root.path().join("chunks.sqlite")); + pipeline.install_metadb(build_push_metadb_guard( + &store, &router, None, &metadata, true, + )); + // No advisory caches or source-repository state can manufacture a hit. + // This exercises the real publisher-to-classifier authority boundary. + let classifier = Arc::new(AddRemoteChunkClassifier { + pipeline, + remote_chunk_index: tokio::sync::OnceCell::new(), + candidate_cache: None, + candidate_cache_hits: std::sync::atomic::AtomicU64::new(0), + candidate_cache_misses: std::sync::atomic::AtomicU64::new(0), + }); + let candidates = classifier.lookup_existing_candidates(&chunks).await; + let consumer_staging_root = cache_root.path().join("consumer-staging"); + let consumer_staging = crab_staging::StagingArea::open(consumer_staging_root.clone()) + .await + .unwrap(); + let remote_staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &consumer_staging, + StreamStageProgress { + existing_lookup: Some(classifier.clone()), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await; + let partial_root = cache_root.path().join("partial-staging"); + let partial_staging = crab_staging::StagingArea::open(partial_root.clone()) + .await + .unwrap(); + let partial_staged = stage_reader_streaming( + &mut std::io::Cursor::new(content.as_slice()), + std::path::Path::new("model.bin"), + &partial_staging, + StreamStageProgress { + existing_lookup: Some(Arc::new(FirstCandidateOnly(classifier.clone()))), + ..Default::default() + }, + &CancellationToken::new(), + ) + .await; + classifier.close().await; + let partial_staged = partial_staged.unwrap(); + partial_staging + .delegate_publication(&[(&partial_staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + partial_staging.close().await.unwrap(); + let remote_staged = remote_staged.unwrap(); + consumer_staging + .delegate_publication(&[(&remote_staged.batch_id, std::path::Path::new("model.bin"))]) + .unwrap(); + consumer_staging.close().await.unwrap(); + let candidates = candidates.unwrap(); + assert!( + !candidates.is_empty() && candidates.iter().all(Option::is_some), + "committed source chunks must be reusable from a cold consumer: {candidates:?}" + ); + + let planned = chunks + .iter() + .zip(&candidates) + .filter_map(|((hash, _), candidate)| candidate.map(|candidate| (*hash, candidate))) + .collect::>(); + let root = crab_metadata::capsule_protocol::load_root(&layout) + .await + .unwrap(); + let catalog = crab_metadata::capsule_protocol::load_pointer_catalog_from_root(&layout, &root) + .await + .unwrap(); + let mut origins = HashMap::new(); + for candidate in planned.values() { + let path = layout.xorb_path(&candidate.xorb_ref.xorb_hash); + let (bytes, identity) = layout.store().get_with_etag(&path).await.unwrap(); + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).unwrap(); + let mut origin = crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + candidate.xorb_ref.xorb_hash.into(), + parser.payload_digest(), + bytes.len() as u64, + identity.e_tag, + identity.version, + ); + // Change the head's proof identity while retaining the actual committed source. + origin.proven_at_unix_secs += 10; + origins.insert(candidate.xorb_ref.xorb_hash, origin); + } + crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), + "repos/source", + &catalog, + &origins, + ) + .await + .unwrap(); + let consumer_layout = + crab_storage::StoreLayout::new(store.as_storage().clone(), "repos/consumer".to_owned()); + let verified = revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + verified.candidates.len(), + planned.len(), + "old exact proof survives a newer shared head" + ); + + for field in ["placement", "origin", "position"] { + let mut forged = planned.clone(); + for candidate in forged.values_mut() { + match field { + "placement" => candidate.placement_id = [9; 32], + "origin" => candidate.origin_proof_id = [9; 32], + "position" => candidate.xorb_ref.chunk_index += 1, + _ => unreachable!(), + } + } + assert!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &forged, + &CancellationToken::new() + ) + .await + .unwrap() + .candidates + .is_empty(), + "accepted forged {field}" + ); + } + + let consumer_base = crab_write::capsule_protocol::initialize( + &consumer_layout, + &"2".repeat(64), + "refs/heads/main", + ) + .await + .unwrap(); + let consumer_staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(consumer_staging_root) + .await + .unwrap(), + ); + reads.full_xorb_gets.store(0, Ordering::Relaxed); + let (delta, stats) = crate::git::xet_publication::prepare_delta( + &consumer_layout, + &consumer_base, + &[crab_types::pointer::Pointer { + file_hash: remote_staged.file_hash, + size: remote_staged.size, + shard_hint: None, + }], + Some(&consumer_staging), + None, + None, + &metadata, + 1, + true, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + (delta.files().len(), stats.xorbs_uploaded), + (1, 0), + "remote-only staging must publish without repacking" + ); + assert_eq!( + reads.full_xorb_gets.load(Ordering::Relaxed), + 0, + "an exact, current source/origin proof must not download the reused xorb body" + ); + drop(consumer_staging); + let partial_staging = Arc::new( + crab_staging::StagingAreaReadOnly::open(partial_root) + .await + .unwrap(), + ); + reads.full_xorb_gets.store(0, Ordering::Relaxed); + let (_, partial_stats) = crate::git::xet_publication::prepare_delta( + &consumer_layout, + &consumer_base, + &[crab_types::pointer::Pointer { + file_hash: partial_staged.file_hash, + size: partial_staged.size, + shard_hint: None, + }], + Some(&partial_staging), + None, + None, + &metadata, + 1, + true, + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!( + ( + partial_stats.xorbs_uploaded, + reads.full_xorb_gets.load(Ordering::Relaxed) + ), + (0, 0), + "partial add-time hints must reuse the authenticated requested closure without packing or downloading" + ); + drop(partial_staging); + + let source_key = layout.capsule_root_path(); + let (source_bytes, _) = layout.store().get_with_etag(&source_key).await.unwrap(); + layout.store().delete(&source_key).await.unwrap(); + let unrooted = revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new(), + ) + .await + .unwrap(); + layout.store().put(&source_key, source_bytes).await.unwrap(); + assert!( + unrooted.candidates.is_empty(), + "physical bytes alone cannot replace a source root" + ); + + for candidate in planned.values() { + let key = layout.xorb_path(&candidate.xorb_ref.xorb_hash); + let (bytes, previous) = layout.store().get_with_etag(&key).await.unwrap(); + // Bypass the immutable adapter only to inject a provider-side replacement. + let replaced = layout + .store() + .inner() + .put(&key, bytes.into()) + .await + .unwrap(); + assert_ne!( + (previous.e_tag, previous.version), + (replaced.e_tag, replaced.version), + "the fault must actually change the provider identity" + ); + } + assert!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &planned, + &CancellationToken::new() + ) + .await + .unwrap() + .candidates + .is_empty(), + "a replacement identity cannot reuse the original proof" + ); + let verified_refs = verified.into_refs(); + let cached = classifier + .pipeline + .verify_xorb_refs_with_committed_receipts(&verified_refs, &origins, true) + .await + .unwrap(); + assert!( + cached.is_empty(), + "warm proofs cannot certify the replacement identity" + ); + + // A fresh attestation for one occurrence must not certify another + // occurrence's stale provider identity in the same immutable xorb. + let mut fresh_origins = HashMap::new(); + for hash in planned + .values() + .map(|candidate| candidate.xorb_ref.xorb_hash) + .collect::>() + { + let key = layout.xorb_path(&hash); + let (bytes, identity) = layout.store().get_with_etag(&key).await.unwrap(); + let parser = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).unwrap(); + parser.verify_payload_digest().unwrap(); + parser.verify_all_chunks().unwrap(); + fresh_origins.insert( + hash, + crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + key.to_string(), + hash.into(), + parser.payload_digest(), + bytes.len() as u64, + identity.e_tag, + identity.version, + ), + ); + } + crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(layout.store().inner()), + &crab_metadata::remote_index::RemoteIndexConfig::for_repo("repos/source"), + "repos/source", + &catalog, + &fresh_origins, + ) + .await + .unwrap(); + let guard = build_push_metadb_guard(&store, &router, None, &metadata, true); + let chunk_store = guard.chunk_index().await.unwrap(); + let hashes = planned.keys().copied().collect::>(); + let fresh = classifier + .pipeline + .lookup_capsule_chunks_for_add(&hashes, &chunk_store) + .await; + guard.close().await.unwrap(); + let fresh = fresh.unwrap(); + assert_eq!( + fresh.len(), + planned.len(), + "replacement proof fixture must be independently valid" + ); + assert!( + fresh.len() > 1, + "mixed-origin coverage needs distinct chunks in one xorb" + ); + let mut mixed = planned.clone(); + let (hash, candidate) = fresh.into_iter().next().unwrap(); + mixed.insert(hash, candidate); + assert!( + matches!( + revalidate_staged_remote_chunks( + &metadata, + &consumer_layout, + &mixed, + &CancellationToken::new(), + ) + .await, + Err(CrabError::CorruptObject { .. }) + ), + "conflicting immutable-xorb identities must not certify one another" + ); +} diff --git a/crab/src/git/xet_publication.rs b/crab/src/git/xet_publication.rs index b9b2c57d1..266fa1d6f 100644 --- a/crab/src/git/xet_publication.rs +++ b/crab/src/git/xet_publication.rs @@ -42,6 +42,7 @@ pub(crate) async fn prepare_delta( staging: Option<&Arc>, caching_store: Option<&crab_cache_store::CachingStore>, metrics: Option<&Metrics>, + metadb: &crate::core::config::MetaDbTomlConfig, upload_concurrency: usize, publish_gc_roots: bool, cancel: &CancellationToken, @@ -66,7 +67,8 @@ pub(crate) async fn prepare_delta( }, ) .await?; - let lookup = view.pointer_catalog_lookup(layout, 2 * 1024 * 1024 * 1024)?; + let lookup = + view.pointer_catalog_lookup(layout, crab_metadata::capsule_protocol::MAX_CATALOG_BYTES)?; drop(view); let mut pointer_sizes = BTreeMap::::new(); @@ -139,7 +141,7 @@ pub(crate) async fn prepare_delta( let mut prepared_files = Vec::with_capacity(unresolved.len()); let mut prepared_candidates = Vec::::new(); let mut candidate_indices = HashMap::::new(); - let mut external_candidates = HashMap::>::new(); + let mut staged_candidates = HashMap::new(); for (file_hash, file_size) in unresolved { check_cancelled(cancel)?; @@ -208,11 +210,8 @@ pub(crate) async fn prepare_delta( expected_size ))); } - let refs = external_candidates - .entry(candidate.xorb_ref.xorb_hash) - .or_default(); - match refs.insert(chunk_hash, candidate.xorb_ref) { - Some(existing) if existing != candidate.xorb_ref => { + match staged_candidates.insert(chunk_hash, candidate) { + Some(existing) if existing != candidate => { return Err(CrabError::StagingCorrupt(format!( "remote xorb authority disagrees for chunk {}", chunk_hash.hex() @@ -265,15 +264,78 @@ pub(crate) async fn prepare_delta( drop(base_chunks); drop(needed_chunks); - let mut external_candidates = external_candidates - .into_iter() - .filter(|(hash, refs)| { - !xorb_entries.contains_key(hash) - && refs - .keys() - .any(|chunk_hash| !placements.contains_key(chunk_hash)) - }) - .collect::>(); + // The recipe retains the exact add-time IDs. Re-read their immutable + // records and current source roots before trusting any remote-only bytes. + staged_candidates.retain(|hash, _| !placements.contains_key(hash)); + let planned_existing = staged_candidates.len(); + let verified_staged = + super::push::revalidate_staged_remote_chunks(metadb, layout, &staged_candidates, cancel) + .await?; + tracing::debug!( + planned_existing, + stale_existing = planned_existing.saturating_sub(verified_staged.candidates.len()), + global_existing = 0, + "capsule staged remote proof adoption complete", + ); + let mut reused_xorbs = verified_staged.xorbs.len(); + let requested = requested_chunks + .iter() + .map(String::as_str) + .collect::>(); + for (hash, (entry, origin)) in verified_staged.xorbs { + if xorb_entries + .get(&hash) + .is_some_and(|existing| existing != &entry) + { + return Err(CrabError::CorruptObject { + path: layout.xorb_path(&hash).to_string(), + reason: "verified xorb differs from the captured destination catalog".to_owned(), + }); + } + // The authenticated closure also proves requested chunks whose rebuildable + // heads were absent during add. Expand only outgoing occurrences, so an + // incomplete global index cannot force repacking the same verified bytes. + for (index, chunk) in entry.chunks().iter().enumerate() { + if requested.contains(chunk.hash()) { + let chunk_hash = parse_merkle_hash(chunk.hash(), "verified catalog chunk")?; + placements.entry(chunk_hash).or_insert(ChunkPlacement { + chunk_hash, + xorb_hash: hash, + chunk_index: u32::try_from(index).map_err(|_| { + CrabError::Internal("verified xorb has too many chunks".to_owned()) + })?, + uncompressed_size: chunk.uncompressed_size(), + }); + } + } + xorb_entries.insert(hash, entry); + origins.insert(hash, origin); + } + drop(requested); + let mut external_candidates = HashMap::>::new(); + for (hash, candidate) in verified_staged.candidates { + let xorb_ref = candidate.xorb_ref; + if xorb_entries.contains_key(&xorb_ref.xorb_hash) { + // Exact source membership and current provider identity already + // certify this descriptor. Re-reading payload would download a + // whole reused file despite the committed origin receipt. + placements.entry(hash).or_insert(ChunkPlacement { + chunk_hash: hash, + xorb_hash: xorb_ref.xorb_hash, + chunk_index: xorb_ref.chunk_index, + uncompressed_size: xorb_ref.uncompressed_size, + }); + } else { + // Manifest receipts shipped in v1.2.4 lack capsule body descriptors. + // Full-body verification supplies that value when importing them + // into capsule publication. + external_candidates + .entry(xorb_ref.xorb_hash) + .or_default() + .insert(hash, xorb_ref); + } + } + let mut external_candidates = external_candidates.into_iter().collect::>(); external_candidates.sort_by_key(|(hash, _)| hash.hex()); let external_reads = futures_util::stream::iter(external_candidates.into_iter().enumerate().map( @@ -303,7 +365,6 @@ pub(crate) async fn prepare_delta( verified_external.push(result?); } verified_external.sort_by_key(|(ordinal, _)| *ordinal); - let mut reused_xorbs = 0_usize; for (_, verified) in verified_external { let Some((xorb_hash, entry, xorb_placements, origin)) = verified else { continue; @@ -1237,6 +1298,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1316,6 +1378,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1397,6 +1460,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), 1, false, &CancellationToken::new(), @@ -1514,6 +1578,7 @@ mod tests { Some(&staging), None, None, + &Default::default(), concurrency, false, &CancellationToken::new(), @@ -1557,6 +1622,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), @@ -1620,6 +1686,7 @@ mod tests { None, None, None, + &Default::default(), 4, true, &CancellationToken::new(), diff --git a/crab/src/metadata/metadb/stores/chunk_index.rs b/crab/src/metadata/metadb/stores/chunk_index.rs index 89dd9597a..0b1d91563 100644 --- a/crab/src/metadata/metadb/stores/chunk_index.rs +++ b/crab/src/metadata/metadb/stores/chunk_index.rs @@ -108,27 +108,58 @@ impl ChunkIndexStore { pub(crate) async fn capsule_candidates( &self, hashes: &[MerkleHash], + ) -> Result { + use crab_metadata::capsule_protocol::capsule_chunk_head_key; + let requests = hashes + .iter() + .map(|hash| (*hash, capsule_chunk_head_key(&(*hash).into()), None)) + .collect::>(); + self.capsule_candidate_records(&requests).await + } + + /// Read immutable capsule placements selected during add, independent of newer heads. + pub(crate) async fn capsule_candidates_by_id( + &self, + candidates: &[(MerkleHash, [u8; 32])], + ) -> Result { + let requests = candidates + .iter() + .map(|(hash, id)| { + ( + *hash, + crab_metadata::key_codec::capsule_chunk_placement_key(id), + Some(*id), + ) + }) + .collect::>(); + self.capsule_candidate_records(&requests).await + } + + async fn capsule_candidate_records( + &self, + requests: &[(MerkleHash, [u8; 33], Option<[u8; 32]>)], ) -> Result { use crab_metadata::capsule_protocol::{ CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, - capsule_chunk_head_key, }; let mut output = CapsuleChunkCandidates::default(); // Stay below Db's dense range-scan threshold: unrelated chunk history // must not turn an add-time point batch into a scan of the global index. - for hashes in hashes.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { - let keys = hashes + for requests in requests.chunks(GET_BATCH_TIER_CHUNK_LIMIT) { + let keys = requests .iter() - .map(|hash| Bytes::copy_from_slice(&capsule_chunk_head_key(&(*hash).into()))) + .map(|(_, key, _)| Bytes::copy_from_slice(key)) .collect::>(); let values = self.db.get_batch(&keys).await?; let mut anchor_ids = HashSet::new(); - for (hash, value) in hashes.iter().zip(values) { + for ((hash, _, expected_id), value) in requests.iter().zip(values) { let Some(value) = value else { continue; }; let placement = CommittedChunkPlacement::decode(&value)?; - if placement.chunk_hash != <[u8; 32]>::from(*hash) { + if placement.chunk_hash != <[u8; 32]>::from(*hash) + || expected_id.is_some_and(|id| placement.placement_id() != id) + { return Err(CrabError::StagingCorrupt( "capsule candidate hash differs from index key".to_owned(), )); diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs index 9553a0c83..7ba9ebd52 100644 --- a/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates.rs @@ -1,11 +1,15 @@ //! Rebuildable chunk hints whose source authority is a capsule catalog. -use std::collections::{BTreeMap, HashMap}; +#[cfg(any(test, feature = "remote-index"))] +use std::collections::BTreeMap; +use std::collections::HashMap; use serde::{Deserialize, Serialize}; use super::XorbCatalogEntry; use crate::error::{MetadataError, Result}; -use crate::receipts::{CommittedChunkPlacement, OriginReceipt, RECEIPT_SCHEMA_VERSION}; +#[cfg(any(test, feature = "remote-index"))] +use crate::receipts::RECEIPT_SCHEMA_VERSION; +use crate::receipts::{CommittedChunkPlacement, OriginReceipt}; use crab_xet::hash::MerkleHash; pub use crate::key_codec::{capsule_chunk_anchor_key, capsule_chunk_head_key}; @@ -110,6 +114,7 @@ pub struct CapsuleChunkCandidates { pub anchors: HashMap<[u8; 32], CapsuleChunkAnchor>, } +#[cfg(any(test, feature = "remote-index"))] /// Encode one bounded xorb's candidates after full verification and ref publication. pub(crate) fn capsule_chunk_candidate_rows( repo_prefix: &str, @@ -152,10 +157,14 @@ pub(crate) fn capsule_chunk_candidate_rows( }); } for (hash, placement) in placements { + let encoded = placement.encode()?.to_vec(); + // Preserve the exact add-time assertion when another publication replaces + // the shared head. The placement digest also commits to the chunk key. rows.push(( - capsule_chunk_head_key(&hash).to_vec(), - placement.encode()?.to_vec(), + crate::key_codec::capsule_chunk_placement_key(&placement.placement_id()).to_vec(), + encoded.clone(), )); + rows.push((capsule_chunk_head_key(&hash).to_vec(), encoded)); } Ok(rows) } diff --git a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs index aff1436c4..c0f82c629 100644 --- a/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/chunk_candidates/tests.rs @@ -38,7 +38,7 @@ fn capsule_candidates_preserve_distinct_digests_and_first_occurrence() { anchor.body_digest, anchor.origin.payload_digest ), - (2, 0, entry.body_digest().to_owned(), origin.payload_digest) + (3, 0, entry.body_digest().to_owned(), origin.payload_digest) ); } diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index 07c1359c9..de338fc22 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -70,8 +70,8 @@ pub use plan::{ #[cfg(feature = "storage")] pub use pointer::PointerCatalogLookup; pub use pointer::{ - FileCatalogEntry, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, XorbCatalogEntry, - XorbChunkEntry, + FileCatalogEntry, MAX_CATALOG_BYTES, PointerCatalog, PointerCatalogAssembly, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }; pub use ref_head::{ CAPSULE_REF_COMPACTION_FAN_IN, CAPSULE_REF_ROLLUP_WINDOW, CapsuleRefHead, CapsuleRefState, @@ -129,4 +129,5 @@ pub use chunk_candidates::{ CapsuleChunkAnchor, CapsuleChunkCandidates, capsule_chunk_anchor_key, capsule_chunk_head_key, }; +#[cfg(feature = "remote-index")] pub(crate) use chunk_candidates::capsule_chunk_candidate_rows; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer.rs b/crates/crab-metadata/src/capsule_protocol/pointer.rs index 2b359154a..85255d410 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer.rs @@ -10,6 +10,7 @@ use crate::validation::validate_content_hash; const POINTER_CATALOG_VERSION: u32 = 1; pub(super) mod paged; +pub use paged::MAX_CATALOG_BYTES; #[cfg(feature = "storage")] pub use paged::PointerCatalogLookup; diff --git a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs index b7e3e9518..66e273430 100644 --- a/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/pointer/paged.rs @@ -10,7 +10,8 @@ pub use reader::PointerCatalogLookup; const MAGIC: &[u8; 8] = b"CRBPCP01"; const TRAILER_BYTES: usize = 12; const MAX_DESCRIPTOR_BYTES: usize = 4096; -pub(in crate::capsule_protocol) const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; +/// Maximum byte length of one encoded pointer catalog. +pub const MAX_CATALOG_BYTES: u64 = 2 * 1024 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/crab-metadata/src/capsule_protocol/run.rs b/crates/crab-metadata/src/capsule_protocol/run.rs index e20d5eef9..c07b638c0 100644 --- a/crates/crab-metadata/src/capsule_protocol/run.rs +++ b/crates/crab-metadata/src/capsule_protocol/run.rs @@ -144,6 +144,7 @@ pub struct CapsuleControl { #[derive(Debug, Clone)] enum ControlVisibility { + #[cfg(any(feature = "storage", test))] Complete(std::sync::Arc), #[cfg(feature = "storage")] Paged(std::sync::Arc), @@ -768,6 +769,9 @@ impl CapsuleControl { pub fn visibility_delta(&self) -> Result> { match &self.visibility { None => Ok(None), + #[cfg(not(any(feature = "storage", test)))] + Some(never) => match *never {}, + #[cfg(any(feature = "storage", test))] Some(ControlVisibility::Complete(delta)) => Ok(Some(delta)), #[cfg(feature = "storage")] Some(ControlVisibility::Paged(_)) => Err(corrupt( diff --git a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs index 3ae87ee33..105e8cad5 100644 --- a/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs +++ b/crates/crab-metadata/src/capsule_protocol/visibility/paged.rs @@ -224,6 +224,7 @@ impl Descriptor { } } +#[cfg(feature = "storage")] impl VisibilityEditHeader { /// Return the prior tip used by the edit. pub fn old_oid(&self) -> Option<&str> { @@ -260,7 +261,9 @@ impl VisibilityEditHeader { _ => false, } } +} +impl VisibilityEditHeader { fn decode(key: &[u8], bytes: &[u8], body_bytes: u64) -> Result { let row: Self = serde_json::from_slice(bytes) .map_err(|error| MetadataError::CapsuleVisibilityRecord { source: error })?; diff --git a/crates/crab-metadata/src/key_codec.rs b/crates/crab-metadata/src/key_codec.rs index b0436ec5a..e95d2d79f 100644 --- a/crates/crab-metadata/src/key_codec.rs +++ b/crates/crab-metadata/src/key_codec.rs @@ -13,6 +13,7 @@ //! - `0x05 || anchor_id[32]` — immutable source-shard anchor. //! - `0x06 || hash[32]` — point-readable capsule chunk candidate. //! - `0x07 || anchor_id[32]` — immutable capsule source/origin anchor. +//! - `0x08 || placement_id[32]` — immutable capsule chunk placement. //! - `0xFF || b"sys:" || name` — system key (format_version, epoch, //! created_at, gc_generation). //! @@ -50,6 +51,7 @@ pub const PREFIX_SOURCE_ANCHOR: u8 = 0x05; /// mistake capsule provenance for a manifest-generation assertion. const PREFIX_CAPSULE_CHUNK_HEAD: u8 = 0x06; const PREFIX_CAPSULE_CHUNK_ANCHOR: u8 = 0x07; +const PREFIX_CAPSULE_CHUNK_PLACEMENT: u8 = 0x08; /// Encode a point-readable capsule candidate key. #[must_use] @@ -63,6 +65,12 @@ pub fn capsule_chunk_anchor_key(hash: &[u8; 32]) -> [u8; 33] { encode_proof_record_key(PREFIX_CAPSULE_CHUNK_ANCHOR, hash) } +/// Encode an immutable capsule placement independently of its replaceable head. +#[must_use] +pub fn capsule_chunk_placement_key(id: &[u8; 32]) -> [u8; 33] { + encode_proof_record_key(PREFIX_CAPSULE_CHUNK_PLACEMENT, id) +} + /// Prefix byte for the `sys:*` system-key namespace. Chosen as `0xFF` /// so it sorts after every content key and is trivial to filter out of /// content-only scans. diff --git a/crates/crab-metadata/src/remote_index/capsule_tests.rs b/crates/crab-metadata/src/remote_index/capsule_tests.rs index a6e396a85..269567268 100644 --- a/crates/crab-metadata/src/remote_index/capsule_tests.rs +++ b/crates/crab-metadata/src/remote_index/capsule_tests.rs @@ -84,3 +84,82 @@ async fn capsule_candidates_writer_rejects_noncanonical_existing_databases() { } } } + +#[tokio::test] +async fn capsule_exact_placement_survives_head_replacement() { + use crate::capsule_protocol::ShardCatalogEntry; + use crate::receipts::CommittedChunkPlacement; + + let store: Arc = Arc::new(InMemory::new()); + let config = RemoteIndexConfig::for_repo("source"); + let xorb = MerkleHash::from([3; 32]); + let chunk = MerkleHash::from([1; 32]); + let mut catalog = PointerCatalog::new(); + catalog + .insert_xorb( + xorb.hex(), + XorbCatalogEntry::new( + 100, + "4".repeat(64), + vec![XorbChunkEntry::new(chunk.hex(), 9)], + ), + ) + .unwrap(); + catalog + .insert_shard("2".repeat(64), ShardCatalogEntry::new(20, vec![xorb.hex()])) + .unwrap(); + let mut origin = OriginReceipt::new( + "canonical-origin".to_owned(), + format!(".crab/xorbs/03/{}", xorb.hex()), + xorb.into(), + [5; 32], + 100, + Some("first".to_owned()), + None, + ); + let mut placements = Vec::new(); + for repo in ["source", "newer-source"] { + write_capsule_chunk_candidates( + Arc::clone(&store), + &config, + repo, + &catalog, + &HashMap::from([(xorb, origin.clone())]), + ) + .await + .unwrap(); + let reader = super::open_reader( + Arc::clone(&store), + &config.chunk_index_path, + "chunk_index_db", + ) + .await + .unwrap() + .unwrap(); + let head = reader + .get(crate::key_codec::capsule_chunk_head_key(&chunk.into())) + .await + .unwrap() + .unwrap(); + reader.close().await.unwrap(); + placements.push(CommittedChunkPlacement::decode(&head).unwrap()); + origin.etag = Some("second".to_owned()); + } + let first = &placements[0]; + let mut key = vec![0x08]; + key.extend_from_slice(&first.placement_id()); + let reader = super::open_reader(store, &config.chunk_index_path, "chunk_index_db") + .await + .unwrap() + .unwrap(); + let exact = reader.get(key).await.unwrap(); + reader.close().await.unwrap(); + assert_eq!( + ( + placements[0] != placements[1], + exact.map(|value| CommittedChunkPlacement::decode(&value).unwrap()) + ), + (true, Some(first.clone())), + "a changed shared head must not erase the proof selected by an earlier add", + ); +} From ab6eb383b66e3ef527b47c55ea75a971e29d542f Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:36:37 -0700 Subject: [PATCH 10/17] test(git): verify capsule reuse events exactly --- .../docs/architecture/indexed-remote-reads.md | 40 ++++++++++++ .../e2e/run_add_commit_push_rustfs_smoke.py | 55 +++++++++++++--- .../scripts/e2e/test_add_commit_push_proof.py | 65 +++++++++++++++++++ 3 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 crab/scripts/e2e/test_add_commit_push_proof.py diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index a13bda5eb..52411228a 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2266,3 +2266,43 @@ remote-helper boundary passes date text through, while wire fetch uses a normalized timestamp. Fixing that adapter and qualifying dated boundaries against native Git are required; changing the expected message alone would hide the missing behavior. No full CLI E2E pass is claimed from this receipt. + + +## Capsule chunk authority and exact staged reuse in progress + +PR #593 publishes rebuildable global chunk candidates after capsule ref +publication. Each candidate names a source shard in the source repository's +current authenticated catalog and a canonical origin receipt in the caller's +trusted global scope. A physical xorb alone is insufficient: its catalog shard +must retain the payload through the existing GC contract. Direct CLI publication +and protected receive share the metadata encoder and writer. Protected receive's +80 native-Git/in-memory tests pass; this is not real-S3 qualification. + +The follow-up stores each placement under its immutable digest as well as the +replaceable chunk head. Push re-reads the exact add-time placement and origin +IDs, validates source membership, and requires current provider identity even +with warm payload caches. Its read-only proof session closes before publication +opens the global writer. Verified source descriptors and original receipts stay +caller-owned, with one retained-metadata byte allowance across source repositories. +Conflicting descriptors or physical assertions fail explicitly. + +A regression measured one redundant full xorb GET after otherwise successful +source/origin proof. Adopting the authenticated descriptor removes that GET: +the first before/after pair fails at one versus zero body reads, then passes the +unchanged zero-read bound. Final-source validation passes all 10 focused CLI/Xet tests, including partial +add-time hints and mixed stale/fresh origin assertions. The cold source fixture +uses native Git and real staging with an in-memory object store. +Released v1.2.4 manifest receipts lack these capsule descriptors and still require +full-body verification when imported into capsule publication. + +The dedicated cross-repository smoke oracle now requires one complete capsule +source-proof event and its matching adoption event. It checks exact planned, +indexed, matched, source-verified and payload-verified counts, with zero stale, +mismatched or globally reclassified chunks. Five oracle tests pass; the original +substring oracle rejected valid capsule proofs and accepted counters assembled +from different events. This representation correction is not product E2E proof. +Payload-only run tests pass (28), and strict minimal metadata Clippy passes. + +Current-source S3 cross-repository add/push/cold reconstruction, the shared +`crab-remote` preparation consumer, final 10-GiB qualification, history scaling, +provider latency, lifecycle and full blame semantics remain open gates. diff --git a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py index 5e9f70b77..fd9cf3c17 100755 --- a/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py +++ b/crab/scripts/e2e/run_add_commit_push_rustfs_smoke.py @@ -102,6 +102,44 @@ class SmokeReport: updated_at: str = "" +def capsule_staged_proof_complete(stderr: str, count: int) -> bool: + """Require complete direct reuse evidence for the capsule duplicate fixture.""" + if type(count) is not int or count <= 0: + return False + proof_message = "revalidated staged remote placement proofs" + adoption_message = "capsule staged remote proof adoption complete" + events = [] + for line in stderr.splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + fields = record.get("fields") if isinstance(record, dict) else None + if isinstance(fields, dict) and fields.get("message") in ( + proof_message, adoption_message + ): + events.append(fields) + # This fixture makes one push with one direct-proof batch. Counts from + # different batches cannot jointly prove that every planned chunk survived. + if len(events) != 2: + return False + proof, adoption = events + if proof.get("message") != proof_message or adoption.get("message") != adoption_message: + return False + expected = { + "planned": count, "indexed": count, "matched": count, + "capsule_source_verified": count, "generation_verified": 0, + "payload_verified": count, "placement_mismatches": 0, + "origin_proof_mismatches": 0, + } + adopted = {"planned_existing": count, "stale_existing": 0, "global_existing": 0} + return all( + type(event.get(key)) is int and event[key] == value + for event, counters in [(proof, expected), (adoption, adopted)] + for key, value in counters.items() + ) + + def utc_now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -2031,13 +2069,7 @@ def run_cross_repository_remote_duplicate_case(self) -> None: remote_chunk_count = inventory["recipe_remote_chunks"] self.check( f"{case_name}-push-revalidates-staged-proof-directly", - "revalidated staged remote placement proofs" in push_stderr - and f'"planned":{remote_chunk_count}' in push_stderr - and f'"matched":{remote_chunk_count}' in push_stderr - and f'"generation_verified":{remote_chunk_count}' in push_stderr - and f'"payload_verified":{remote_chunk_count}' in push_stderr - and '"stale_existing":0' in push_stderr - and '"global_existing":0' in push_stderr, + capsule_staged_proof_complete(push_stderr, remote_chunk_count), {"remote_chunks": remote_chunk_count}, ) self.head_repository_state(consumer_prefix) @@ -2052,12 +2084,19 @@ def run_cross_repository_remote_duplicate_case(self) -> None: ) clone_dir = self.run_root / case_name / "consumer-clone" + clone_cache = self.run_root / case_name / "consumer-clone-cache" + self.check(f"{case_name}-clone-cache-starts-empty", not clone_cache.exists()) + clone_env = {"CRAB_CACHE_DIR": str(clone_cache)} self.run_cmd( f"{case_name} consumer clone", [self.crab_bin, "clone", consumer_url, str(clone_dir), "--jsonl"], self.run_root, + extra_env=clone_env, + ) + self.run_crab( + clone_dir, ["hydrate", "--all"], name=f"{case_name} hydrate", + extra_env=clone_env, ) - self.run_crab(clone_dir, ["hydrate", "--all"], name=f"{case_name} hydrate") self.check( f"{case_name}-fresh-hydrate-is-byte-identical", sha256_file(clone_dir / "model.bin") == expected_sha256, diff --git a/crab/scripts/e2e/test_add_commit_push_proof.py b/crab/scripts/e2e/test_add_commit_push_proof.py new file mode 100644 index 000000000..e9ced8bb2 --- /dev/null +++ b/crab/scripts/e2e/test_add_commit_push_proof.py @@ -0,0 +1,65 @@ +"""Regression tests for the capsule smoke's direct-proof oracle.""" + +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from run_add_commit_push_rustfs_smoke import capsule_staged_proof_complete + + +class CapsuleProofTest(unittest.TestCase): + def proof(self): + return { + "message": "revalidated staged remote placement proofs", + "planned": 7, "indexed": 7, "matched": 7, + "generation_verified": 0, "capsule_source_verified": 7, + "payload_verified": 7, "placement_mismatches": 0, + "origin_proof_mismatches": 0, + } + + def adoption(self): + return { + "message": "capsule staged remote proof adoption complete", + "planned_existing": 7, "stale_existing": 0, "global_existing": 0, + } + + def log(self, *events): + return "\n".join(json.dumps({"fields": e}, separators=(",", ":")) for e in events) + + def test_accepts_complete_capsule_evidence(self): + self.assertTrue(capsule_staged_proof_complete(self.log(self.proof(), self.adoption()), 7)) + + def test_rejects_incomplete_or_invalid_counters(self): + for key in self.proof(): + if key == "message": + continue + for value in [None, -1, True, "7", 8]: + with self.subTest(key=key, value=value): + event = {**self.proof(), key: value} + self.assertFalse(capsule_staged_proof_complete(self.log(event, self.adoption()), 7)) + for key in ["planned_existing", "stale_existing", "global_existing"]: + with self.subTest(key=key): + event = {**self.adoption(), key: 1} + self.assertFalse(capsule_staged_proof_complete(self.log(self.proof(), event), 7)) + + def test_rejects_counts_spread_across_events(self): + first = {**self.proof(), "generation_verified": 7, "matched": 0} + second = {**first, "matched": 7, "payload_verified": 0} + self.assertFalse(capsule_staged_proof_complete(self.log(first, second, self.adoption()), 7)) + + def test_rejects_missing_reordered_or_ambiguous_events(self): + for events in [[], [self.proof()], [self.adoption()], + [self.adoption(), self.proof()], + [self.proof(), self.proof(), self.adoption()]]: + with self.subTest(events=events): + self.assertFalse(capsule_staged_proof_complete(self.log(*events), 7)) + + def test_rejects_non_positive_expected_count(self): + for count in [0, -1, True]: + self.assertFalse(capsule_staged_proof_complete(self.log(self.proof(), self.adoption()), count)) + + +if __name__ == "__main__": + unittest.main() From b2d752a7d34f5a7daba6c9883aaaaf8bb413646e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 05:46:57 -0700 Subject: [PATCH 11/17] docs: record exact reuse qualification --- .../docs/architecture/indexed-remote-reads.md | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 52411228a..8a50df839 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2301,8 +2301,26 @@ indexed, matched, source-verified and payload-verified counts, with zero stale, mismatched or globally reclassified chunks. Five oracle tests pass; the original substring oracle rejected valid capsule proofs and accepted counters assembled from different events. This representation correction is not product E2E proof. -Payload-only run tests pass (28), and strict minimal metadata Clippy passes. - -Current-source S3 cross-repository add/push/cold reconstruction, the shared -`crab-remote` preparation consumer, final 10-GiB qualification, history scaling, -provider latency, lifecycle and full blame semantics remain open gates. +Payload-only run tests pass (28), candidate/key-codec/writer tests pass (19), +existing origin/range siblings pass (12), and strict minimal and remote-index +metadata Clippy pass. The production CLI binary build also passes. + +The current-source RustFS cross-repository fixture passes 34 commands and 32 +checks at 4 MiB (`capsule-exact-proof-r2-c6a202ae`, code head `098859c02f1`). +The source worktree is clean, and its recorded source hashes remain unchanged +throughout the run. Consumer add retains neither segment nor prepared payload; +push revalidates all 58 exact placements with zero stale, mismatched or globally +reclassified chunks and uploads no new xorb. Add and push each read two xorb +ranges totaling 2,658 bytes (footer and chunk metadata), with no full xorb GET. +A separate empty clone cache then performs a real remote payload GET during +hydration and produces the original SHA-256. This is one local debug-profile +correctness fixture, not a latency or history-scaling qualification. + +The binary SHA-256 is +`5002c1c59f8e2d6d3fa82abed1c5686c67d46feb93f975c0c1d76c1930f3dc70`; +the report SHA-256 is +`36d6ec1b744a2f5545a9a998710271bc2fa092ceadf756dd56c95acf75eeff2f`. + +The shared `crab-remote` preparation consumer, final 10-GiB qualification, +history scaling, provider latency, lifecycle and full blame semantics remain +open gates. From 05d31170328c4675a6f563868b4a81cb6805bb8b Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 06:21:44 -0700 Subject: [PATCH 12/17] fix(remote): bind capsule preparation to proofs --- Cargo.lock | 1 + .../docs/architecture/indexed-remote-reads.md | 44 ++ crates/crab-remote/Cargo.toml | 1 + crates/crab-remote/README.md | 13 + crates/crab-remote/src/prepare.rs | 481 ++------------ crates/crab-remote/src/prepare/capsule.rs | 495 ++++++++++++++ .../crab-remote/tests/capsule_preparation.rs | 621 ++++++++++++++++++ 7 files changed, 1226 insertions(+), 430 deletions(-) create mode 100644 crates/crab-remote/src/prepare/capsule.rs create mode 100644 crates/crab-remote/tests/capsule_preparation.rs diff --git a/Cargo.lock b/Cargo.lock index aa7fddfa3..dc6ffd938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2539,6 +2539,7 @@ dependencies = [ "serde", "serde_json", "sha1 0.10.6", + "slatedb", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index 8a50df839..eb032baae 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2324,3 +2324,47 @@ the report SHA-256 is The shared `crab-remote` preparation consumer, final 10-GiB qualification, history scaling, provider latency, lifecycle and full blame semantics remain open gates. + + +## Shared capsule preparation in progress + +The shared `Prepared::upload_capsule` caller in HTTP receive now uses captured +file, shard and xorb lookups instead of materializing unrelated historical +catalogs. An ordinary Git upload with no new Crab pointer performs zero catalog +reads. Selected pointer and prepared-content closures preserve size, membership +and descriptor conflict checks. Capsule-specific implementation lives in +`crates/crab-remote/src/prepare/capsule.rs`; its public API remains under +`crab_remote::prepare`. + +Four regressions failed before the change and pass afterward: one redundant +historical-catalog read for a Git-only upload; Xet-word-order formatting of a raw +body digest; acceptance of rewritten xorb/shard scratch bodies after inspection; +and no shared chunk candidate index after a successful native-content commit. +The initial Git-only fixture sent an all-history pack containing an old pointer; +its corrected incremental pack excludes the advertised tip before measuring. +No weakened assertion or production failure is inferred from that fixture issue. + +Six public-API integration tests and strict all-target Clippy pass. The tests use +native Git packs, real Xet artifacts, explicit publication leases, exact origin +and source-anchor checks, complete file reconstruction, and a noncanonical index +fault that must leave committed refs successful. The SlateDB test dependency uses +the same pinned 0.15.0 release already used by metadata; the lockfile adds only +that test dependency edge. Production dependency versions do not change. + +The dedicated RustFS public-API test passes for both newly created and occupied +xorb keys. It uses native Git preparation, real S3 publication, cold source and +origin-anchor checks, and complete file-hash reconstruction. The historical +pointer integration also accepts the correct declared size and rejects a +conflicting one. The final source passes strict all-target Clippy and the +no-default-feature check. + +Eight HTTP receive/fault tests pass. Three dedicated real-RustFS cells also pass: +the public native-content API, native Git push/fetch and browser editing over the +HTTP listener, and the six receive-fault/restart scenarios. Source hashes remain +unchanged throughout these runs. The API test-binary SHA-256 is +`fe8a75004fd4ef7ec0946b999d32a829b83b28f94b51317a9c34e9990e9ebf5d`; the HTTP test-binary SHA-256 is +`92318648278160dd57f96f35dbcffc01f44799a75a43b84442251965c07fef51`. These are small correctness fixtures, separate from the required +large-file, growth and latency cells. + +The final combined 10-GiB matrix, unrelated-history/ref/frontier scaling, broader +lifecycle/provider proof and fuller blame semantics still prevent completion. diff --git a/crates/crab-remote/Cargo.toml b/crates/crab-remote/Cargo.toml index 67659fc1a..bf30ec429 100644 --- a/crates/crab-remote/Cargo.toml +++ b/crates/crab-remote/Cargo.toml @@ -41,6 +41,7 @@ futures-util = { workspace = true, optional = true } object_store = { workspace = true, optional = true } [dev-dependencies] +slatedb = "=0.15.0" sha1 = "0.10" tracing-subscriber = "0.3" async-trait = { workspace = true } diff --git a/crates/crab-remote/README.md b/crates/crab-remote/README.md index 1ba312482..5c391f6d2 100644 --- a/crates/crab-remote/README.md +++ b/crates/crab-remote/README.md @@ -58,6 +58,19 @@ The pinned-view outcome also reports the last inventory published by that pass, or the input inventory if neither phase published. It deliberately excludes later ref-head updates instead of rescanning them for CLI statistics. +Capsule preparation resolves outgoing pointer files and only the prepared +shard/xorb identities against a captured paged catalog. Ordinary Git and LFS-only +uploads do not read historical Xet catalog pages. Selected dependencies retain +complete closure, size and immutable-descriptor conflict checks. + +Scratch ownership does not freeze artifact bytes. Bounded reads revalidate the +inspected raw Blake3 body identity before upload; Xet protocol identities retain +their distinct word-ordered representation. Capsule commitment retains exact +verified create/read identities with their source catalog, publishes rebuildable +chunk hints only after proven ref visibility, and preserves a committed outcome +if hint-index maintenance fails. Uncertain publication does not publish hints. +Caller-owned ref leases and GC fences cover this complete operation. + `objects` owns bounded tree reads, deterministic tree edits and encoding, commit encoding, and Git object identities. `prepare` owns normalized packs and hydrated content artifacts used by remote commits and recovery. File bodies and diff --git a/crates/crab-remote/src/prepare.rs b/crates/crab-remote/src/prepare.rs index 4d8ba63bf..7210a1e20 100644 --- a/crates/crab-remote/src/prepare.rs +++ b/crates/crab-remote/src/prepare.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashMap}, fs::File, io::{BufRead, BufReader}, path::PathBuf, @@ -20,7 +20,8 @@ use crab_git::{ }; use crab_metadata::{ capsule_protocol::{ - FileCatalogEntry, PointerCatalog, ShardCatalogEntry, XorbCatalogEntry, XorbChunkEntry, + FileCatalogEntry, PointerCatalog, PointerCatalogLookup, ShardCatalogEntry, + XorbCatalogEntry, XorbChunkEntry, }, git_object_locator::GitObjectKind, git_visibility::{GitCatalogVisibilityIndex, GitVisibilityEdit, GitVisibilityIndex}, @@ -28,8 +29,11 @@ use crab_metadata::{ use crab_remote_git::{OperationContext, OperationKind, RemoteGitRepository}; use gix_hash::ObjectId; use gix_object::Kind; +use tokio::io::AsyncReadExt; use tokio_util::sync::CancellationToken; +mod capsule; + /// Failure while validating and preparing incoming Git objects. #[derive(Debug, thiserror::Error)] pub enum Error { @@ -125,6 +129,41 @@ impl ContentArtifact { pub const fn size(&self) -> u64 { self.size } + + // Scratch ownership does not freeze file bytes. Revalidate the inspected + // identity with a bounded read before publishing content or its receipt. + async fn read_verified(&self, maximum: u64, cancel: &CancellationToken) -> Result { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + if self.size > maximum { + return Err(Error::Content( + "prepared artifact exceeds its byte limit".to_owned(), + )); + } + let mut bytes = Vec::new(); + tokio::fs::File::open(&self.path) + .await? + .take(self.size + 1) + .read_to_end(&mut bytes) + .await?; + let bytes = Bytes::from(bytes); + let size = self.size; + let hash = self.body_hash; + let bytes = tokio::task::spawn_blocking(move || { + if bytes.len() as u64 != size || blake3::hash(&bytes).as_bytes() != &hash { + return Err(Error::Content( + "prepared artifact changed after inspection".to_owned(), + )); + } + Ok(bytes) + }) + .await??; + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + Ok(bytes) + } } /// One Crab pointer supplied by a prepared reconstruction shard. @@ -236,15 +275,14 @@ impl PreparedContent { { return Err(Error::Content("shard identity is invalid".to_owned())); } - let bytes = tokio::fs::read(&artifact.path).await?; - if bytes.len() as u64 != artifact.size - || blake3::hash(&bytes).as_bytes() != &artifact.body_hash - || <[u8; 32]>::from(crab_xet::hash::compute_data_hash(&bytes)) - != artifact.protocol_hash + let bytes = artifact + .read_verified(crab_xet::shard::MAX_PUSH_SHARD_SIZE_BYTES, cancel) + .await?; + if <[u8; 32]>::from(crab_xet::hash::compute_data_hash(&bytes)) != artifact.protocol_hash { return Err(Error::Content("shard binding changed".to_owned())); } - shard_bodies.insert(artifact.protocol_hash, Bytes::from(bytes)); + shard_bodies.insert(artifact.protocol_hash, bytes); } let mut seen_files = std::collections::BTreeSet::new(); let mut used_shards = std::collections::BTreeSet::new(); @@ -330,7 +368,9 @@ impl PreparedContent { crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), XorbCatalogEntry::new( artifact.size, - crab_xet::hash::MerkleHash::from(artifact.body_hash).hex(), + blake3::Hash::from_bytes(artifact.body_hash) + .to_hex() + .to_string(), inspected .chunks .iter() @@ -428,6 +468,8 @@ pub struct CapsuleArtifacts { edits: Vec, packs: Vec, sections: Vec, + pointer_delta: PointerCatalog, + origins: HashMap, changes_namespace: bool, } @@ -809,427 +851,6 @@ impl Prepared { shards, }) } - - /// Upload artifacts for one captured capsule-protocol repository view. - /// - /// The caller must hold ref leases and GC fences from before opening - /// `view` through commitment. This validates all historical dependencies - /// against that view, uploads same-publication Xet content before Git - /// visibility, and retains the exact root snapshot for per-ref CAS. - pub async fn upload_capsule( - &self, - view: &crab_read::capsule_protocol::CapsuleRepositoryView, - limits: crab_read::dependency_proof::DependencyProofLimits, - cancel: &CancellationToken, - ) -> Result { - let started = Instant::now(); - let repository_refs = self - .repository - .refs() - .entries - .iter() - .map(|reference| (reference.name.clone(), reference.target.to_string())) - .collect::>(); - if repository_refs != *view.refs() - || self.repository.generation() != view.root().root().generation() - { - return Err(Error::Request( - "Capsule view differs from the prepared repository version", - )); - } - let lookup = view.pointer_catalog_lookup(&self.layout, 2 * 1024 * 1024 * 1024)?; - let local = self.content.as_ref().map(PreparedContent::pointer_catalog); - let mut files = self - .plan - .pointers() - .iter() - .filter_map(|dependency| match &dependency.pointer { - crab_git::pointer_detect::PointerKind::Crab(pointer) => { - Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) - } - _ => None, - }) - .collect::>(); - files.extend( - local - .into_iter() - .flat_map(|catalog| catalog.files().keys().cloned()), - ); - let files = files.into_iter().collect::>(); - let shards = local - .into_iter() - .flat_map(|catalog| catalog.shards().keys().cloned()) - .collect::>(); - let xorbs = local - .into_iter() - .flat_map(|catalog| catalog.xorbs().keys().cloned()) - .collect::>(); - // Local artifacts can collide with historical identities even when - // none of their files are present. Select those keys and the outgoing - // file closure together; unrelated history must stay cold. - let base_catalog = lookup - .dependencies(self.layout.store(), &files, &shards, &xorbs) - .await?; - let prepared_files = self - .content - .as_ref() - .map(|content| { - content - .files - .iter() - .map(|file| file.file_hash) - .collect::>() - }) - .unwrap_or_default(); - crab_read::dependency_proof::verify_capsule_dependencies_except_crab( - &self.layout, - &base_catalog, - self.plan.pointers(), - &prepared_files, - limits, - cancel, - ) - .await - .map_err(|error| Error::Dependency(Box::new(error)))?; - - let pointer_delta = match &self.content { - Some(content) => { - upload_capsule_content(&self.layout, &base_catalog, content, cancel).await? - } - None => PointerCatalog::new(), - }; - let mut packs = Vec::new(); - if let Some(pack) = &self.pack { - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let (pack_bytes, index, reverse_index, locator) = tokio::try_join!( - tokio::fs::read(pack.pack_path()), - tokio::fs::read(pack.index_path()), - tokio::fs::read(pack.reverse_path()), - tokio::fs::read(pack.kinds_path()), - )?; - let external_delta_bases = pack - .external_delta_bases() - .iter() - .map(ToString::to_string) - .collect(); - packs.push( - crab_metadata::capsule_protocol::CapsuleGitPack::new_with_external_delta_bases( - Bytes::from(pack_bytes), - Bytes::from(index), - Bytes::from(reverse_index), - Bytes::from(locator), - pack.git_sha1().to_string(), - u64::from(pack.object_count()), - external_delta_bases, - )?, - ); - } - let mut sections = Vec::with_capacity(2); - if !pointer_delta.is_empty() { - sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( - crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta, - pointer_delta.encode_delta()?, - )); - } - if !self.visibility.is_empty() { - let visibility = crab_metadata::capsule_protocol::CapsuleVisibilityDelta::new( - self.visibility.clone(), - )?; - sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( - crab_metadata::capsule_protocol::CapsuleSectionKind::VisibilityDelta, - visibility.encode()?, - )); - } - let edits = self - .updates - .iter() - .map(|update| { - crab_metadata::capsule_protocol::CapsuleRefEdit::new( - update.name.clone(), - update.old.map(|oid| oid.to_string()), - update.new.map(|oid| oid.to_string()), - self.plan - .peeled() - .get(&update.name) - .map(ToString::to_string), - ) - }) - .collect::>(); - let changes_namespace = edits - .iter() - .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); - tracing::info!( - phase = "capsule_input", - packs = packs.len(), - elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, - "Git receive phase completed" - ); - Ok(CapsuleArtifacts { - layout: self.layout.clone(), - base: view.root_snapshot().clone(), - edits, - packs, - sections, - changes_namespace, - }) - } -} - -impl CapsuleArtifacts { - /// Commit through independently mutable ref heads and one capsule authority. - /// - /// Revalidate authorization and repository policy immediately before this - /// call. The caller must retain its original ref leases and GC fences until - /// the returned outcome is known and all lease cleanup has completed. - pub async fn commit( - self, - plan_id: Option<&str>, - namespace_ttl: Duration, - cancel: &CancellationToken, - ) -> Result { - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let transaction = match plan_id { - Some(plan_id) => crab_metadata::capsule_protocol::CapsuleTransaction::for_plan( - self.base.record().digest(), - plan_id, - self.edits, - )?, - None => crab_metadata::capsule_protocol::CapsuleTransaction::new( - self.base.record().digest(), - self.edits, - )?, - }; - let transaction_id = transaction.id()?; - let started = Instant::now(); - let capsule = crab_metadata::capsule_protocol::Capsule::build( - &transaction, - self.packs, - self.sections, - )?; - tracing::info!( - phase = "capsule_encode", - %transaction_id, - bytes = capsule.bytes().len(), - elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, - "Git receive phase completed" - ); - let publish = if self.changes_namespace { - let ref_names = transaction - .edits() - .iter() - .map(|edit| edit.ref_name().to_owned()) - .collect::>(); - let layout = self.layout.clone(); - let commit_layout = layout.clone(); - let base = self.base; - crab_write::with_ref_namespaces( - layout.store(), - &layout, - &ref_names, - namespace_ttl, - cancel, - move |scoped| async move { - if scoped.is_cancelled() { - return Err(crab_write::WriteError::Cancelled); - } - crab_write::capsule_protocol::validate_ref_namespace( - &commit_layout, - base.record().root(), - transaction.edits(), - ) - .await?; - crab_write::capsule_protocol::publish( - &commit_layout, - base, - &transaction, - &capsule, - ) - .await - }, - ) - .await - } else { - crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) - .await - }; - match publish { - Ok(_) => Ok(CapsuleCommitOutcome::Committed { transaction_id }), - Err(error @ crab_write::WriteError::CapsuleCommitUncertain { .. }) => { - Ok(CapsuleCommitOutcome::Indeterminate { - transaction_id, - source: Box::new(error), - }) - } - Err(error) => Err(error.into()), - } - } -} - -async fn upload_capsule_content( - layout: &crab_storage::StoreLayout, - base: &PointerCatalog, - content: &PreparedContent, - cancel: &CancellationToken, -) -> Result { - let local = content.pointer_catalog(); - let mut delta = PointerCatalog::new(); - let mut needed_shards = std::collections::BTreeSet::new(); - for (file_hash, file) in local.files() { - if let Some(existing) = base.files().get(file_hash) { - if existing.size() != file.size() { - return Err(Error::Content(format!( - "file {file_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - needed_shards.insert(file.shard_hash().to_owned()); - delta.insert_file(file_hash.clone(), file.clone())?; - } - let mut needed_xorbs = std::collections::BTreeSet::new(); - for shard_hash in &needed_shards { - let shard = local - .shards() - .get(shard_hash) - .ok_or_else(|| Error::Content("prepared shard catalog entry is missing".to_owned()))?; - if let Some(existing) = base.shards().get(shard_hash) { - if existing != shard { - return Err(Error::Content(format!( - "shard {shard_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - needed_xorbs.extend(shard.xorb_hashes().iter().cloned()); - delta.insert_shard(shard_hash.clone(), shard.clone())?; - } - let xorb_artifacts = content - .xorbs - .iter() - .map(|artifact| { - ( - crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), - artifact, - ) - }) - .collect::>(); - for xorb_hash in needed_xorbs { - let local_entry = local - .xorbs() - .get(&xorb_hash) - .ok_or_else(|| Error::Content("prepared xorb catalog entry is missing".to_owned()))?; - if let Some(existing) = base.xorbs().get(&xorb_hash) { - if existing.chunks() != local_entry.chunks() { - return Err(Error::Content(format!( - "xorb {xorb_hash} conflicts with the captured pointer catalog" - ))); - } - continue; - } - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let artifact = xorb_artifacts - .get(&xorb_hash) - .ok_or_else(|| Error::Content("prepared xorb artifact is missing".to_owned()))?; - let bytes = Bytes::from(tokio::fs::read(&artifact.path).await?); - let hash = crab_xet::hash::MerkleHash::from(artifact.protocol_hash); - let entry = match layout - .store() - .create_or_read_immutable( - &layout.xorb_path(&hash), - bytes, - crab_xet::xorb::format::MAX_XORB_SIZE as u64, - ) - .await? - { - crab_storage::ImmutableCreateOutcome::Created(_) => local_entry.clone(), - crab_storage::ImmutableCreateOutcome::Existing { - bytes: existing, .. - } => catalog_xorb_from_existing(&xorb_hash, local_entry, existing)?, - }; - delta.insert_xorb(xorb_hash, entry)?; - } - let shard_artifacts = content - .shards - .iter() - .map(|artifact| { - ( - crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), - artifact, - ) - }) - .collect::>(); - for shard_hash in needed_shards { - if base.shards().contains_key(&shard_hash) { - continue; - } - if cancel.is_cancelled() { - return Err(Error::Cancelled); - } - let artifact = shard_artifacts - .get(&shard_hash) - .ok_or_else(|| Error::Content("prepared shard artifact is missing".to_owned()))?; - let bytes = Bytes::from(tokio::fs::read(&artifact.path).await?); - layout - .store() - .put_if_absent_verified( - &layout.shard_path(&crab_xet::hash::MerkleHash::from(artifact.protocol_hash)), - bytes, - ) - .await?; - } - if !delta.shards().is_empty() { - crab_metadata::ref_registry::union_register_repo_shards( - layout.store(), - layout, - delta.shards().keys().cloned().collect(), - ) - .await?; - } - let mut complete = base.clone(); - complete.apply(&delta)?; - Ok(delta) -} - -fn catalog_xorb_from_existing( - xorb_hash: &str, - local: &XorbCatalogEntry, - bytes: Bytes, -) -> Result { - let parser = - crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).map_err(Error::ContentFormat)?; - parser - .verify_payload_digest() - .map_err(Error::ContentFormat)?; - parser.verify_all_chunks().map_err(Error::ContentFormat)?; - if parser.hash().hex() != xorb_hash { - return Err(Error::Content( - "existing xorb has the wrong logical identity".to_owned(), - )); - } - let chunks = (0..parser.num_chunks()) - .map(|index| { - parser - .chunk_meta(index) - .map(|chunk| XorbChunkEntry::new(chunk.hash.hex(), chunk.uncompressed_len)) - }) - .collect::, _>>() - .map_err(Error::ContentFormat)?; - if chunks != local.chunks() { - return Err(Error::Content( - "existing xorb has a conflicting chunk layout".to_owned(), - )); - } - Ok(XorbCatalogEntry::new( - bytes.len() as u64, - blake3::hash(&bytes).to_hex().to_string(), - chunks, - )) } impl Artifacts<'_> { diff --git a/crates/crab-remote/src/prepare/capsule.rs b/crates/crab-remote/src/prepare/capsule.rs new file mode 100644 index 000000000..dcf26b988 --- /dev/null +++ b/crates/crab-remote/src/prepare/capsule.rs @@ -0,0 +1,495 @@ +use super::*; + +impl Prepared { + /// Upload artifacts for one captured capsule-protocol repository view. + /// + /// The caller must hold ref leases and GC fences from before opening + /// `view` through commitment. This validates all historical dependencies + /// against that view, uploads same-publication Xet content before Git + /// visibility, and retains the exact root snapshot for per-ref CAS. + pub async fn upload_capsule( + &self, + view: &crab_read::capsule_protocol::CapsuleRepositoryView, + limits: crab_read::dependency_proof::DependencyProofLimits, + cancel: &CancellationToken, + ) -> Result { + let started = Instant::now(); + let repository_refs = self + .repository + .refs() + .entries + .iter() + .map(|reference| (reference.name.clone(), reference.target.to_string())) + .collect::>(); + if repository_refs != *view.refs() + || self.repository.generation() != view.root().root().generation() + { + return Err(Error::Request( + "Capsule view differs from the prepared repository version", + )); + } + if self.plan.pointers().len() > limits.max_dependencies { + return Err(Error::Dependency(Box::new( + crab_read::dependency_proof::DependencyProofError::Limit("pointer count"), + ))); + } + let lookup = view.pointer_catalog_lookup( + &self.layout, + crab_metadata::capsule_protocol::MAX_CATALOG_BYTES, + )?; + let hashes = self + .plan + .pointers() + .iter() + .filter_map(|dependency| match &dependency.pointer { + crab_git::pointer_detect::PointerKind::Crab(pointer) => { + Some(crab_xet::hash::MerkleHash::from(pointer.file_hash).hex()) + } + _ => None, + }) + .collect::>(); + // Ordinary Git and LFS-only changes need no Xet catalog pages. Selected + // Crab pointers retain complete authenticated shard/xorb closure checks. + let base_catalog = if hashes.is_empty() { + PointerCatalog::new() + } else { + lookup.files(self.layout.store(), &hashes).await? + }; + let prepared_files = self + .content + .as_ref() + .map(|content| { + content + .files + .iter() + .map(|file| file.file_hash) + .collect::>() + }) + .unwrap_or_default(); + crab_read::dependency_proof::verify_capsule_dependencies_except_crab( + &self.layout, + &base_catalog, + self.plan.pointers(), + &prepared_files, + limits, + cancel, + ) + .await + .map_err(|error| Error::Dependency(Box::new(error)))?; + + let (pointer_delta, origins) = match &self.content { + Some(content) => { + upload_capsule_content(&self.layout, &lookup, base_catalog, content, cancel).await? + } + None => (PointerCatalog::new(), HashMap::new()), + }; + let mut packs = Vec::new(); + if let Some(pack) = &self.pack { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let (pack_bytes, index, reverse_index, locator) = tokio::try_join!( + tokio::fs::read(pack.pack_path()), + tokio::fs::read(pack.index_path()), + tokio::fs::read(pack.reverse_path()), + tokio::fs::read(pack.kinds_path()), + )?; + let external_delta_bases = pack + .external_delta_bases() + .iter() + .map(ToString::to_string) + .collect(); + packs.push( + crab_metadata::capsule_protocol::CapsuleGitPack::new_with_external_delta_bases( + Bytes::from(pack_bytes), + Bytes::from(index), + Bytes::from(reverse_index), + Bytes::from(locator), + pack.git_sha1().to_string(), + u64::from(pack.object_count()), + external_delta_bases, + )?, + ); + } + let mut sections = Vec::with_capacity(1); + if !self.visibility.is_empty() { + let visibility = crab_metadata::capsule_protocol::CapsuleVisibilityDelta::new( + self.visibility.clone(), + )?; + sections.push(crab_metadata::capsule_protocol::CapsuleSection::new( + crab_metadata::capsule_protocol::CapsuleSectionKind::VisibilityDelta, + visibility.encode()?, + )); + } + let edits = self + .updates + .iter() + .map(|update| { + crab_metadata::capsule_protocol::CapsuleRefEdit::new( + update.name.clone(), + update.old.map(|oid| oid.to_string()), + update.new.map(|oid| oid.to_string()), + self.plan + .peeled() + .get(&update.name) + .map(ToString::to_string), + ) + }) + .collect::>(); + let changes_namespace = edits + .iter() + .any(|edit| edit.expected_old().is_none() != edit.new_oid().is_none()); + tracing::info!( + phase = "capsule_input", + packs = packs.len(), + elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, + "Git receive phase completed" + ); + Ok(CapsuleArtifacts { + layout: self.layout.clone(), + base: view.root_snapshot().clone(), + edits, + packs, + sections, + pointer_delta, + origins, + changes_namespace, + }) + } +} + +impl CapsuleArtifacts { + /// Commit through independently mutable ref heads and one capsule authority. + /// + /// Revalidate authorization and repository policy immediately before this + /// call. The caller must retain its original ref leases and GC fences until + /// the returned outcome is known and all lease cleanup has completed. + pub async fn commit( + self, + plan_id: Option<&str>, + namespace_ttl: Duration, + cancel: &CancellationToken, + ) -> Result { + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let transaction = match plan_id { + Some(plan_id) => crab_metadata::capsule_protocol::CapsuleTransaction::for_plan( + self.base.record().digest(), + plan_id, + self.edits, + )?, + None => crab_metadata::capsule_protocol::CapsuleTransaction::new( + self.base.record().digest(), + self.edits, + )?, + }; + let transaction_id = transaction.id()?; + let started = Instant::now(); + let mut sections = self.sections; + if !self.pointer_delta.is_empty() { + sections.insert( + 0, + crab_metadata::capsule_protocol::CapsuleSection::new( + crab_metadata::capsule_protocol::CapsuleSectionKind::CatalogDelta, + self.pointer_delta.encode_delta()?, + ), + ); + } + let capsule = + crab_metadata::capsule_protocol::Capsule::build(&transaction, self.packs, sections)?; + tracing::info!( + phase = "capsule_encode", + %transaction_id, + bytes = capsule.bytes().len(), + elapsed_ms = started.elapsed().as_secs_f64() * 1000.0, + "Git receive phase completed" + ); + let publish = if self.changes_namespace { + let ref_names = transaction + .edits() + .iter() + .map(|edit| edit.ref_name().to_owned()) + .collect::>(); + let layout = self.layout.clone(); + let commit_layout = layout.clone(); + let base = self.base; + crab_write::with_ref_namespaces( + layout.store(), + &layout, + &ref_names, + namespace_ttl, + cancel, + move |scoped| async move { + if scoped.is_cancelled() { + return Err(crab_write::WriteError::Cancelled); + } + crab_write::capsule_protocol::validate_ref_namespace( + &commit_layout, + base.record().root(), + transaction.edits(), + ) + .await?; + crab_write::capsule_protocol::publish( + &commit_layout, + base, + &transaction, + &capsule, + ) + .await + }, + ) + .await + } else { + crab_write::capsule_protocol::publish(&self.layout, self.base, &transaction, &capsule) + .await + }; + match publish { + Ok(_) => { + // Ref visibility owns the payload before hints become discoverable. + // Failed hint maintenance cannot reject an already committed Git push. + let config = + crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + self.layout.repo_prefix(), + self.layout.global_prefix(), + ); + if let Err(error) = crab_metadata::remote_index::write_capsule_chunk_candidates( + Arc::clone(self.layout.store().inner()), + &config, + self.layout.repo_prefix(), + &self.pointer_delta, + &self.origins, + ) + .await + { + tracing::warn!(%error, "capsule committed; chunk candidate index needs repair"); + } + Ok(CapsuleCommitOutcome::Committed { transaction_id }) + } + Err(error @ crab_write::WriteError::CapsuleCommitUncertain { .. }) => { + Ok(CapsuleCommitOutcome::Indeterminate { + transaction_id, + source: Box::new(error), + }) + } + Err(error) => Err(error.into()), + } + } +} + +async fn upload_capsule_content( + layout: &crab_storage::StoreLayout, + lookup: &PointerCatalogLookup, + mut base: PointerCatalog, + content: &PreparedContent, + cancel: &CancellationToken, +) -> Result<( + PointerCatalog, + HashMap, +)> { + let local = content.pointer_catalog(); + let mut delta = PointerCatalog::new(); + let mut origins = HashMap::new(); + let mut needed_shards = std::collections::BTreeSet::new(); + for (file_hash, file) in local.files() { + if let Some(existing) = base.files().get(file_hash) { + if existing.size() != file.size() { + return Err(Error::Content(format!( + "file {file_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + needed_shards.insert(file.shard_hash().to_owned()); + delta.insert_file(file_hash.clone(), file.clone())?; + } + if !needed_shards.is_empty() { + base.apply( + &lookup + .shards( + layout.store(), + &needed_shards.iter().cloned().collect::>(), + ) + .await?, + )?; + } + let mut needed_xorbs = std::collections::BTreeSet::new(); + for shard_hash in &needed_shards { + let shard = local + .shards() + .get(shard_hash) + .ok_or_else(|| Error::Content("prepared shard catalog entry is missing".to_owned()))?; + if let Some(existing) = base.shards().get(shard_hash) { + if existing != shard { + return Err(Error::Content(format!( + "shard {shard_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + needed_xorbs.extend(shard.xorb_hashes().iter().cloned()); + delta.insert_shard(shard_hash.clone(), shard.clone())?; + } + if !needed_xorbs.is_empty() { + base.apply( + &lookup + .xorbs( + layout.store(), + &needed_xorbs.iter().cloned().collect::>(), + ) + .await?, + )?; + } + let xorb_artifacts = content + .xorbs + .iter() + .map(|artifact| { + ( + crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), + artifact, + ) + }) + .collect::>(); + for xorb_hash in needed_xorbs { + let local_entry = local + .xorbs() + .get(&xorb_hash) + .ok_or_else(|| Error::Content("prepared xorb catalog entry is missing".to_owned()))?; + if let Some(existing) = base.xorbs().get(&xorb_hash) { + if existing.chunks() != local_entry.chunks() { + return Err(Error::Content(format!( + "xorb {xorb_hash} conflicts with the captured pointer catalog" + ))); + } + continue; + } + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let artifact = xorb_artifacts + .get(&xorb_hash) + .ok_or_else(|| Error::Content("prepared xorb artifact is missing".to_owned()))?; + let bytes = artifact + .read_verified(crab_xet::xorb::format::MAX_XORB_SIZE as u64, cancel) + .await?; + let hash = crab_xet::hash::MerkleHash::from(artifact.protocol_hash); + let path = layout.xorb_path(&hash); + let payload_digest = crab_xet::xorb::parser::XorbParser::parse(bytes.clone()) + .map_err(Error::ContentFormat)? + .payload_digest(); + let (entry, payload_digest, identity) = match layout + .store() + .create_or_read_immutable(&path, bytes, crab_xet::xorb::format::MAX_XORB_SIZE as u64) + .await? + { + crab_storage::ImmutableCreateOutcome::Created(identity) => { + (local_entry.clone(), payload_digest, identity) + } + crab_storage::ImmutableCreateOutcome::Existing { + bytes: existing, + identity, + } => { + let expected = local_entry.clone(); + let expected_hash = xorb_hash.clone(); + let (entry, payload_digest) = tokio::task::spawn_blocking(move || { + catalog_xorb_from_existing(&expected_hash, &expected, existing) + }) + .await??; + (entry, payload_digest, identity) + } + }; + origins.insert( + hash, + crab_metadata::receipts::OriginReceipt::new( + "canonical-origin".to_owned(), + path.to_string(), + hash.into(), + payload_digest, + entry.encoded_size(), + identity.e_tag, + identity.version, + ), + ); + delta.insert_xorb(xorb_hash, entry)?; + } + let shard_artifacts = content + .shards + .iter() + .map(|artifact| { + ( + crab_xet::hash::MerkleHash::from(artifact.protocol_hash).hex(), + artifact, + ) + }) + .collect::>(); + for shard_hash in needed_shards { + if base.shards().contains_key(&shard_hash) { + continue; + } + if cancel.is_cancelled() { + return Err(Error::Cancelled); + } + let artifact = shard_artifacts + .get(&shard_hash) + .ok_or_else(|| Error::Content("prepared shard artifact is missing".to_owned()))?; + let bytes = artifact + .read_verified(crab_xet::shard::MAX_PUSH_SHARD_SIZE_BYTES, cancel) + .await?; + layout + .store() + .put_if_absent_verified( + &layout.shard_path(&crab_xet::hash::MerkleHash::from(artifact.protocol_hash)), + bytes, + ) + .await?; + } + if !delta.shards().is_empty() { + crab_metadata::ref_registry::union_register_repo_shards( + layout.store(), + layout, + delta.shards().keys().cloned().collect(), + ) + .await?; + } + base.apply(&delta)?; + Ok((delta, origins)) +} + +fn catalog_xorb_from_existing( + xorb_hash: &str, + local: &XorbCatalogEntry, + bytes: Bytes, +) -> Result<(XorbCatalogEntry, [u8; 32])> { + let parser = + crab_xet::xorb::parser::XorbParser::parse(bytes.clone()).map_err(Error::ContentFormat)?; + parser + .verify_payload_digest() + .map_err(Error::ContentFormat)?; + parser.verify_all_chunks().map_err(Error::ContentFormat)?; + if parser.hash().hex() != xorb_hash { + return Err(Error::Content( + "existing xorb has the wrong logical identity".to_owned(), + )); + } + let chunks = (0..parser.num_chunks()) + .map(|index| { + parser + .chunk_meta(index) + .map(|chunk| XorbChunkEntry::new(chunk.hash.hex(), chunk.uncompressed_len)) + }) + .collect::, _>>() + .map_err(Error::ContentFormat)?; + if chunks != local.chunks() { + return Err(Error::Content( + "existing xorb has a conflicting chunk layout".to_owned(), + )); + } + Ok(( + XorbCatalogEntry::new( + bytes.len() as u64, + blake3::hash(&bytes).to_hex().to_string(), + chunks, + ), + parser.payload_digest(), + )) +} diff --git a/crates/crab-remote/tests/capsule_preparation.rs b/crates/crab-remote/tests/capsule_preparation.rs new file mode 100644 index 000000000..52d9db6fe --- /dev/null +++ b/crates/crab-remote/tests/capsule_preparation.rs @@ -0,0 +1,621 @@ +#![cfg(feature = "publication")] + +use std::sync::Arc; + +use bytes::Bytes; +use crab_remote::prepare::{ContentArtifact, ContentFile, PreparedContent}; +use crab_xet::hash::{MerkleHash, compute_data_hash}; +use crab_xet::shard::{PushShardSession, file_info_from_placements, xorb_info_from_placements}; +use crab_xet::xorb::builder::{RunId, XorbBuilder}; +use crab_xet::xorb::format::Chunk; +use tokio_util::sync::CancellationToken; + +const LIMIT: u64 = 8 * 1024 * 1024; + +async fn content_fixture() -> (tempfile::TempDir, PreparedContent) { + let directory = tempfile::tempdir().unwrap(); + let data = Bytes::from_static(b"prepared capsule content must remain reconstructable"); + let chunk_hash = compute_data_hash(&data); + let file_hash = MerkleHash::from(*blake3::hash(&data).as_bytes()); + let mut builder = XorbBuilder::new(); + builder + .push( + &Chunk { + hash: chunk_hash, + data: data.clone(), + }, + RunId(0), + ) + .unwrap(); + let xorb = builder.finalize().unwrap().pop().unwrap(); + let placements = xorb + .placements + .iter() + .map(|placement| (placement.chunk_hash, placement.clone())) + .collect(); + let file = file_info_from_placements(file_hash, &[chunk_hash], &placements).unwrap(); + let xorb_info = Arc::new(xorb_info_from_placements(xorb.hash, &xorb.placements).unwrap()); + let mut shards = PushShardSession::new(); + shards.add_file_bundle(file, &[xorb_info]).unwrap(); + let (shard, shard_hash) = shards.finalize().unwrap().pop().unwrap(); + let xorb_path = directory.path().join("content.xorb"); + let shard_path = directory.path().join("content.shard"); + std::fs::write(&xorb_path, &xorb.bytes).unwrap(); + std::fs::write(&shard_path, &shard).unwrap(); + let content = PreparedContent::new( + vec![ContentArtifact::new( + xorb.hash.into(), + *blake3::hash(&xorb.bytes).as_bytes(), + xorb.bytes.len() as u64, + xorb_path, + )], + vec![ContentArtifact::new( + shard_hash.into(), + *blake3::hash(&shard).as_bytes(), + shard.len() as u64, + shard_path, + )], + vec![ContentFile::new( + file_hash.into(), + data.len() as u64, + shard_hash.into(), + )], + LIMIT, + &CancellationToken::new(), + ) + .await + .unwrap(); + (directory, content) +} + +#[tokio::test] +async fn prepared_catalog_preserves_raw_blake3_body_identity() { + let (directory, content) = content_fixture().await; + let expected = blake3::hash(&std::fs::read(directory.path().join("content.xorb")).unwrap()) + .to_hex() + .to_string(); + let actual = content + .pointer_catalog() + .xorbs() + .values() + .next() + .unwrap() + .body_digest(); + assert_eq!( + actual, expected, + "body digests use raw Blake3 hex; only protocol identities use Xet word order" + ); +} + +#[derive(Default)] +struct Reads(std::sync::atomic::AtomicUsize); +impl crab_storage::StorageObserver for Reads { + fn started(&self, _: crab_storage::StorageOperation) {} + fn finished(&self, observation: crab_storage::StorageObservation) { + if matches!( + observation.operation, + crab_storage::StorageOperation::Get | crab_storage::StorageOperation::Range + ) { + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + } +} + +fn git(directory: &std::path::Path, args: &[&str]) -> Vec { + git_input(directory, args, b"") +} + +fn git_input(directory: &std::path::Path, args: &[&str], input: &[u8]) -> Vec { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args([ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgSign=false", + "-c", + "core.hooksPath=/dev/null", + ]) + .args(args) + .current_dir(directory) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env_remove("GIT_DIR") + .env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(input).unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + +fn source_commit(directory: &std::path::Path, content: &PreparedContent) { + git(directory, &["init", "-b", "main"]); + let file = &content.files()[0]; + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\n", + blake3::Hash::from_bytes(*file.file_hash()).to_hex(), + file.size() + ); + std::fs::write(directory.join("model.bin"), pointer).unwrap(); + git(directory, &["add", "model.bin"]); + git(directory, &["commit", "-m", "publish pointer"]); +} + +async fn layout() -> (crab_storage::StoreLayout, Arc) { + let reads = Arc::new(Reads::default()); + let store = crab_storage::Store::new(Arc::new(object_store::memory::InMemory::new())) + .with_storage_observer(reads.clone()); + let layout = crab_storage::StoreLayout::new(store, "repositories/prepared".to_owned()); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + (layout, reads) +} + +fn proof_limits() -> crab_read::dependency_proof::DependencyProofLimits { + crab_read::dependency_proof::DependencyProofLimits { + max_dependencies: 16, + max_total_file_bytes: LIMIT, + max_duration: std::time::Duration::from_secs(60), + lookup: crab_metadata::file_index_lookup::FileIndexLookupLimits { + max_files: 16, + max_shard_visits: 16, + max_shard_bytes: LIMIT, + max_recipe_entries: 1024, + }, + content: crab_read::pointer_proof::PointerProofLimits { + max_file_bytes: LIMIT, + max_shard_bytes: LIMIT, + max_xorb_bytes: LIMIT, + max_read_bytes: LIMIT, + max_chunks: 1024, + max_duration: std::time::Duration::from_secs(60), + }, + } +} + +async fn prepare( + layout: &crab_storage::StoreLayout, + directory: &std::path::Path, + content: Option, +) -> ( + crab_remote::prepare::Prepared, + crab_read::capsule_protocol::CapsuleRepositoryView, + Arc, + crab_remote::publication::PublicationLeases, +) { + let cancel = CancellationToken::new(); + let leases = crab_remote::publication::acquire_leases( + layout.store(), + layout, + ["refs/heads/main".to_owned()], + crab_remote::publication::LeaseOptions::renewing(std::time::Duration::from_secs(60)), + &cancel, + ) + .await + .unwrap(); + let root = crab_write::capsule_protocol::open_root(layout) + .await + .unwrap(); + let view = crab_read::capsule_protocol::open_git_view_from_root( + layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); + let runtime = Arc::new(crab_remote_git::RemoteGitRuntime::default()); + let repository = view + .git_repository_from_store( + layout.clone(), + crab_remote_git::RepositoryIdentity::new("memory", "prepared", 1).unwrap(), + runtime.clone(), + crab_remote_git::RepositoryOptions::default(), + LIMIT, + &cancel, + ) + .await + .unwrap(); + let oid = gix_hash::ObjectId::from_hex( + String::from_utf8(git(directory, &["rev-parse", "HEAD"])) + .unwrap() + .trim() + .as_bytes(), + ) + .unwrap(); + let mut revisions = String::from("HEAD\n"); + if let Some(old) = view.refs().get("refs/heads/main") { + revisions.push_str(&format!("^{old}\n")); + } + let pack = git_input( + directory, + &["pack-objects", "--stdout", "--revs"], + revisions.as_bytes(), + ); + let incoming = directory.join("incoming.pack"); + std::fs::write(&incoming, pack).unwrap(); + let limits = crab_git::incoming_pack::ReceiveLimits { + max_pack_bytes: LIMIT, + max_objects: 1024, + max_object_bytes: LIMIT as usize, + max_inflated_bytes: LIMIT, + max_delta_depth: 128, + }; + let prepared = crab_remote::prepare::prepare_capsule( + repository, + view.git_visibility_index().await.unwrap(), + crab_git::incoming_pack::ScratchSpace::new(directory, limits), + Some(std::io::BufReader::new( + std::fs::File::open(incoming).unwrap(), + )), + vec![crab_git::receive_plan::RefUpdate { + name: "refs/heads/main".to_owned(), + old: view + .refs() + .get("refs/heads/main") + .map(|oid| gix_hash::ObjectId::from_hex(oid.as_bytes()).unwrap()), + new: Some(oid), + }], + std::collections::BTreeMap::new(), + &cancel, + crab_remote::prepare::Options { + layout: layout.clone(), + graph: crab_git::receive_plan::GraphLimits { + max_ref_updates: 16, + max_graph_steps: 1024, + max_object_bytes: LIMIT as usize, + max_read_bytes: LIMIT, + }, + pack: limits, + policy: |_: &str| crab_git::receive_plan::RefPolicy { + allow_delete: true, + allow_non_fast_forward: false, + }, + }, + ) + .await + .unwrap(); + let prepared = match content { + Some(content) => prepared.attach_content(content).unwrap(), + None => prepared, + }; + (prepared, view, runtime, leases) +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_upload_rejects_changed_prepared_bytes() { + let mut rejected = Vec::new(); + for artifact in ["content.xorb", "content.shard"] { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + std::fs::write(directory.path().join(artifact), b"changed after inspection").unwrap(); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + leases.release().await; + runtime.shutdown().await; + rejected.push((artifact, result.is_err())); + } + assert!( + rejected.iter().all(|(_, failed)| *failed), + "changed artifacts must fail before ref publication: {rejected:?}" + ); +} + +async fn verify_indexed_publication( + layout: crab_storage::StoreLayout, + occupied: bool, +) { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let chunk = content + .pointer_catalog() + .xorbs() + .values() + .next() + .unwrap() + .chunks()[0] + .hash() + .to_owned(); + if occupied { + let artifact = &content.xorbs()[0]; + layout + .store() + .put( + &layout.xorb_path(&MerkleHash::from(*artifact.protocol_hash())), + Bytes::from(std::fs::read(directory.path().join("content.xorb")).unwrap()), + ) + .await + .unwrap(); + } + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let result = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!(matches!( + result.unwrap(), + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + let reader = + slatedb::DbReader::builder(config.chunk_index_path, Arc::clone(layout.store().inner())) + .build() + .await + .expect("shared publication must populate cold native-add candidates"); + let hash = MerkleHash::from_hex(&chunk).unwrap(); + let candidate = reader + .get(crab_metadata::key_codec::capsule_chunk_head_key( + &hash.into(), + )) + .await + .unwrap() + .expect("published payload candidate"); + let placement = crab_metadata::receipts::CommittedChunkPlacement::decode(&candidate).unwrap(); + let anchor = reader + .get(crab_metadata::key_codec::capsule_chunk_anchor_key( + &placement.source_anchor_id, + )) + .await + .unwrap() + .unwrap(); + let anchor = crab_metadata::capsule_protocol::CapsuleChunkAnchor::decode( + &placement.source_anchor_id, + &anchor, + ) + .unwrap(); + reader.close().await.unwrap(); + let path = layout.xorb_path(&MerkleHash::from(placement.xorb_hash)); + let (body, identity) = layout.store().get_with_etag(&path).await.unwrap(); + let root = crab_write::capsule_protocol::open_root(&layout) + .await + .unwrap(); + let current = crab_read::capsule_protocol::open_git_view_from_root( + &layout, + root, + crab_read::capsule_protocol::CapsuleReadLimits { + max_capsule_bytes: LIMIT, + max_frontier_bytes: LIMIT, + }, + ) + .await + .unwrap(); + let catalog = current.pointer_catalog().await.unwrap(); + anchor + .validate_placement(&placement, &catalog, path.as_ref()) + .unwrap(); + assert_eq!( + ( + anchor.origin.etag, + anchor.origin.object_version, + anchor.body_digest + ), + ( + identity.e_tag, + identity.version, + blake3::hash(&body).to_hex().to_string() + ) + ); + let crab_git::pointer_detect::PointerKind::Crab(pointer) = crab_git::pointer_detect::classify( + &std::fs::read(directory.path().join("model.bin")).unwrap(), + ) else { + panic!("fixture pointer"); + }; + crab_read::pointer_proof::verify_crab_pointer( + &layout, + &pointer, + MerkleHash::from_hex(catalog.files().values().next().unwrap().shard_hash()).unwrap(), + proof_limits().content, + &CancellationToken::new(), + ) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn capsule_commit_indexes_verified_native_payload() { + for occupied in [false, true] { + let (layout, _) = layout().await; + verify_indexed_publication(layout, occupied).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "requires CRAB_E2E_FLAT_BUCKET and explicit scratch S3 credentials"] +async fn live_capsule_preparation_publishes_and_reconstructs_native_content() { + let bucket = std::env::var("CRAB_E2E_FLAT_BUCKET").expect("explicit scratch bucket required"); + let store = + crab_storage::build_static_env_store(&bucket, crab_storage::StorageProviderKind::S3) + .unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + for occupied in [false, true] { + let prefix = format!("shared-preparation/{nonce}/{occupied}"); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.clone(), + format!("{prefix}/repo"), + format!("{prefix}/global"), + ); + crab_write::capsule_protocol::initialize(&layout, &"1".repeat(64), "refs/heads/main") + .await + .unwrap(); + verify_indexed_publication(layout, occupied).await; + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn git_only_capsule_upload_does_not_read_historical_catalog() { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, reads) = layout().await; + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let committed = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + committed, + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + leases.release_after_commit().await; + runtime.shutdown().await; + git(directory.path(), &["rm", "model.bin"]); + std::fs::write(directory.path().join("README.md"), b"ordinary Git update").unwrap(); + git(directory.path(), &["add", "README.md"]); + git( + directory.path(), + &["commit", "-m", "replace pointer with ordinary Git file"], + ); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), None).await; + assert!(prepared.plan().pointers().is_empty()); + reads.0.store(0, std::sync::atomic::Ordering::Relaxed); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + let count = reads.0.load(std::sync::atomic::Ordering::Relaxed); + leases.release().await; + runtime.shutdown().await; + result.unwrap(); + assert_eq!( + count, 0, + "ordinary Git publication must not read unrelated Xet history" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn committed_capsule_survives_candidate_index_failure() { + let (directory, content) = content_fixture().await; + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let config = crab_metadata::remote_index::RemoteIndexConfig::for_repo_with_global_prefix( + layout.repo_prefix(), + layout.global_prefix(), + ); + let db = slatedb::Db::open(config.chunk_index_path, Arc::clone(layout.store().inner())) + .await + .unwrap(); + db.put( + crab_metadata::key_codec::encode_system_key(crab_metadata::key_codec::SYS_FORMAT_VERSION), + 2u32.to_le_bytes(), + ) + .await + .unwrap(); + db.close().await.unwrap(); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let result = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await; + leases.release_after_commit().await; + runtime.shutdown().await; + assert!( + matches!( + result.unwrap(), + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + ), + "repairable hint-index failure must not reject committed refs" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn historical_pointer_lookup_preserves_declared_size_checks() { + for size_delta in [0, 1] { + let (directory, content) = content_fixture().await; + let file = content.files()[0].clone(); + source_commit(directory.path(), &content); + let (layout, _) = layout().await; + let (prepared, view, runtime, leases) = + prepare(&layout, directory.path(), Some(content)).await; + let artifacts = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await + .unwrap(); + let committed = artifacts + .commit( + None, + std::time::Duration::from_secs(60), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert!(matches!( + committed, + crab_remote::prepare::CapsuleCommitOutcome::Committed { .. } + )); + leases.release_after_commit().await; + runtime.shutdown().await; + // A hint changes the Git blob identity while retaining the existing file. + // Native negotiation therefore sends a real historical-pointer dependency. + let pointer = format!( + "version https://crab.build/spec/v1\nfile-hash {}\nsize {}\nshard-hint {}\n", + blake3::Hash::from_bytes(*file.file_hash()).to_hex(), + file.size() + size_delta, + blake3::Hash::from_bytes(*file.shard_hash()).to_hex() + ); + std::fs::write(directory.path().join("model.bin"), pointer).unwrap(); + git(directory.path(), &["add", "model.bin"]); + git( + directory.path(), + &["commit", "-m", "update pointer binding"], + ); + let (prepared, view, runtime, leases) = prepare(&layout, directory.path(), None).await; + assert_eq!(prepared.plan().pointers().len(), 1); + let result = prepared + .upload_capsule(&view, proof_limits(), &CancellationToken::new()) + .await; + leases.release().await; + runtime.shutdown().await; + assert_eq!( + result.is_ok(), + size_delta == 0, + "historical pointer size delta {size_delta}" + ); + } +} From 688961db993ce4b6bd69b0f04fc5273f06c65c85 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 06:42:53 -0700 Subject: [PATCH 13/17] test(gc): adapt captured catalog sweep call --- crab/src/cmd/gc/capsule_cleanup_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crab/src/cmd/gc/capsule_cleanup_tests.rs b/crab/src/cmd/gc/capsule_cleanup_tests.rs index 9a3a7f618..d1e0311e1 100644 --- a/crab/src/cmd/gc/capsule_cleanup_tests.rs +++ b/crab/src/cmd/gc/capsule_cleanup_tests.rs @@ -187,7 +187,6 @@ async fn captured_catalog_pages_survive_checkpoint_replacement_and_gc() { &layout, root.record().root(), current.capsule_run_pointers(), - ¤t.state_digest(), &snapshot.manifest, &HashSet::new(), &CancellationToken::new(), From 160aea94baf2a20f9996b8cbac8b4b81f546f925 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 07:27:30 -0700 Subject: [PATCH 14/17] test(http): serialize corruption fault injection Drain the preceding repair worker and bind each injected fault to current readiness. A published ready record can precede worker completion, so overlapping mutations can legitimately observe graph-only readiness. --- .../docs/architecture/indexed-remote-reads.md | 31 +++++++++++++++++++ crates/crab-http-server/src/receive_tests.rs | 18 ++++++++--- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/crab/docs/architecture/indexed-remote-reads.md b/crab/docs/architecture/indexed-remote-reads.md index eb032baae..866cc3b3d 100644 --- a/crab/docs/architecture/indexed-remote-reads.md +++ b/crab/docs/architecture/indexed-remote-reads.md @@ -2368,3 +2368,34 @@ large-file, growth and latency cells. The final combined 10-GiB matrix, unrelated-history/ref/frontier scaling, broader lifecycle/provider proof and fuller blame semantics still prevent completion. + + +## Combined catalog/staging qualification and HTTP fault boundaries + +The combined revision carries selected publication, verified catalog bindings, +source/browse retention, retained-source fsck, exact capsule chunk reuse and +shared native-content preparation. Focused checks passed: 558 metadata tests, +81 protected-receive tests, six shared-preparation tests, 242 staging tests, +26 CLI integration tests, 57 CLI siblings, 20 origin/lease tests and eight HTTP +receive/fault tests. Explicit live tests remain separately qualified; these +counts do not establish the full release gate. + +The first combined RustFS native HTTP fixture returned 202 during its +corrupt-descriptor case, where the corruption oracle required 503. Readiness +can become visible before the publishing repair worker finishes. A diagnostic +assertion with a one-second delayed generation-owner release response proved +that the fixture could inject its next fault while that worker was active. +The shared native HTTP fixture now awaits maintenance completion before each +fault, reloads the current descriptor and checks its snapshot identity. All +original status, error-code, recovery and Git assertions remain intact. The +controlled probe then passed across ten delayed lease releases, and all eight +HTTP unit/fault checks passed. This changes test synchronization, not the valid +production distinction between incomplete readiness and corrupt metadata. + +Raw before/after evidence remains outside the checkout in +`final-integration-http-metadata-probe-r4.json`, +`final-integration-http-metadata-probe-r5.json` and +`http-fault-boundary-after-r1.log`. Combined-source 10 GiB qualification, +representative S3 latency, additional history/ref scaling and fuller Git blame +semantics remain open gates. Earlier component receipts retain their original +source identities and are not final combined-source proof. diff --git a/crates/crab-http-server/src/receive_tests.rs b/crates/crab-http-server/src/receive_tests.rs index 498dcb05e..60240fca3 100644 --- a/crates/crab-http-server/src/receive_tests.rs +++ b/crates/crab-http-server/src/receive_tests.rs @@ -415,17 +415,25 @@ async fn exercise(server: Arc, branch: &str) { .await .unwrap(); - let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&repo.layout) - .await - .unwrap() - .unwrap(); - let path_state_hash = indexes.path_state_hash().unwrap(); for corruption in [ "missing descriptor", "corrupt descriptor", "malformed record", "oversized record", ] { + // Readiness can precede its repair worker's completion. Drain it before + // injecting another fault, or that worker can replace the tested record + // with legitimate graph-only readiness before the corruption probe. + server.finish_maintenance().await.unwrap(); + let indexes = crab_metadata::capsule_protocol::load_browse_indexes(&repo.layout) + .await + .unwrap() + .unwrap(); + assert_eq!( + indexes.state_digest(), + repo.open_view().await.unwrap().state_digest() + ); + let path_state_hash = indexes.path_state_hash().unwrap(); let descriptor = repo .layout .bulk_manifest_path("path-state", path_state_hash); From 97491a352d03d185bf62f1ed8e795601dbc0a920 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 07:49:46 -0700 Subject: [PATCH 15/17] fix(fsck): pin source lifetime through verification Retain decoded head CAS tokens and the pruning floor through source, data and Git verification. Reject concurrent authority movement or corrupt control bytes without rescanning immutable history. --- crab/src/cmd/fsck_store.rs | 31 +++- .../cmd/fsck_store/capsule_history_tests.rs | 153 +++++++++++++++++- crates/crab-metadata/README.md | 3 + .../src/capsule_protocol/source_retention.rs | 44 ++++- .../source_retention/tests.rs | 50 ++++++ 5 files changed, 272 insertions(+), 9 deletions(-) diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 8dc5aae48..af740c9f0 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -95,6 +95,7 @@ pub struct StoreChecker { struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, + source_lifetime: Arc, } impl StoreChecker { @@ -134,13 +135,17 @@ impl StoreChecker { ) .await?; let catalog = view.pointer_catalog().await?; - verify_capsule_history(&layout, view.root().root()).await?; + let source_lifetime = verify_capsule_history(&layout, view.root().root()).await?; Ok(Self { store, prefix, router, multipart_journal: None, - capsule: Some(CapsuleFsckState { view, catalog }), + capsule: Some(CapsuleFsckState { + view, + catalog, + source_lifetime: Arc::new(source_lifetime), + }), }) } @@ -233,6 +238,7 @@ impl StoreChecker { } } + self.check_capsule_root_stability(state).await?; Ok(issues) } @@ -258,7 +264,7 @@ impl StoreChecker { .to_owned(), )); } - Ok(()) + require_current_source_lifetime(&layout, &state.source_lifetime).await } /// Prove source pointers against the exact catalog captured by a capsule view. @@ -888,7 +894,7 @@ impl StoreChecker { async fn verify_capsule_history( layout: &crab_storage::StoreLayout, root: &crab_metadata::capsule_protocol::RepositoryRoot, -) -> Result<()> { +) -> Result { let segments = match root.history() { Some(history) => { crab_metadata::capsule_protocol::load_history_chain( @@ -981,7 +987,22 @@ async fn verify_capsule_history( .buffer_unordered(16) .try_collect::>() .await?; - Ok(()) + // Per-ref retirement and pruning can move without changing visible refs. + // Keep the capture for the later data/Git checks instead of rescanning history. + require_current_source_lifetime(layout, &retained).await?; + Ok(retained) +} + +async fn require_current_source_lifetime( + layout: &crab_storage::StoreLayout, + capture: &crab_metadata::capsule_protocol::RetainedCapsuleSources, +) -> Result<()> { + if capture.is_current(layout).await? { + return Ok(()); + } + Err(CrabError::Protocol( + "source lifetime changed during fsck; retry against one stable capture".to_owned(), + )) } fn insert_history_pointer( diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index c8b0cb691..364dbc243 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -5,8 +5,91 @@ use crab_metadata::capsule_protocol::{ LayeredCheckpoint, PackLayer, PackSourceDescriptor, PointerCatalog, RootSnapshot, }; use crab_storage::{StorageObservation, StorageObserver, StorageOperation}; -use object_store::{ObjectStoreExt, memory::InMemory}; -use std::sync::atomic::{AtomicUsize, Ordering}; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, memory::InMemory, + path::Path as ObjectPath, +}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CorruptLifetimeDuringRead { + inner: Arc, + source: ObjectPath, + head: ObjectPath, + fired: AtomicBool, +} + +impl std::fmt::Display for CorruptLifetimeDuringRead { + fn fmt(&self, output: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + output.write_str("source lifetime corruption fixture") + } +} + +#[async_trait::async_trait] +impl ObjectStore for CorruptLifetimeDuringRead { + async fn put_opts( + &self, + path: &ObjectPath, + payload: PutPayload, + options: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + + async fn put_multipart_opts( + &self, + path: &ObjectPath, + options: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + + async fn get_opts( + &self, + path: &ObjectPath, + options: GetOptions, + ) -> object_store::Result { + let full_body = !options.head && options.range.is_none(); + let response = self.inner.get_opts(path, options).await?; + if full_body && *path == self.source && !self.fired.swap(true, Ordering::SeqCst) { + self.inner + .put(&self.head, Bytes::from_static(b"corrupt").into()) + .await?; + } + Ok(response) + } + + fn delete_stream( + &self, + paths: futures_util::stream::BoxStream<'static, object_store::Result>, + ) -> futures_util::stream::BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + + fn list( + &self, + prefix: Option<&ObjectPath>, + ) -> futures_util::stream::BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&ObjectPath>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &ObjectPath, + to: &ObjectPath, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} async fn history_fixture() -> ( crab_storage::StoreLayout, @@ -104,6 +187,72 @@ struct SourceReads { count: AtomicUsize, } +#[tokio::test] +async fn fsck_rejects_lifetime_corruption_during_original_source_verification() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + let head = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await + .unwrap() + .remove(0) + .location; + let backend = Arc::new(CorruptLifetimeDuringRead { + inner: Arc::clone(layout.store().inner()), + source: layout.capsule_path(run.hash()), + head, + fired: AtomicBool::new(false), + }); + let observed = crab_storage::StoreLayout::new( + crab_storage::Store::new(backend.clone()), + layout.repo_prefix().to_owned(), + ); + let result = verify_capsule_history(&observed, root.record().root()).await; + assert!(backend.fired.load(Ordering::SeqCst)); + assert!( + result.is_err(), + "fsck accepted corrupt lifetime metadata after its original source read: {result:?}" + ); +} + +#[tokio::test] +async fn data_checks_reject_lifetime_movement_after_checker_construction() { + let (layout, root, run) = history_fixture().await; + let pointer = run_pointer(&run, root.record().digest()); + crab_metadata::capsule_protocol::retain_capsule_sources( + &layout, + "refs/heads/main", + 0, + &[pointer], + ) + .await + .unwrap(); + let checker = StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(layout.store().clone()), + layout.repo_prefix().to_owned(), + root, + ) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + crab_metadata::capsule_protocol::prune_retained_capsule_sources(&layout, 0) + .await + .unwrap(); + assert!(matches!( + checker.check_data_chain().await, + Err(CrabError::Protocol(reason)) if reason.contains("source lifetime changed") + )); +} + #[tokio::test] async fn retired_sources_are_checked_without_checkpoint_history() { let (layout, root, run) = history_fixture().await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 69c34c83d..2754bab09 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -150,6 +150,9 @@ a per-ref head under `v2/history/source-heads/` and authenticated source-pointer segments under `v2/history/source-roots/`. Foreground catalog sessions stay read-only and keep their original source ranges. GC discovers these roots in its existing history enumeration; fsck authenticates their original run proofs. +Fsck retains the decoded heads' CAS tokens and the pruning floor through later +data and Git checks. It rechecks mutable authority without decoding history again; +concurrent movement requests a retry, while invalid control bytes fail validation. Explicit history pruning advances a persistent generation floor after history CAS, trims older segments, and prevents expired publishers from restoring them. These new lifetime records do not change capsule, checkpoint, or catalog formats. diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention.rs b/crates/crab-metadata/src/capsule_protocol/source_retention.rs index 65bf1df2e..1053e3a64 100644 --- a/crates/crab-metadata/src/capsule_protocol/source_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/source_retention.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crab_storage::{ETag, Store, StoreLayout}; +use futures_util::{StreamExt, TryStreamExt}; use object_store::path::Path; use serde::{Deserialize, Serialize}; @@ -48,9 +49,12 @@ struct Segment { } /// Authenticated lifetime dependencies; they never authorize ref or object visibility. +#[derive(Debug)] pub struct RetainedCapsuleSources { paths: Vec, runs: Vec, + head_versions: BTreeMap, + floor: Option<(u64, ETag)>, } impl RetainedCapsuleSources { @@ -65,6 +69,38 @@ impl RetainedCapsuleSources { pub fn runs(&self) -> &[CapsulePointer] { &self.runs } + + /// Check whether a complete capture still names the current lifetime authority. + /// + /// The loader must have received every source head. Invalid control records + /// return an error; valid concurrent movement returns false for caller retry. + pub async fn is_current(&self, layout: &StoreLayout) -> Result { + let heads = layout + .store() + .list_prefix(&layout.capsule_source_heads_prefix()) + .await?; + if heads.len() != self.head_versions.len() + || heads + .iter() + .any(|meta| !self.head_versions.contains_key(&meta.location)) + { + return Ok(false); + } + // LIST can omit version IDs. Compare the GET CAS tokens that bound + // decoded heads so versioned buckets keep the same stability proof. + let mut versions = futures_util::stream::iter(self.head_versions.clone()) + .map(|(path, version)| async move { + let current = load_head(layout, &path).await?; + Ok::<_, MetadataError>(current.is_some_and(|(_, current)| current == version)) + }) + .buffer_unordered(16); + while let Some(current) = versions.try_next().await? { + if !current { + return Ok(false); + } + } + Ok(load_floor(layout).await?.as_ref() == self.floor.as_ref()) + } } impl Tip { @@ -347,20 +383,22 @@ pub async fn load_retained_capsule_sources( ) -> Result { let mut paths = BTreeSet::new(); let mut runs = BTreeMap::new(); + let mut head_versions = BTreeMap::new(); let mut roots = 0usize; let mut bytes = 0u64; let floor = load_floor(layout).await?; if floor.is_some() { paths.insert(layout.capsule_source_floor_path()); } - let minimum_generation = floor.map_or(0, |(floor, _)| floor); + let minimum_generation = floor.as_ref().map_or(0, |(floor, _)| *floor); for path in heads { if !paths.insert(path.clone()) { continue; } - let (head, _) = load_head(layout, path) + let (head, version) = load_head(layout, path) .await? .ok_or_else(|| corrupt("source lifetime head disappeared"))?; + head_versions.insert(path.clone(), version); if head.minimum_generation > minimum_generation { return Err(corrupt("source head exceeds the repository pruning floor")); } @@ -392,6 +430,8 @@ pub async fn load_retained_capsule_sources( Ok(RetainedCapsuleSources { paths: paths.into_iter().collect(), runs: runs.into_values().collect(), + head_versions, + floor, }) } diff --git a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs index 62d06c3d0..e12965453 100644 --- a/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/source_retention/tests.rs @@ -75,6 +75,56 @@ async fn repeated_retirement_does_not_append_or_rewrite_lifetime() { assert_eq!(layout.store().get_with_etag(&path).await.unwrap(), before); } +#[tokio::test] +async fn source_snapshot_detects_appends_and_new_ref_heads() { + let layout = fixture(); + let first = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[first]) + .await + .unwrap(); + let captured = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + assert!(captured.is_current(&layout).await.unwrap()); + let second = source(&layout, 2).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, std::slice::from_ref(&second)) + .await + .unwrap(); + assert!(!captured.is_current(&layout).await.unwrap()); + let captured = load_retained_capsule_sources(&layout, &heads(&layout).await) + .await + .unwrap(); + retain_capsule_sources(&layout, "refs/heads/other", 0, &[second]) + .await + .unwrap(); + assert!(!captured.is_current(&layout).await.unwrap()); +} + +#[tokio::test] +async fn source_snapshot_tracks_floor_without_rewriting_ref_heads() { + let layout = fixture(); + let pointer = source(&layout, 1).await; + retain_capsule_sources(&layout, "refs/heads/main", 0, &[pointer]) + .await + .unwrap(); + let paths = heads(&layout).await; + let before = layout.store().get_with_etag(&paths[0]).await.unwrap(); + let captured = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 0).await.unwrap(); + assert_eq!( + layout.store().get_with_etag(&paths[0]).await.unwrap(), + before + ); + assert!(!captured.is_current(&layout).await.unwrap()); + let captured = load_retained_capsule_sources(&layout, &paths) + .await + .unwrap(); + prune_retained_capsule_sources(&layout, 0).await.unwrap(); + assert!(captured.is_current(&layout).await.unwrap()); +} + #[tokio::test] async fn independent_ref_heads_share_source_identity_without_sharing_cas() { let layout = fixture(); From acca0aad3fd46e7664bbd3c15af139986dc2ae78 Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 08:49:35 -0700 Subject: [PATCH 16/17] fix(fsck): retain browse capture through checks --- crab/src/cmd/fsck_store.rs | 34 ++++-- .../cmd/fsck_store/capsule_history_tests.rs | 111 ++++++++++++++++++ crates/crab-metadata/README.md | 6 +- .../src/capsule_protocol/browse_retention.rs | 29 ++++- .../browse_retention/tests.rs | 33 ++++++ .../crab-metadata/src/capsule_protocol/mod.rs | 2 +- 6 files changed, 200 insertions(+), 15 deletions(-) diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index af740c9f0..98e9005d3 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -96,6 +96,8 @@ struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, source_lifetime: Arc, + browse_lifetime: + Arc>, } impl StoreChecker { @@ -145,6 +147,7 @@ impl StoreChecker { view, catalog, source_lifetime: Arc::new(source_lifetime), + browse_lifetime: Arc::new(tokio::sync::OnceCell::new()), }), }) } @@ -163,11 +166,15 @@ impl StoreChecker { ); // Current catalog health cannot prove cold captures still have their // original sources. Full fsck checks lifetime roots; mirror recipe probes stay narrow. - crab_metadata::capsule_protocol::verify_retained_browse_indexes( - &layout, - MAX_FSCK_FRONTIER_BYTES, - ) - .await?; + state + .browse_lifetime + .get_or_try_init(|| { + crab_metadata::capsule_protocol::verify_retained_browse_indexes( + &layout, + MAX_FSCK_FRONTIER_BYTES, + ) + }) + .await?; let mut issues = Vec::new(); for (hash, entry) in state.catalog.xorbs() { @@ -264,7 +271,17 @@ impl StoreChecker { .to_owned(), )); } - require_current_source_lifetime(&layout, &state.source_lifetime).await + require_current_source_lifetime(&layout, &state.source_lifetime).await?; + // Full fsck keeps browse authority through payload and final checks. + // Mirror recipe probes never initialize this expensive maintenance proof. + if let Some(capture) = state.browse_lifetime.get() + && !capture.is_current(&layout).await? + { + return Err(CrabError::Protocol( + "browse retention changed during fsck; retry against one stable capture".to_owned(), + )); + } + Ok(()) } /// Prove source pointers against the exact catalog captured by a capsule view. @@ -1902,7 +1919,7 @@ mod tests { create_manifest(store, &router, &manifest).await.unwrap(); } - async fn capsule_checker_fixture() -> ( + pub(super) async fn capsule_checker_fixture() -> ( Store, String, crab_metadata::capsule_protocol::RootSnapshot, @@ -1930,7 +1947,8 @@ mod tests { let parser = XorbParser::parse(xorb_bytes.clone()).unwrap(); assert_eq!(parser.num_chunks(), 1); let chunk = parser.chunk_meta(0).unwrap(); - let file_hash = chunk.hash; + // File identity uses ordinary Blake3; Xet's chunk hash is keyed. + let file_hash = MerkleHash::from(*blake3::hash(&content).as_bytes()); let placement = crab_xet::xorb::format::ChunkPlacement { chunk_hash: chunk.hash, xorb_hash, diff --git a/crab/src/cmd/fsck_store/capsule_history_tests.rs b/crab/src/cmd/fsck_store/capsule_history_tests.rs index 364dbc243..bf3fa7de8 100644 --- a/crab/src/cmd/fsck_store/capsule_history_tests.rs +++ b/crab/src/cmd/fsck_store/capsule_history_tests.rs @@ -224,6 +224,117 @@ async fn fsck_rejects_lifetime_corruption_during_original_source_verification() ); } +#[tokio::test] +async fn data_checks_reject_browse_corruption_during_xorb_verification() { + for readiness in [false, true] { + let (store, prefix, root, xorb_hash, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let backend = Arc::new(CorruptLifetimeDuringRead { + inner: Arc::clone(store.inner()), + source: router.xorb_path(&xorb_hash), + head: if readiness { + layout.capsule_browse_indexes_path() + } else { + layout.browse_retained_head_path() + }, + fired: AtomicBool::new(false), + }); + let store = crab_storage::Store::new(backend.clone()); + let checker = StoreChecker::for_capsule_repository( + crate::storage::store::Store::from_storage(store), + prefix, + root, + ) + .await + .unwrap(); + let result = checker.check_data_chain().await; + assert!(backend.fired.load(Ordering::SeqCst)); + assert!( + result.is_err(), + "fsck accepted corrupt browse metadata after its xorb read: readiness={readiness}, {result:?}" + ); + } +} + +#[tokio::test] +async fn final_snapshot_check_rejects_browse_movement_after_payload_verification() { + for readiness in [false, true] { + let (store, prefix, root, _, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let checker = StoreChecker::for_capsule_repository(store.clone(), prefix, root) + .await + .unwrap(); + assert!(checker.check_data_chain().await.unwrap().is_empty()); + let indexes = crab_metadata::capsule_protocol::BrowseIndexes::new( + "a".repeat(64), + "b".repeat(64), + None, + ) + .unwrap(); + if readiness { + store + .put( + &layout.capsule_browse_indexes_path(), + indexes.encode().unwrap(), + ) + .await + .unwrap(); + } else { + crab_metadata::capsule_protocol::retain_browse_indexes(&layout, &indexes, 0, None, &[]) + .await + .unwrap(); + } + assert!(matches!( + checker.check_orphan_file_index().await, + Err(CrabError::Protocol(reason)) if reason.contains("browse retention changed") + )); + } +} + +#[tokio::test] +async fn capsule_recipe_probe_does_not_require_browse_maintenance() { + let (store, prefix, root, _, _) = super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + store + .put( + &router.repo_path("v2/browse-retention"), + Bytes::from_static(b"corrupt"), + ) + .await + .unwrap(); + let checker = StoreChecker::for_capsule_repository(store, prefix, root) + .await + .unwrap(); + let (hash, file) = checker + .capsule + .as_ref() + .unwrap() + .catalog + .files() + .first_key_value() + .unwrap(); + let pointer = Pointer { + file_hash: MerkleHash::from_hex(hash).unwrap().into(), + size: file.size(), + shard_hint: None, + }; + let proof = checker + .verify_capsule_pointer_data(&[pointer], &CancellationToken::new()) + .await + .unwrap(); + assert!(proof.verified == 1 && proof.issues.is_empty(), "{proof:?}"); +} + #[tokio::test] async fn data_checks_reject_lifetime_movement_after_checker_construction() { let (layout, root, run) = history_fixture().await; diff --git a/crates/crab-metadata/README.md b/crates/crab-metadata/README.md index 2754bab09..993fdbcbb 100644 --- a/crates/crab-metadata/README.md +++ b/crates/crab-metadata/README.md @@ -133,8 +133,10 @@ requires background browse maintenance before GC; there is no scan fallback. Full capsule fsck also authenticates the retained chain, decodes complete graph/path-state indexes, and verifies physical-index and original source bytes, including superseded ref-only runs. Shared captures stream each source body once -during this phase. Lifetime/readiness movement during verification requires a retry; -mirror pointer-recipe probes do not acquire this full maintenance workload. +during this phase. Fsck holds the verified lifetime/readiness capture through +payload verification and the final snapshot check. Revalidation reads only bounded +control records; valid movement requires a retry and corruption fails validation. +Mirror pointer-recipe probes do not initialize this full maintenance workload. Explicit history pruning trims superseded captures below the oldest retained checkpoint generation only after the authoritative history CAS succeeds. Current diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs index 48b26afb3..407d15424 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention.rs @@ -288,12 +288,13 @@ pub async fn load_retained_browse_indexes( /// Verify retained index/source integrity against stable lifetime and readiness heads. /// /// Maintenance decodes complete graph/path indexes and streams original source -/// and physical-index bodies. This proves retained bytes, never ref visibility. +/// and physical-index bodies. Retain the returned capture and recheck it after +/// later phases. This proves retained bytes, never ref visibility. #[cfg(feature = "remote-index")] pub async fn verify_retained_browse_indexes( layout: &StoreLayout, maximum_bytes: u64, -) -> Result<()> { +) -> Result { let head = load_head(layout).await?; let current = super::load_browse_indexes(layout).await?; let records = match &head { @@ -363,12 +364,32 @@ pub async fn verify_retained_browse_indexes( } // Browse maintenance can change lifetime without moving a Git ref. A // concurrent prune/rebuild must yield a retry, never a clean mixed capture. - if load_head(layout).await? != head || super::load_browse_indexes(layout).await? != current { + let capture = BrowseRetentionCapture { head, current }; + if !capture.is_current(layout).await? { return Err(corrupt( "browse retention changed during verification; retry fsck", )); } - Ok(()) + Ok(capture) +} + +/// Verified browse lifetime and readiness authority retained across maintenance phases. +#[cfg(feature = "remote-index")] +#[derive(Debug)] +pub struct BrowseRetentionCapture { + head: Option<(Head, ETag)>, + current: Option, +} + +#[cfg(feature = "remote-index")] +impl BrowseRetentionCapture { + /// Revalidate control identity without decoding indexes or original sources again. + /// + /// Invalid records return an error; valid concurrent movement returns false. + pub async fn is_current(&self, layout: &StoreLayout) -> Result { + Ok(load_head(layout).await? == self.head + && super::load_browse_indexes(layout).await? == self.current) + } } /// Trim captured generations after authoritative history pruning commits. diff --git a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs index e5fb4d855..4fb9f5491 100644 --- a/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs +++ b/crates/crab-metadata/src/capsule_protocol/browse_retention/tests.rs @@ -248,6 +248,39 @@ async fn integrity_checks_every_retained_body_after_readiness_replacement() { .unwrap(); } +#[tokio::test] +#[cfg(feature = "remote-index")] +async fn verified_capture_distinguishes_control_movement_from_corruption() { + for readiness in [false, true] { + let layout = fixture(); + let capture = verify_retained_browse_indexes(&layout, 1024 * 1024) + .await + .unwrap(); + assert!(capture.is_current(&layout).await.unwrap()); + let path = if readiness { + let path = layout.capsule_browse_indexes_path(); + layout + .store() + .put(&path, indexes('a').encode().unwrap()) + .await + .unwrap(); + path + } else { + retain_browse_indexes(&layout, &indexes('a'), 1, None, &[]) + .await + .unwrap(); + layout.browse_retained_head_path() + }; + assert!(!capture.is_current(&layout).await.unwrap()); + layout + .store() + .put_overwrite(&path, Bytes::from_static(b"corrupt")) + .await + .unwrap(); + assert!(capture.is_current(&layout).await.is_err()); + } +} + #[tokio::test] #[cfg(feature = "remote-index")] async fn advertised_readiness_requires_authenticated_lifetime_ownership() { diff --git a/crates/crab-metadata/src/capsule_protocol/mod.rs b/crates/crab-metadata/src/capsule_protocol/mod.rs index de338fc22..bbecfb74f 100644 --- a/crates/crab-metadata/src/capsule_protocol/mod.rs +++ b/crates/crab-metadata/src/capsule_protocol/mod.rs @@ -33,7 +33,7 @@ mod visibility; pub use browse::load_browse_indexes; pub use browse::{BrowseIndexes, MAX_BROWSE_INDEXES_BYTES}; #[cfg(feature = "remote-index")] -pub use browse_retention::verify_retained_browse_indexes; +pub use browse_retention::{BrowseRetentionCapture, verify_retained_browse_indexes}; #[cfg(feature = "storage")] pub use browse_retention::{ RetainedBrowseIndexes, load_retained_browse_indexes, prune_retained_browse_indexes, From 3daffd89531e9341c3c50aa0fd1915a1ae46985e Mon Sep 17 00:00:00 2001 From: forhappy Date: Thu, 15 Oct 2026 10:40:20 -0700 Subject: [PATCH 17/17] fix(fsck): verify reachable pointer payloads --- crab/docs/guides/fsck.md | 8 +- crab/src/cmd/fsck_store.rs | 46 +++- crab/src/cmd/fsck_store/pointer_data_tests.rs | 226 ++++++++++++++++++ 3 files changed, 269 insertions(+), 11 deletions(-) create mode 100644 crab/src/cmd/fsck_store/pointer_data_tests.rs diff --git a/crab/docs/guides/fsck.md b/crab/docs/guides/fsck.md index e07f692c3..0312f9136 100644 --- a/crab/docs/guides/fsck.md +++ b/crab/docs/guides/fsck.md @@ -13,6 +13,12 @@ crab fsck [OPTIONS] `crab fsck` cross-checks Crab manifests, pack/index presence, the shard/xorb data chain, and coordination state against the object store. +For protocol-v2 repositories, the check pins an authenticated root, verifies +Git connectivity, and follows every reachable Crab or Git LFS pointer to its +origin payload. Crab files must reconstruct to the pointer's exact Blake3 hash +and size; LFS objects must match their SHA-256 identity and size. Full checking +reads the payload bytes, even when a local cache already contains them. + With `--repair`, it can automatically fix certain categories of issues. ## Options @@ -104,7 +110,7 @@ crab fsck: 1 error(s), 0 info, 1 repaired, 0 repair failure(s) ## Verification Boundaries -- The Git-object check currently confirms that ref-journal records are +- For legacy layouts, the Git-object check confirms that ref-journal records are readable; it does not reconstruct the current repository and run full Git reachability for missing commits, trees, or blobs. Use `crab recover history verify ` for strict Git fsck of a selected historical root, and diff --git a/crab/src/cmd/fsck_store.rs b/crab/src/cmd/fsck_store.rs index 98e9005d3..a3845ee03 100644 --- a/crab/src/cmd/fsck_store.rs +++ b/crab/src/cmd/fsck_store.rs @@ -91,13 +91,13 @@ pub struct StoreChecker { capsule: Option, } -#[derive(Clone)] struct CapsuleFsckState { view: crab_read::capsule_protocol::CapsuleRepositoryView, catalog: crab_metadata::capsule_protocol::PointerCatalog, source_lifetime: Arc, browse_lifetime: Arc>, + git_workspace: tokio::sync::OnceCell, } impl StoreChecker { @@ -148,6 +148,7 @@ impl StoreChecker { catalog, source_lifetime: Arc::new(source_lifetime), browse_lifetime: Arc::new(tokio::sync::OnceCell::new()), + git_workspace: tokio::sync::OnceCell::new(), }), }) } @@ -245,6 +246,25 @@ impl StoreChecker { } } + if issues.is_empty() { + // Keep the authenticated Git database across phases. Catalog bodies + // alone cannot prove reachable pointer coverage or whole-file/LFS bytes. + let workspace = state + .git_workspace + .get_or_try_init(|| { + check_capsule_git_connectivity(&self.store, &self.router, state) + }) + .await?; + crab_read::capsule_protocol::verify_installed_dependencies( + &layout, + workspace.path(), + state.view.refs(), + &state.catalog, + CAPSULE_GIT_SCAN_LIMITS, + &CancellationToken::new(), + ) + .await?; + } self.check_capsule_root_stability(state).await?; Ok(issues) } @@ -1214,10 +1234,10 @@ async fn install_capsule_git_packs( } async fn check_capsule_git_connectivity( - store: Store, - router: StoreLayout, - state: CapsuleFsckState, -) -> Result<()> { + store: &Store, + router: &StoreLayout, + state: &CapsuleFsckState, +) -> Result { let workspace = tempfile::tempdir()?; crab_git::initialize_bare_git_dir(workspace.path())?; let storage = store.as_storage().clone(); @@ -1233,7 +1253,7 @@ async fn check_capsule_git_connectivity( .map(|(name, oid)| (name.clone(), oid.clone())) .collect::>(); install_capsule_git_packs( - state.view, + state.view.clone(), layout, workspace.path().to_owned(), MAX_FSCK_FRONTIER_BYTES, @@ -1247,7 +1267,8 @@ async fn check_capsule_git_connectivity( .await .map_err(|error| CrabError::Internal(format!("Git connectivity scan failed: {error}")))??; crab_git::batch::verify_git_dir_blobs(workspace.path(), &scan.unchecked_blobs, &|| false) - .map_err(CrabError::Io) + .map_err(CrabError::Io)?; + Ok(workspace) } impl FsckChecker for StoreChecker { @@ -1257,13 +1278,15 @@ impl FsckChecker for StoreChecker { let store = self.store.clone(); let router = self.router.clone(); let prefix = self.prefix.clone(); - let capsule = self.capsule.clone(); Box::pin(async move { - if let Some(state) = capsule { + if let Some(state) = &self.capsule { // Pack installation validates every advertised pack. The reachable // walker then proves each ref's commit/tree/blob closure instead of // treating an authenticated ref OID as sufficient evidence. - check_capsule_git_connectivity(store, router, state).await?; + state + .git_workspace + .get_or_try_init(|| check_capsule_git_connectivity(&store, &router, state)) + .await?; return Ok(Vec::new()); } // Git-object connectivity requires a local git repo and gix-fsck. @@ -1757,6 +1780,9 @@ mod verification_tests; #[cfg(test)] mod capsule_history_tests; +#[cfg(test)] +mod pointer_data_tests; + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { diff --git a/crab/src/cmd/fsck_store/pointer_data_tests.rs b/crab/src/cmd/fsck_store/pointer_data_tests.rs new file mode 100644 index 000000000..051bc6562 --- /dev/null +++ b/crab/src/cmd/fsck_store/pointer_data_tests.rs @@ -0,0 +1,226 @@ +#![expect(clippy::unwrap_used, reason = "test assertions")] + +use super::*; +use bytes::Bytes; +use crab_metadata::capsule_protocol::{ + Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind, + CapsuleTransaction, CapsuleVisibilityDelta, FileCatalogEntry, PointerCatalog, + ShardCatalogEntry, +}; +use crab_metadata::git_visibility::GitVisibilityEdit; +use sha2::{Digest, Sha256}; + +async fn publish_pointer( + layout: &crab_storage::StoreLayout, + root: crab_metadata::capsule_protocol::RootSnapshot, + body: &[u8], + delta: &PointerCatalog, +) -> crab_metadata::capsule_protocol::RootSnapshot { + let kind = gix_object::Kind::Blob; + let oid = crab_remote::objects::object_id(kind, body) + .unwrap() + .to_string(); + let mut bytes = Vec::new(); + crab_git::pack_writer::write_pack( + &mut bytes, + std::iter::once(Ok((kind, body.len() as u64, body))), + 1024 * 1024, + || false, + ) + .unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let path = scratch.path().join("source.pack"); + std::fs::write(&path, &bytes).unwrap(); + let indexed = crab_git::pack::install_pack_file_from_path( + &scratch.path().join("indexed"), + &path, + blake3::hash(&bytes).to_hex().as_ref(), + 1024 * 1024, + true, + ) + .unwrap(); + let checksum = gix_hash::ObjectId::from_hex(indexed.git_sha1.as_bytes()).unwrap(); + let kinds = crab_git::pack_locator::encode_pack_kind_metadata(checksum, &[kind]).unwrap(); + let pack = CapsuleGitPack::new( + Bytes::from(bytes), + Bytes::from(std::fs::read(indexed.idx_path).unwrap()), + Bytes::from(std::fs::read(indexed.rev_path).unwrap()), + Bytes::from(kinds), + indexed.git_sha1, + 1, + ) + .unwrap(); + let name = "refs/tags/payload"; + let transaction = CapsuleTransaction::new( + root.record().digest(), + vec![CapsuleRefEdit::new(name, None, Some(oid.clone()), None)], + ) + .unwrap(); + let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([( + name.to_owned(), + GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid]), + )])) + .unwrap(); + let capsule = Capsule::build( + &transaction, + vec![pack], + vec![ + CapsuleSection::new( + CapsuleSectionKind::VisibilityDelta, + visibility.encode().unwrap(), + ), + CapsuleSection::new( + CapsuleSectionKind::CatalogDelta, + delta.encode_delta().unwrap(), + ), + ], + ) + .unwrap(); + crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule) + .await + .unwrap() +} + +async fn pointer_checker(case: &str) -> StoreChecker { + let (store, prefix, root, xorb_hash, shard_hash) = + super::tests::capsule_checker_fixture().await; + let router = StoreLayout::new(store.clone(), prefix.clone()); + let layout = crab_storage::StoreLayout::with_global_prefix( + store.as_storage().clone(), + router.repo_prefix().to_owned(), + router.global_prefix().to_owned(), + ); + let mut pointer = Pointer { + file_hash: blake3::hash(&[0x61; 1024]).into(), + size: 1024, + shard_hint: None, + }; + let mut delta = PointerCatalog::new(); + match case { + "missing" => pointer.file_hash = [42; 32], + "size" => pointer.size += 1, + "hash" => { + let (body, _) = store + .get_with_etag_bounded(&router.shard_path(&shard_hash), MAX_FSCK_SHARD_BYTES) + .await + .unwrap(); + let reader = ShardReader::from_bytes(body, shard_hash); + let mut recipe = reader + .get_file_info(&MerkleHash::from(pointer.file_hash)) + .unwrap() + .unwrap(); + pointer.file_hash = [42; 32]; + recipe.metadata.file_hash = MerkleHash::from(pointer.file_hash); + let mut shard = crab_xet::shard::ShardWriter::new(); + shard + .add_xorb(Arc::new(reader.get_xorb_info(&xorb_hash).unwrap().unwrap())) + .unwrap(); + shard.add_file(recipe).unwrap(); + let (body, hash) = shard.finalize().unwrap(); + delta + .insert_shard( + hash.hex(), + ShardCatalogEntry::new(body.len() as u64, vec![xorb_hash.hex()]), + ) + .unwrap(); + delta + .insert_file( + MerkleHash::from(pointer.file_hash).hex(), + FileCatalogEntry::new(pointer.size, hash.hex()), + ) + .unwrap(); + store + .put(&router.shard_path(&hash), Bytes::from(body)) + .await + .unwrap(); + } + _ => {} + } + let body = if case.starts_with("lfs-") { + let lfs = crab_git::lfs_pointer::LfsPointer { + oid: Sha256::digest(b"hello").into(), + size: 5, + extensions: Vec::new(), + }; + if case != "lfs-missing" { + let path = crab_lfs::LfsObjectStore::new(layout.store().clone(), layout.repo_prefix()) + .object_path_for(&lfs.oid); + let body = if case == "lfs-corrupt" { + b"wrong" + } else { + b"hello" + }; + layout + .store() + .put(&path, Bytes::copy_from_slice(body)) + .await + .unwrap(); + } + lfs.serialize() + } else { + pointer.serialize() + }; + let root = publish_pointer(&layout, root, &body, &delta).await; + StoreChecker::for_capsule_repository(store, prefix, root) + .await + .unwrap() +} + +#[tokio::test] +async fn full_fsck_proves_reachable_crab_and_lfs_payloads() { + let mut outcomes = Vec::new(); + for case in [ + "valid", + "missing", + "size", + "hash", + "lfs-valid", + "lfs-missing", + "lfs-corrupt", + ] { + let checker = Box::pin(pointer_checker(case)).await; + let (_, outcome) = crate::cmd::fsck::run_fsck( + &crate::cmd::fsck::FsckArgs::default(), + &checker, + &crate::cmd::fsck::NullRepairer, + &CancellationToken::new(), + Duration::from_hours(1), + None, + ) + .await + .unwrap(); + outcomes.push((case, outcome.errors == 0)); + } + assert_eq!( + outcomes, + vec![ + ("valid", true), + ("missing", false), + ("size", false), + ("hash", false), + ("lfs-valid", true), + ("lfs-missing", false), + ("lfs-corrupt", false) + ] + ); +} + +#[tokio::test] +async fn fsck_retains_one_git_database_through_data_checks_then_releases_it() { + let checker = Box::pin(pointer_checker("valid")).await; + checker.check_git_objects().await.unwrap(); + let state = checker.capsule.as_ref().unwrap(); + let original = state.git_workspace.get().unwrap().path().to_owned(); + let clean = checker.check_data_chain().await.unwrap().is_empty(); + let reused = state.git_workspace.get().unwrap().path() == original; + drop(checker); + assert!(clean && reused && !original.exists()); +} + +#[tokio::test] +async fn data_checks_prove_pointer_coverage_without_a_prior_git_phase() { + let checker = Box::pin(pointer_checker("missing")).await; + assert!(matches!(checker.check_data_chain().await, + Err(CrabError::CorruptObject { reason, .. }) if reason.contains("absent from the catalog") + )); +}