From c8a1effe74cbbc15ae8fcf754cc5817924b58310 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:42:19 -0800 Subject: [PATCH 01/70] fix(graph): reconcile verified retirement and retry state Refs #4616 Combine the proven bounded receipt/ownership cleanup fixes with post-unlink collection-state ordering and compatible filesystem regressions. Replace the old separately reviewed PR history without rewriting either branch, and preserve all exclusion and fail-closed checks. --- .../src/generation_runtime.rs | 65 +- .../tracedecay-graph-db/src/sealed_store.rs | 576 ++++++++++++++++-- .../tests/graph_db_suite/runtime_contract.rs | 41 +- .../src/capability_dir.rs | 355 ++++++++++- .../tracedecay-private-fs/src/framed_log.rs | 9 +- 5 files changed, 976 insertions(+), 70 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation_runtime.rs b/crates/tracedecay-graph-db/src/generation_runtime.rs index d2fdd35dd58..a514bbc76de 100644 --- a/crates/tracedecay-graph-db/src/generation_runtime.rs +++ b/crates/tracedecay-graph-db/src/generation_runtime.rs @@ -1840,6 +1840,11 @@ impl GraphDb { self.retire_sealed_generation_store(locator)?; return Ok(GenerationContentsDeletion::RetentionPending); } + // Native row deletion may already have committed, but the generation + // is not collected until its independently durable sealed artifact is + // gone. If unlink refuses, keep the in-memory retirement state retryable + // instead of advertising collection while bytes still exist on disk. + self.retire_sealed_generation_store(locator)?; let mut state = self.wait_verified_generations_write()?; state.known.remove(locator); state.quarantined.remove(locator); @@ -1858,7 +1863,6 @@ impl GraphDb { state.heads.remove(&locator.projection); } drop(state); - self.retire_sealed_generation_store(locator)?; Ok(GenerationContentsDeletion::Deleted) } @@ -4001,20 +4005,73 @@ mod tests { .delete_generation_contents(&locator, &|| Ok(())) .expect_err("a failed sealed unlink must not complete retention"); assert!( - error - .to_string() - .contains("sealed directory removal failed") + matches!( + &error, + GraphDbError::Unavailable { message } + if message.starts_with("sealed generation store sealed directory removal failed:") + || message.starts_with("sealed generation store sealed generation retirement refused:") + ), + "an unremovable artifact must fail at the sealed-directory unlink boundary: {error}" ); assert_eq!(std::fs::read(&directory).unwrap(), b"not a directory"); + assert!( + !database + .wait_verified_generations_write() + .unwrap() + .collected + .contains(&locator), + "a failed sealed unlink must not mark its generation collected" + ); std::fs::remove_file(&directory).unwrap(); std::fs::create_dir(&directory).unwrap(); + // Retirement requires an exact durable identity from the directory + // being removed. A receipt-free synthetic directory must never confer + // deletion authority when the strict collector is integrated. + let identity = generation.identity(); + let receipt = serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": locator.projection.namespace.as_str(), + "projection": locator.projection.projection.as_str(), + "generation": locator.generation.as_str(), + "physical_namespace": namespace.as_str(), + "recovered_digest": sealed_digest(&generation).as_str(), + "entities": 0, + "relations": 0, + "graph_format": GraphFormatVersion::current().get(), + "row_sum": null, + "base": null, + "source_generation": identity.source_generation.as_str(), + "watermark": identity.watermark.as_str() + }); + std::fs::write( + directory.join("sealed.json"), + serde_json::to_vec(&receipt).unwrap(), + ) + .unwrap(); std::fs::write(directory.join("derived-rows"), b"retired").unwrap(); + let unrelated = directory.parent().unwrap().join("unrelated-entry"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("sentinel"), b"untouched").unwrap(); assert!(matches!( database.delete_generation_contents(&locator, &|| Ok(())), Ok(GenerationContentsDeletion::Deleted) )); + assert_eq!( + std::fs::read(unrelated.join("sentinel")).unwrap(), + b"untouched", + "retiring one sealed generation must not traverse another child" + ); assert!(!directory.exists()); + assert!( + database + .wait_verified_generations_write() + .unwrap() + .collected + .contains(&locator), + "only successful physical retirement may mark the generation collected" + ); // Idempotent replay after the directory is gone still succeeds. assert!(matches!( database.delete_generation_contents(&locator, &|| Ok(())), diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e6441489824..988c8ccf94e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -112,7 +112,7 @@ pub(crate) fn open_direct_sealed_generation( sealed_generation_directory(&sealed_store_root(database_path), &physical_namespace); let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); let sealed_path = directory.join(SEALED_STORE_DATABASE_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { + let receipt_bytes = match read_sealed_receipt_bytes(&receipt_path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), @@ -207,6 +207,23 @@ pub(crate) fn open_direct_sealed_generation( const SEALED_COPY_GUARD_CHUNK_ROWS: usize = 4096; const SEALED_STORE_RECEIPT_VERSION: u32 = 1; +/// A proof marker contains one canonical recovered digest, never a payload. +const MAX_SEALED_CHECK_BYTES: usize = 256; +/// Identity metadata, never an unbounded content read. +const MAX_SEALED_RECEIPT_BYTES: usize = 64 * 1024; + +fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result> { + // The shared reader opens without following a final symlink, rejects FIFOs + // and other nonregular files, and checks size against the same held handle. + match tracedecay_private_fs::framed_log::read_bounded(path, MAX_SEALED_RECEIPT_BYTES)? { + Some(bytes) => Ok(bytes), + None => Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "sealed generation receipt is missing", + )), + } +} + pub(crate) const SEALED_STORE_DATABASE_FILE: &str = "generation.grafeo"; const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` @@ -477,7 +494,11 @@ impl SealedGenerationStore { /// owner to retry; a sealed-only live head never enters this path. fn discard(&self) -> Result<(), GraphDbError> { let _ = self.close(); - remove_sealed_directory_checked(&self.directory) + retire_sealed_directory_checked( + &self.directory, + &self.locator, + Some(&self.recovered_digest), + ) } } @@ -566,7 +587,7 @@ pub fn census_sealed_store( census.abandoned_staging_bytes += bytes; continue; } - let receipt = std::fs::read(path.join(SEALED_STORE_RECEIPT_FILE)) + let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) .ok() .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); let Some(receipt) = receipt else { @@ -596,7 +617,9 @@ fn directory_bytes( while let Some(directory) = pending.pop() { for entry in std::fs::read_dir(&directory)? { let entry = entry?; - let metadata = entry.metadata()?; + // Count only the link itself: following a foreign directory link + // can mischarge its entire tree or revisit an ancestor forever. + let metadata = std::fs::symlink_metadata(entry.path())?; if metadata.is_dir() { pending.push(entry.path()); continue; @@ -624,40 +647,40 @@ pub(crate) fn discard_superseded_sealed_generations( database_path: &Path, ) -> Result<(), GraphDbError> { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(sealed_store_io_failure("sealed root read failed", error)), - }; - for entry in entries { - let entry = - entry.map_err(|error| sealed_store_io_failure("sealed root read failed", error))?; - let path = entry.path(); - if entry - .file_name() - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) - { - continue; - } - let current = load_sealed_store_receipt(&path) - .ok() - .flatten() - .is_some_and(|receipt| receipt.graph_format == GraphFormatVersion::current().get()); - if !current { - match std::fs::remove_dir_all(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(sealed_store_io_failure( - "superseded sealed generation removal failed", - error, - )); - } + let reclaimed = tracedecay_private_fs::capability_dir::remove_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), + MAX_SEALED_RECEIPT_BYTES, + |name, bytes| { + let Some(name) = name.to_str() else { + return false; + }; + if name.len() != 64 || !name.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return false; } - } + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return false; + }; + let recognized_form = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + recognized_form + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.physical_namespace == format!("generation:{name}") + && receipt.graph_format > 0 + && receipt.graph_format < GraphFormatVersion::current().get() + }, + ); + match reclaimed { + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(sealed_store_io_failure( + "superseded sealed generation removal failed", + error, + )), } - Ok(()) } /// Removes every `.staging-*` directory a seal left behind under the store's @@ -695,9 +718,12 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { continue; } let path = entry.path(); - match std::fs::remove_dir_all(&path) { - Ok(()) => removed += 1, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + match tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + &root, + &entry.file_name(), + ) { + Ok(true) => removed += 1, + Ok(false) => {} Err(error) => tracing::warn!( event = "sealed_staging_sweep_failed", path = %path.display(), @@ -737,9 +763,10 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) { let path = entry.path(); - if let Err(error) = std::fs::remove_dir_all(&path) - && error.kind() != std::io::ErrorKind::NotFound - { + if let Err(error) = tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + &root, + &entry.file_name(), + ) { tracing::warn!( event = "graph_row_spill_sweep_failed", path = %path.display(), @@ -752,14 +779,65 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { - match std::fs::remove_dir_all(directory) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(sealed_store_io_failure( - "sealed directory removal failed", - error, - )), + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed directory has no parent"))?; + let name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed directory has no entry name"))?; + tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(root, name) + .map(|_| ()) + .map_err(|error| sealed_store_io_failure("sealed directory removal failed", error)) +} + +/// Accepted replay retirement is not permission to recursively delete a +/// substituted or unrecognizable artifact. Require the exact durable identity +/// under a held child capability; preserve an unverified directory for retry. +fn retire_sealed_directory_checked( + directory: &Path, + locator: &GenerationLocator, + expected_digest: Option<&str>, +) -> Result<(), GraphDbError> { + let physical = locator.physical_namespace()?; + let root = directory + .parent() + .ok_or_else(|| GraphDbError::unavailable("sealed retirement directory has no parent"))?; + if sealed_generation_directory(root, &physical) != directory { + return Err(GraphDbError::unavailable( + "sealed retirement directory does not match generation identity", + )); } + let name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed retirement directory has no entry name"))?; + let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), + MAX_SEALED_RECEIPT_BYTES, + |bytes| { + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return false; + }; + let form_valid = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + form_valid + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.graph_format > 0 + && receipt.graph_format <= GraphFormatVersion::current().get() + && receipt.physical_namespace == physical.as_str() + && receipt.namespace == locator.projection.namespace.as_str() + && receipt.projection == locator.projection.projection.as_str() + && receipt.generation == locator.generation.as_str() + && expected_digest.is_none_or(|digest| receipt.recovered_digest == digest) + }, + ); + result + .map(|_| ()) + .map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error)) } /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted @@ -1694,7 +1772,11 @@ impl GraphDb { }; let physical_namespace = locator.physical_namespace()?; let root = sealed_store_root(database_path); - remove_sealed_directory_checked(&sealed_generation_directory(&root, &physical_namespace)) + retire_sealed_directory_checked( + &sealed_generation_directory(&root, &physical_namespace), + locator, + None, + ) } } @@ -1786,8 +1868,16 @@ fn build_or_open_sealed_store( .and_then(|name| name.to_str()) .unwrap_or("sealed") )); - remove_sealed_directory(&staging); - std::fs::create_dir_all(&staging) + // A failed cleanup must never be followed by create_dir_all: it would + // accept a pre-existing staging symlink and write into its foreign target. + std::fs::create_dir_all(&root) + .map_err(|error| sealed_store_io_failure("sealed root create failed", error))?; + let staging_name = staging + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; + tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(&root, staging_name) + .map_err(|error| sealed_store_io_failure("staging directory cleanup refused", error))?; + std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check) .inspect_err(|_| remove_sealed_directory(&staging))?; @@ -2426,7 +2516,7 @@ fn load_sealed_store_receipt( directory: &Path, ) -> Result, GraphDbError> { let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); - let receipt_bytes = match std::fs::read(&receipt_path) { + let receipt_bytes = match read_sealed_receipt_bytes(&receipt_path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(sealed_store_io_failure("receipt read failed", error)), @@ -2441,9 +2531,12 @@ fn sealed_store_check_matches( expected_digest: &str, ) -> Result { let path = directory.join(SEALED_STORE_CHECKED_FILE); - match std::fs::read_to_string(&path) { - Ok(digest) => Ok(digest == expected_digest), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + // This is a rebuildable proof hint. Invalid or overlarge bytes cause a + // fresh proof, never an unbounded allocation or a trusted cache hit. + match tracedecay_private_fs::framed_log::read_bounded(&path, MAX_SEALED_CHECK_BYTES) { + Ok(Some(bytes)) => Ok(bytes == expected_digest.as_bytes()), + Ok(None) => Ok(false), + Err(error) if error.kind() == std::io::ErrorKind::InvalidData => Ok(false), Err(error) => Err(sealed_store_io_failure("sealed check read failed", error)), } } @@ -2452,8 +2545,15 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result if sealed_store_check_matches(directory, expected_digest)? { return Ok(()); } - std::fs::write(directory.join(SEALED_STORE_CHECKED_FILE), expected_digest) - .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) + // Never truncate an existing symlink target: the marker is a rebuildable + // cache, so stage a regular file and publish it by atomic replacement. + tracedecay_private_fs::framed_log::atomic_write( + &directory.join(SEALED_STORE_CHECKED_FILE), + "sealed-check", + expected_digest.as_bytes(), + tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, + ) + .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) } fn open_sealed_store_checked( @@ -2730,6 +2830,298 @@ pub(crate) fn sealed_copy_proof( Ok(canonical_bytes) } +#[cfg(test)] +mod receipt_budget_tests { + #[cfg(unix)] + use std::collections::BTreeSet; + use std::path::Path; + + use super::*; + + fn receipt(format: u32, name: &str, generation: &str) -> Vec { + serde_json::to_vec(&SealedStoreReceiptV1 { + version: SEALED_STORE_RECEIPT_VERSION, + form: SEALED_STORE_FORM_COMPACT.to_owned(), + namespace: "test".to_owned(), + projection: "code".to_owned(), + generation: generation.to_owned(), + physical_namespace: format!("generation:{name}"), + recovered_digest: "digest".to_owned(), + entities: 0, + relations: 0, + graph_format: format, + row_sum: None, + base: None, + source_generation: None, + watermark: None, + }) + .unwrap() + } + + fn generation(root: &Path, digit: char, contents: &[u8]) -> PathBuf { + let directory = root.join(digit.to_string().repeat(64)); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write(directory.join(SEALED_STORE_RECEIPT_FILE), contents).unwrap(); + directory + } + + #[test] + fn sealed_receipt_reads_bound_memory_even_for_json_padded_beyond_the_limit() { + let temp = tempfile::tempdir().unwrap(); + let receipt_path = temp.path().join(SEALED_STORE_RECEIPT_FILE); + let normal = receipt(GraphFormatVersion::current().get(), &"a".repeat(64), "head"); + std::fs::write(&receipt_path, &normal).unwrap(); + assert_eq!(read_sealed_receipt_bytes(&receipt_path).unwrap(), normal); + + let oversized = [normal, vec![b' '; MAX_SEALED_RECEIPT_BYTES]].concat(); + assert!(serde_json::from_slice::(&oversized).is_ok()); + std::fs::write(&receipt_path, oversized).unwrap(); + assert_eq!( + read_sealed_receipt_bytes(&receipt_path).unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + // On Unix this is sparse, so the multi-gigabyte bound is exercised + // without allocating a multi-gigabyte test artifact. + #[cfg(unix)] + { + std::fs::File::create(&receipt_path) + .unwrap() + .set_len(15 * 1024 * 1024 * 1024) + .unwrap(); + assert_eq!( + read_sealed_receipt_bytes(&receipt_path).unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); + } + } + + #[test] + fn sealed_check_marker_is_bounded_and_never_overwrites_a_foreign_target() { + let temp = tempfile::tempdir().unwrap(); + let directory = temp.path().join("sealed"); + std::fs::create_dir(&directory).unwrap(); + let marker = directory.join(SEALED_STORE_CHECKED_FILE); + let digest = format!("sha256:{}", "a".repeat(64)); + + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + assert_eq!(std::fs::read(&marker).unwrap(), digest.as_bytes()); + + let mut oversized = digest.as_bytes().to_vec(); + oversized.extend(vec![b' '; MAX_SEALED_CHECK_BYTES]); + std::fs::write(&marker, oversized).unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + + #[cfg(unix)] + { + // Sparse bytes establish the large-file refusal without writing + // or allocating a multi-gigabyte test payload. + std::fs::File::create(&marker) + .unwrap() + .set_len(15 * 1024 * 1024 * 1024) + .unwrap(); + assert!(!sealed_store_check_matches(&directory, &digest).unwrap()); + persist_sealed_store_check(&directory, &digest).unwrap(); + assert!(sealed_store_check_matches(&directory, &digest).unwrap()); + + let foreign = temp.path().join("foreign"); + std::fs::remove_file(&marker).unwrap(); + std::fs::write(&foreign, digest.as_bytes()).unwrap(); + std::os::unix::fs::symlink(&foreign, &marker).unwrap(); + assert!( + sealed_store_check_matches(&directory, &digest).is_err(), + "a symlink cannot supply proof evidence" + ); + + std::fs::remove_file(&marker).unwrap(); + std::fs::write(&foreign, b"foreign sentinel").unwrap(); + std::os::unix::fs::symlink(&foreign, &marker).unwrap(); + assert!( + persist_sealed_store_check(&directory, &digest).is_err(), + "publishing a proof marker must refuse a foreign symlink" + ); + assert_eq!(std::fs::read(&foreign).unwrap(), b"foreign sentinel"); + } + } + + #[test] + fn obsolete_format_cleanup_keeps_unknown_receipts_and_other_files() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir_all(&root).unwrap(); + let current = GraphFormatVersion::current().get(); + assert!(current > 0); + let old = generation(&root, 'a', &receipt(current - 1, &"a".repeat(64), "old")); + let live = generation(&root, 'b', &receipt(current, &"b".repeat(64), "live")); + let invalid = generation(&root, 'c', b"not a receipt"); + // Zero is not a valid GraphFormatVersion, even when the JSON is well formed. + let zero_format = generation(&root, '0', &receipt(0, &"0".repeat(64), "zero")); + let future = generation( + &root, + 'f', + &receipt(current.saturating_add(1), &"f".repeat(64), "future"), + ); + let padded = [ + receipt(current - 1, &"d".repeat(64), "padded"), + vec![b' '; MAX_SEALED_RECEIPT_BYTES], + ] + .concat(); + let oversized = generation(&root, 'd', &padded); + let missing = root.join("e".repeat(64)); + std::fs::create_dir_all(&missing).unwrap(); + let unrelated = root.join("unrelated.txt"); + std::fs::write(&unrelated, b"foreign").unwrap(); + // A symlink to a syntactically valid old receipt must never confer + // deletion authority over the directory that holds the link. + #[cfg(unix)] + let linked_receipt = { + let directory = root.join("1".repeat(64)); + std::fs::create_dir_all(&directory).unwrap(); + let foreign = temp.path().join("foreign-receipt.json"); + std::fs::write(&foreign, receipt(current - 1, &"1".repeat(64), "foreign")).unwrap(); + std::os::unix::fs::symlink(&foreign, directory.join(SEALED_STORE_RECEIPT_FILE)) + .unwrap(); + std::fs::write(directory.join("sentinel"), b"unowned").unwrap(); + (directory, foreign) + }; + for dir in [ + &old, + &live, + &invalid, + &zero_format, + &oversized, + &missing, + &future, + ] { + std::fs::write(dir.join("sentinel"), b"owned").unwrap(); + } + + discard_superseded_sealed_generations(&database).unwrap(); + assert!(!old.exists(), "verified old-format receipt is reclaimable"); + for dir in [&live, &invalid, &zero_format, &oversized, &missing, &future] { + assert_eq!(std::fs::read(dir.join("sentinel")).unwrap(), b"owned"); + } + assert_eq!(std::fs::read(&unrelated).unwrap(), b"foreign"); + #[cfg(unix)] + { + assert_eq!( + std::fs::read(linked_receipt.0.join("sentinel")).unwrap(), + b"unowned" + ); + assert!(linked_receipt.1.is_file()); + assert_eq!( + read_sealed_receipt_bytes(&linked_receipt.0.join(SEALED_STORE_RECEIPT_FILE)) + .unwrap_err() + .kind(), + std::io::ErrorKind::InvalidInput + ); + } + } + + #[test] + fn retired_sealed_generation_requires_its_exact_receipt_identity() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new("generation:retirement").unwrap(), + ); + let physical = locator.physical_namespace().unwrap(); + let directory = sealed_generation_directory(&root, &physical); + std::fs::create_dir_all(&directory).unwrap(); + let name = directory.file_name().unwrap().to_str().unwrap(); + let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); + std::fs::write( + &receipt_path, + receipt(GraphFormatVersion::current().get(), name, "generation:other"), + ) + .unwrap(); + std::fs::write(directory.join("sentinel"), b"preserve").unwrap(); + assert!(retire_sealed_directory_checked(&directory, &locator, None).is_err()); + assert_eq!( + std::fs::read(directory.join("sentinel")).unwrap(), + b"preserve" + ); + std::fs::write( + &receipt_path, + receipt( + GraphFormatVersion::current().get(), + name, + locator.generation.as_str(), + ), + ) + .unwrap(); + assert!(retire_sealed_directory_checked(&directory, &locator, Some("wrong")).is_err()); + assert_eq!( + std::fs::read(directory.join("sentinel")).unwrap(), + b"preserve" + ); + retire_sealed_directory_checked(&directory, &locator, Some("digest")).unwrap(); + assert!( + !directory.exists(), + "verified retired generation is physically removed" + ); + retire_sealed_directory_checked(&directory, &locator, Some("digest")).unwrap(); + } + + #[cfg(unix)] + #[test] + fn census_handles_seventy_large_sparse_generations_without_loading_containers() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir_all(&root).unwrap(); + const FILE_BYTES: u64 = 220 * 1024 * 1024; + for index in 0..70 { + let name = format!("{index:064x}"); + let dir = root.join(&name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join(SEALED_STORE_RECEIPT_FILE), + receipt( + GraphFormatVersion::current().get(), + &name, + &format!("g{index}"), + ), + ) + .unwrap(); + std::fs::File::create(dir.join(SEALED_STORE_DATABASE_FILE)) + .unwrap() + .set_len(FILE_BYTES) + .unwrap(); + } + // A sealed directory must never account bytes outside its own tree. + // A missing target must also remain a harmless link, not abort census. + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::File::create(foreign.join("payload")) + .unwrap() + .set_len(20 * 1024 * 1024 * 1024) + .unwrap(); + let oldest = root.join("0".repeat(64)); + std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); + std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); + let census = census_sealed_store(&database, &BTreeSet::from(["g69".to_owned()])).unwrap(); + assert_eq!(census.head_count, 1); + assert_eq!(census.superseded_count, 69); + assert_eq!(census.unrecognized_count, 0); + assert!(census.head_bytes >= FILE_BYTES); + assert!(census.superseded_bytes >= 69 * FILE_BYTES); + assert!( + census.superseded_bytes < 70 * FILE_BYTES, + "foreign link targets must not contribute to sealed generation bytes" + ); + } +} + #[cfg(test)] mod build_tests { use std::collections::{BTreeMap, BTreeSet}; @@ -2995,6 +3387,74 @@ mod build_tests { ); } + /// A leftover staging symlink must not become a writable artifact root. + /// Refusing it preserves the foreign target rather than following it after + /// a best-effort cleanup that did not actually unlink the entry. + #[cfg(unix)] + #[test] + fn sealed_build_refuses_a_foreign_staging_symlink() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let directory = + sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + std::fs::create_dir_all(&root).unwrap(); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"unchanged").unwrap(); + std::os::unix::fs::symlink(&foreign, &staging).unwrap(); + + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!(result.is_err(), "a symlink must refuse the build"); + assert!( + std::fs::symlink_metadata(&staging) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"unchanged" + ); + assert!( + !foreign.join(SEALED_STORE_DATABASE_FILE).exists() + && !foreign.join("sealed.json").exists(), + "the foreign directory must not receive a container or receipt" + ); + assert!(!directory.exists(), "no generation was installed"); + + // Removing the foreign link must restore a successful, verified seal. + // A fixture that always refuses to build would not prove the fix. + std::fs::remove_file(&staging).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("a normal staging directory must still produce a sealed generation"); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!( + directory.join(SEALED_STORE_DATABASE_FILE).is_file(), + "valid staging must install a verified sealed container" + ); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index 123b4e9f3f7..c69dbf3222a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -1026,7 +1026,7 @@ fn wrong_tracedecay_format_requires_reset() { } #[test] -fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generations() { +fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); // Format 4 kept a relation's payload properties on its locator as well as @@ -1063,13 +1063,46 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati ) .unwrap(); raw.close().unwrap(); - let sealed_generation = temp.path().join("graph.sealed").join("0".repeat(64)); + let sealed_generation_name = "0".repeat(64); + let sealed_generation = temp + .path() + .join("graph.sealed") + .join(&sealed_generation_name); std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), b"format 4 bytes", ) .unwrap(); + // A prior-format sealed directory is deletable only when its receipt + // proves both the old format and the matching physical namespace. + std::fs::write( + sealed_generation.join("sealed.json"), + serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": "project", + "projection": "code", + "generation": "g0", + "physical_namespace": format!("generation:{sealed_generation_name}"), + "recovered_digest": "sha256:old", + "entities": 1, + "relations": 0, + "graph_format": previous_format as u32, + }) + .to_string(), + ) + .unwrap(); + + // An unverified artifact may belong to a future build. A format reset + // cannot infer its age or ownership from a directory name alone. + let unproven_generation = temp.path().join("graph.sealed").join("2".repeat(64)); + std::fs::create_dir_all(&unproven_generation).unwrap(); + std::fs::write( + unproven_generation.join("generation.grafeo"), + b"unknown sealed bytes", + ) + .unwrap(); // A generation this build sealed before the staging container was // replaced, as a publication racing the rebuild leaves it. @@ -1096,6 +1129,10 @@ fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generati let (_registered, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); assert!(!sealed_generation.exists()); + assert!( + unproven_generation.join("generation.grafeo").exists(), + "an unverified generation must survive a format reset" + ); assert!(current_generation.join("sealed.json").exists()); assert_eq!( db.entity(&namespace(), &entity_id("stale"), live()) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c0c846fb8f4..15e7e6cf401 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -5,11 +5,19 @@ //! Every mutation is relative to an open `Dir` handle so a parent path //! swapped for a symlink cannot redirect the operation. +#[cfg(unix)] +use std::ffi::CString; use std::ffi::OsStr; use std::io; +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd}; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; +use std::path::Path; -#[cfg(windows)] use cap_fs_ext::DirExt; +#[cfg(windows)] +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; use cap_std::fs::{Dir, File, OpenOptions}; @@ -72,8 +80,6 @@ pub fn rename_noreplace( ) -> io::Result<()> { #[cfg(all(target_os = "linux", target_env = "gnu"))] { - use std::os::fd::AsRawFd; - crate::rename_noreplace::rename_noreplace_at( from_parent.as_raw_fd(), from, @@ -83,8 +89,6 @@ pub fn rename_noreplace( } #[cfg(target_os = "macos")] { - use std::os::fd::AsRawFd; - crate::rename_noreplace::rename_noreplace_at( from_parent.as_raw_fd(), from, @@ -212,6 +216,179 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { } } +fn single_component(name: &OsStr) -> io::Result<()> { + let mut parts = Path::new(name).components(); + if !matches!(parts.next(), Some(std::path::Component::Normal(_))) + || parts.next().is_some() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "directory operation requires one entry name", + )); + } + Ok(()) +} + +fn open_root_nofollow(root_path: &Path) -> io::Result { + let parent_path = root_path + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "sealed root has no parent"))?; + let root_name = root_path + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "sealed root has no name"))?; + let parent = Dir::open_ambient_dir(parent_path, cap_std::ambient_authority())?; + parent.open_dir_nofollow(root_name) +} + +/// Remove exactly one held child directory without following a substituted +/// sealed-root or child symlink. Missing entries are idempotent; other failures +/// remain errors so callers can retain retry ownership. +pub fn remove_child_directory_nofollow(root_path: &Path, name: &OsStr) -> io::Result { + single_component(name)?; + let root = match open_root_nofollow(root_path) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let child = match root.open_dir_nofollow(name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + remove_open_dir_all_nofollow(child)?; + Ok(true) +} + +/// Retire a named generation only when a bounded receipt read from the SAME +/// opened directory proves the requested identity. Missing is idempotent; +/// unknown receipts are refusals, never permission to discard durable bytes. +pub fn remove_receipted_child_directory_nofollow( + root_path: &Path, + child_name: &OsStr, + receipt_name: &OsStr, + maximum: usize, + authorized: impl FnOnce(&[u8]) -> bool, +) -> io::Result { + single_component(child_name)?; + single_component(receipt_name)?; + let root = match open_root_nofollow(root_path) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let child = match root.open_dir_nofollow(child_name) { + Ok(child) => child, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let receipt = read_bounded_child_receipt(&child, receipt_name, maximum)?; + if !authorized(&receipt) { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sealed generation retirement receipt did not prove its ownership", + )); + } + remove_open_dir_all_nofollow(child)?; + Ok(true) +} + +/// Removes only children authorized by a bounded receipt read from the exact +/// directory handle that will be deleted. The sealed root and each child are +/// held capabilities: replacing either pathname after receipt validation +/// cannot redirect recursive removal to a different tree. +/// +/// The caller must own exclusive store maintenance. Unknown, invalid and +/// unreadable child receipts preserve their directory; an actual deletion +/// failure is returned so retirement can retry instead of claiming success. +pub fn remove_receipted_directories_nofollow( + root_path: &Path, + receipt_name: &OsStr, + maximum: usize, + mut authorized: impl FnMut(&OsStr, &[u8]) -> bool, +) -> io::Result { + // Receipt names are always a single sibling: a slash-bearing name could + // otherwise traverse a different directory even with O_NOFOLLOW. + let mut receipt_components = Path::new(receipt_name).components(); + if !matches!( + receipt_components.next(), + Some(std::path::Component::Normal(_)) + ) || receipt_components.next().is_some() + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "receipt must be a single filename", + )); + } + let root = open_root_nofollow(root_path)?; + let mut removed = 0usize; + for entry in root.read_dir(".")? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + let name = entry.file_name(); + // Do not trust the DirEntry's former pathname kind after a rename. + let child = match root.open_dir_nofollow(&name) { + Ok(child) => child, + Err(error) => { + tracing::warn!(%error, entry = ?name, "sealed child could not be verified"); + continue; + } + }; + let receipt = match read_bounded_child_receipt(&child, receipt_name, maximum) { + Ok(receipt) => receipt, + Err(error) => { + tracing::warn!(%error, entry = ?name, "unverified sealed receipt preserved"); + continue; + } + }; + if authorized(&name, &receipt) { + remove_open_dir_all_nofollow(child)?; + removed = removed.saturating_add(1); + } + } + Ok(removed) +} + +#[cfg(unix)] +fn read_bounded_child_receipt( + directory: &Dir, + name: &OsStr, + maximum: usize, +) -> io::Result> { + let name = CString::new(name.as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL") + })?; + // SAFETY: the held directory fd is valid; name is a NUL-terminated + // sibling component. O_NOFOLLOW prevents link traversal and O_NONBLOCK + // prevents a substituted FIFO from blocking before the regular-file check. + let descriptor = unsafe { + libc::openat( + directory.as_raw_fd(), + name.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, + ) + }; + if descriptor < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: openat returned a new owned fd and this File is its sole owner. + let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; + crate::framed_log::read_bounded_open_file(file, maximum) +} + +#[cfg(windows)] +fn read_bounded_child_receipt( + directory: &Dir, + name: &OsStr, + maximum: usize, +) -> io::Result> { + let mut options = OpenOptions::new(); + options.read(true).follow(FollowSymlinks::No); + let file = directory.open_with(name, &options)?.into_std(); + crate::framed_log::read_bounded_open_file(file, maximum) +} + /// Removes the held directory without following child symlinks. pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(not(windows))] @@ -373,6 +550,174 @@ mod tests { ); } + + /// A parent-path swap after receipt validation must not redirect cleanup: + /// the already-open child is removed, and matching foreign bytes survive. + #[cfg(unix)] + #[test] + fn verified_cleanup_remains_bound_when_sealed_root_is_substituted() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("owned-displaced"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join("a")).unwrap(); + std::fs::create_dir_all(foreign.join("a")).unwrap(); + std::fs::write(root.join("a/sealed.json"), b"owned receipt").unwrap(); + std::fs::write(root.join("a/owned"), b"owned").unwrap(); + std::fs::write(foreign.join("a/sentinel"), b"foreign").unwrap(); + + let removed = remove_receipted_directories_nofollow( + &root, + OsStr::new("sealed.json"), + 1024, + |name, bytes| { + assert_eq!(name, OsStr::new("a")); + assert_eq!(bytes, b"owned receipt"); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + true + }, + ) + .unwrap(); + assert_eq!(removed, 1); + assert!(!displaced.join("a").exists()); + assert_eq!(std::fs::read(foreign.join("a/sentinel")).unwrap(), b"foreign"); + } + + /// The child also remains the exact opened directory when its original + /// entry name becomes occupied by an unrelated real directory. + #[cfg(unix)] + #[test] + fn verified_cleanup_does_not_remove_a_replacement_real_child() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir_all(root.join("a")).unwrap(); + std::fs::write(root.join("a/sealed.json"), b"owned receipt").unwrap(); + let moved = root.join("moved"); + let removed = remove_receipted_directories_nofollow( + &root, + OsStr::new("sealed.json"), + 1024, + |_, bytes| { + assert_eq!(bytes, b"owned receipt"); + std::fs::rename(root.join("a"), &moved).unwrap(); + std::fs::create_dir(root.join("a")).unwrap(); + std::fs::write(root.join("a/sentinel"), b"foreign").unwrap(); + true + }, + ) + .unwrap(); + assert_eq!(removed, 1); + assert!(!moved.exists()); + assert_eq!(std::fs::read(root.join("a/sentinel")).unwrap(), b"foreign"); + } + + #[test] + fn a_receipt_filename_cannot_escape_its_held_child() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + std::fs::create_dir(&root).unwrap(); + for receipt in ["../outside", "a/b", "."] { + let error = remove_receipted_directories_nofollow( + &root, + OsStr::new(receipt), + 512, + |_, _| true, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + } + } + + #[cfg(unix)] + #[test] + fn exact_receipted_retirement_preserves_unknown_and_swapped_real_children() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let child = root.join("a"); + let moved = root.join("moved"); + std::fs::create_dir_all(&child).unwrap(); + std::fs::write(child.join("sealed.json"), b"owned receipt").unwrap(); + std::fs::write(child.join("owned"), b"owned").unwrap(); + assert_eq!( + remove_receipted_child_directory_nofollow( + &root, + OsStr::new("a"), + OsStr::new("sealed.json"), + 1024, + |_| false, + ) + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied, + ); + assert_eq!(std::fs::read(child.join("owned")).unwrap(), b"owned"); + let removed = remove_receipted_child_directory_nofollow( + &root, + OsStr::new("a"), + OsStr::new("sealed.json"), + 1024, + |receipt| { + assert_eq!(receipt, b"owned receipt"); + std::fs::rename(&child, &moved).unwrap(); + std::fs::create_dir(&child).unwrap(); + std::fs::write(child.join("foreign"), b"preserve").unwrap(); + true + }, + ) + .unwrap(); + assert!(removed); + assert!( + !moved.exists(), + "only the receipt-authorized directory is deleted" + ); + assert_eq!( + std::fs::read(child.join("foreign")).unwrap(), + b"preserve" + ); + } + + #[cfg(unix)] + #[test] + fn exact_cleanup_refuses_a_substituted_root_symlink() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("original"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join("a")).unwrap(); + std::fs::create_dir_all(foreign.join("a")).unwrap(); + std::fs::write(root.join("a/sealed.json"), b"owned").unwrap(); + std::fs::write(foreign.join("a/sealed.json"), b"foreign").unwrap(); + std::fs::write(foreign.join("a/sentinel"), b"preserve").unwrap(); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + assert!(remove_child_directory_nofollow(&root, OsStr::new("a")).is_err()); + assert!( + remove_receipted_child_directory_nofollow( + &root, + OsStr::new("a"), + OsStr::new("sealed.json"), + 1024, + |_| true, + ) + .is_err() + ); + assert_eq!( + std::fs::read(foreign.join("a/sentinel")).unwrap(), + b"preserve" + ); + assert_eq!( + std::fs::read(displaced.join("a/sealed.json")).unwrap(), + b"owned" + ); + assert_eq!( + remove_child_directory_nofollow(&displaced, OsStr::new("../a")) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput, + ); + } + #[test] fn a_transient_missing_answer_is_looked_at_again() { let mut attempts = 0; diff --git a/crates/tracedecay-private-fs/src/framed_log.rs b/crates/tracedecay-private-fs/src/framed_log.rs index 005e20779c2..408bc6780e9 100644 --- a/crates/tracedecay-private-fs/src/framed_log.rs +++ b/crates/tracedecay-private-fs/src/framed_log.rs @@ -278,6 +278,13 @@ pub fn read_bounded(path: &Path, maximum: usize) -> io::Result>> Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), Err(error) => return Err(error), }; + read_bounded_open_file(file, maximum).map(Some) +} + +/// Applies the same bounded regular-file proof to a file opened relative to a +/// held directory capability. The caller supplies a no-follow, nonblocking +/// handle; this function never resolves another pathname. +pub(crate) fn read_bounded_open_file(file: File, maximum: usize) -> io::Result> { let metadata = file.metadata()?; if !metadata.file_type().is_file() { return Err(io::Error::new( @@ -300,7 +307,7 @@ pub fn read_bounded(path: &Path, maximum: usize) -> io::Result>> "bounded read length mismatch", )); } - Ok(Some(bytes)) + Ok(bytes) } const TEMPORARY_SUFFIX: &str = ".tmp"; From e304bfe22296d1237fd14bdd0ba878c2412361db Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:12:06 -0800 Subject: [PATCH 02/70] style(graph): apply Bazel rustfmt to sealed retirement Refs #4616 Reconcile all eleven formatting deltas from the exact-head Bazel repository-gates failure without modifying retirement behavior or tests. --- .../tracedecay-graph-db/src/sealed_store.rs | 25 ++++++++------ .../src/capability_dir.rs | 34 +++++++------------ 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 988c8ccf94e..f1f014c6eb1 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -763,10 +763,12 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) { let path = entry.path(); - if let Err(error) = tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( - &root, - &entry.file_name(), - ) { + if let Err(error) = + tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + &root, + &entry.file_name(), + ) + { tracing::warn!( event = "graph_row_spill_sweep_failed", path = %path.display(), @@ -807,9 +809,9 @@ fn retire_sealed_directory_checked( "sealed retirement directory does not match generation identity", )); } - let name = directory - .file_name() - .ok_or_else(|| GraphDbError::unavailable("sealed retirement directory has no entry name"))?; + let name = directory.file_name().ok_or_else(|| { + GraphDbError::unavailable("sealed retirement directory has no entry name") + })?; let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( root, name, @@ -3041,7 +3043,11 @@ mod receipt_budget_tests { let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); std::fs::write( &receipt_path, - receipt(GraphFormatVersion::current().get(), name, "generation:other"), + receipt( + GraphFormatVersion::current().get(), + name, + "generation:other", + ), ) .unwrap(); std::fs::write(directory.join("sentinel"), b"preserve").unwrap(); @@ -3400,8 +3406,7 @@ mod build_tests { let identity = manifest.identity(); let expected = manifest.expected_recovered_digest(check).unwrap(); let root = sealed_store_root(&database_path); - let directory = - sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); let staging = root.join(format!( ".staging-{}", directory.file_name().unwrap().to_str().unwrap() diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 15e7e6cf401..4b4013345fb 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -16,10 +16,10 @@ use std::os::unix::ffi::OsStrExt; use std::path::Path; use cap_fs_ext::DirExt; -#[cfg(windows)] -use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; #[cfg(not(windows))] use cap_fs_ext::OpenOptionsMaybeDirExt; +#[cfg(windows)] +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt}; use cap_std::fs::{Dir, File, OpenOptions}; /// How many times a create-or-open that reports the entry missing is asked @@ -218,9 +218,7 @@ pub fn sync_directory(directory: &Dir) -> io::Result<()> { fn single_component(name: &OsStr) -> io::Result<()> { let mut parts = Path::new(name).components(); - if !matches!(parts.next(), Some(std::path::Component::Normal(_))) - || parts.next().is_some() - { + if !matches!(parts.next(), Some(std::path::Component::Normal(_))) || parts.next().is_some() { return Err(io::Error::new( io::ErrorKind::InvalidInput, "directory operation requires one entry name", @@ -356,9 +354,8 @@ fn read_bounded_child_receipt( name: &OsStr, maximum: usize, ) -> io::Result> { - let name = CString::new(name.as_bytes()).map_err(|_| { - io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL") - })?; + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "receipt name contains NUL"))?; // SAFETY: the held directory fd is valid; name is a NUL-terminated // sibling component. O_NOFOLLOW prevents link traversal and O_NONBLOCK // prevents a substituted FIFO from blocking before the regular-file check. @@ -550,7 +547,6 @@ mod tests { ); } - /// A parent-path swap after receipt validation must not redirect cleanup: /// the already-open child is removed, and matching foreign bytes survive. #[cfg(unix)] @@ -581,7 +577,10 @@ mod tests { .unwrap(); assert_eq!(removed, 1); assert!(!displaced.join("a").exists()); - assert_eq!(std::fs::read(foreign.join("a/sentinel")).unwrap(), b"foreign"); + assert_eq!( + std::fs::read(foreign.join("a/sentinel")).unwrap(), + b"foreign" + ); } /// The child also remains the exact opened directory when its original @@ -618,13 +617,9 @@ mod tests { let root = temp.path().join("sealed"); std::fs::create_dir(&root).unwrap(); for receipt in ["../outside", "a/b", "."] { - let error = remove_receipted_directories_nofollow( - &root, - OsStr::new(receipt), - 512, - |_, _| true, - ) - .unwrap_err(); + let error = + remove_receipted_directories_nofollow(&root, OsStr::new(receipt), 512, |_, _| true) + .unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::InvalidInput); } } @@ -671,10 +666,7 @@ mod tests { !moved.exists(), "only the receipt-authorized directory is deleted" ); - assert_eq!( - std::fs::read(child.join("foreign")).unwrap(), - b"preserve" - ); + assert_eq!(std::fs::read(child.join("foreign")).unwrap(), b"preserve"); } #[cfg(unix)] From 3c66144e84acc390542d3c7dd9f1e0eabfb3a86c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:13:44 -0800 Subject: [PATCH 03/70] fix(graph): preserve unrelated directories during store recovery --- .../tracedecay-graph-db/src/sealed_store.rs | 27 ++++++++++++++----- .../sealed_store.rs | 14 +++++++++- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index f1f014c6eb1..d20e41665de 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -579,10 +579,7 @@ pub fn census_sealed_store( let path = entry.path(); let bytes = directory_bytes(&path, &mut counted)?; let name = entry.file_name(); - if name - .to_str() - .is_some_and(|name| name.starts_with(".staging-")) - { + if name.to_str().is_some_and(sealed_staging_name) { census.abandoned_staging_count += 1; census.abandoned_staging_bytes += bytes; continue; @@ -683,6 +680,15 @@ pub(crate) fn discard_superseded_sealed_generations( } } +fn sealed_staging_name(name: &str) -> bool { + name.strip_prefix(".staging-").is_some_and(|suffix| { + suffix.len() == 64 + && suffix + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) +} + /// Removes every `.staging-*` directory a seal left behind under the store's /// sealed root. /// @@ -714,7 +720,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Some(name) = name.to_str() else { continue; }; - if !name.starts_with(".staging-") { + if !sealed_staging_name(name) { continue; } let path = entry.path(); @@ -755,12 +761,19 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let Ok(entries) = std::fs::read_dir(&root) else { return; }; - let own = format!("{ROW_SPILL_PREFIX}{}-", std::process::id()); + let own_pid = std::process::id(); for entry in entries.flatten() { if entry .file_name() .to_str() - .is_some_and(|name| name.starts_with(ROW_SPILL_PREFIX) && !name.starts_with(&own)) + .and_then(|name| name.strip_prefix(ROW_SPILL_PREFIX)) + .and_then(|suffix| suffix.split_once('-')) + .is_some_and(|(pid, sequence)| { + let (Ok(owner), Ok(counter)) = (pid.parse::(), sequence.parse::()) else { + return false; + }; + owner != own_pid && owner.to_string() == pid && counter.to_string() == sequence + }) { let path = entry.path(); if let Err(error) = diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 3bb165ee0f6..7d360753e71 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1263,6 +1263,9 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { .map(|entry| entry.path()) .find(|path| path.join("sealed.json").is_file()) .expect("the sealed artifact directory must exist"); + let unrelated = root.join(".staging-notes"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("notes"), b"keep").unwrap(); let abandoned = [ root.join(format!(".staging-{}", "c".repeat(64))), root.join(format!( @@ -1287,6 +1290,7 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { &record.publication.key.projection, ) .unwrap(); + assert_eq!(std::fs::read(unrelated.join("notes")).unwrap(), b"keep"); for staging in &abandoned { assert!( !staging.exists(), @@ -2371,7 +2375,15 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { let own = format!(".rows-{}-999999", std::process::id()); std::fs::create_dir_all(root.join(&own)).unwrap(); + let foreign = ".rows-notes"; + std::fs::create_dir(root.join(foreign)).unwrap(); + std::fs::write(root.join(foreign).join("notes"), b"keep").unwrap(); let _registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); - assert_eq!(row_spills(temp.path()), vec![own]); + assert_eq!( + std::fs::read(root.join(foreign).join("notes")).unwrap(), + b"keep" + ); + assert!(root.join(own).is_dir()); + assert!(!root.join(".rows-4194305-0").exists()); } From 65bd98f5b4a49250ecf9a2d3094f06d5fd3ca0b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:16:38 -0800 Subject: [PATCH 04/70] fix(graph): refuse unverified sealed paths during rebuild Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 130 +++++++++++++++++- 1 file changed, 125 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index d20e41665de..a4608aa6258 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1862,6 +1862,7 @@ fn build_or_open_sealed_store( // Idempotent replay: an artifact from an earlier seal of this exact // generation is adopted if its receipt binds the same digest. Adoption // never enumerates `source`'s rows, so it yields no staging proof. + let locator = GenerationLocator::new(identity.projection.clone(), identity.generation.clone()); match open_sealed_store_checked( &directory, identity, @@ -1870,11 +1871,26 @@ fn build_or_open_sealed_store( trusted_base.as_ref(), ) { Ok(Some(store)) => return Ok((store, None)), - Ok(None) => {} - Err(_) => remove_sealed_directory(&directory), - } - if directory.exists() { - remove_sealed_directory(&directory); + Ok(None) => match std::fs::symlink_metadata(&directory) { + Ok(_) => { + return Err(GraphDbError::unavailable( + "unverified sealed generation occupies the artifact directory", + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(sealed_store_io_failure( + "sealed artifact metadata read failed", + error, + )); + } + }, + Err(GraphDbError::Cancelled) => return Err(GraphDbError::Cancelled), + Err(_) => { + // A corrupt owned artifact may be rebuilt, but an unknown receipt + // (or a replacement directory) never grants deletion authority. + retire_sealed_directory_checked(&directory, &locator, Some(expected.as_str()))?; + } } let staging = root.join(format!( ".staging-{}", @@ -1887,6 +1903,15 @@ fn build_or_open_sealed_store( // accept a pre-existing staging symlink and write into its foreign target. std::fs::create_dir_all(&root) .map_err(|error| sealed_store_io_failure("sealed root create failed", error))?; + // create_dir_all accepts symlinked directory components. Refuse a + // preexisting substituted sealed root before writing staging bytes. + let root_metadata = std::fs::symlink_metadata(&root) + .map_err(|error| sealed_store_io_failure("sealed root metadata failed", error))?; + if !root_metadata.is_dir() { + return Err(GraphDbError::unavailable( + "sealed root must be a real directory", + )); + } let staging_name = staging .file_name() .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; @@ -3406,6 +3431,101 @@ mod build_tests { ); } + /// An unverified existing generation directory cannot be treated as + /// disposable scratch space. Rebuilding is allowed only after its owner + /// removes the unknown data. + #[test] + fn sealed_build_preserves_an_unverified_occupied_generation() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + std::fs::create_dir_all(&directory).unwrap(); + let sentinel = directory.join("foreign-sentinel"); + std::fs::write(&sentinel, b"not owned").unwrap(); + + for foreign_receipt in [false, true] { + if foreign_receipt { + std::fs::write(directory.join(SEALED_STORE_RECEIPT_FILE), b"invalid receipt") + .unwrap(); + } + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!( + result.is_err(), + "unreceipted or malformed foreign data must refuse rebuilding" + ); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"not owned"); + assert!( + !directory.join(SEALED_STORE_DATABASE_FILE).exists(), + "refused rebuilding must never write a container into foreign data" + ); + } + + std::fs::remove_dir_all(&directory).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("clearing the unknown artifact allows a verified sealed build"); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + } + + #[cfg(unix)] + #[test] + fn sealed_build_refuses_a_foreign_root_symlink() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"not owned").unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!(result.is_err(), "a symlinked sealed root must refuse writes"); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"not owned"); + assert_eq!( + std::fs::read_dir(&foreign).unwrap().count(), + 1, + "no staged container or receipt may be written into the foreign root" + ); + + std::fs::remove_file(&root).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("a genuine sealed root still supports building"); + assert_eq!(store.recovered_digest(), expected.as_str()); + } + /// A leftover staging symlink must not become a writable artifact root. /// Refusing it preserves the foreign target rather than following it after /// a best-effort cleanup that did not actually unlink the entry. From 3d9892140f4d33b40b6bc087a41bdca7e1e8d1a9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:43:07 -0800 Subject: [PATCH 05/70] style(graph): format sealed generation retirement tests --- crates/tracedecay-graph-db/src/sealed_store.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a4608aa6258..7f063d87f50 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3450,8 +3450,11 @@ mod build_tests { for foreign_receipt in [false, true] { if foreign_receipt { - std::fs::write(directory.join(SEALED_STORE_RECEIPT_FILE), b"invalid receipt") - .unwrap(); + std::fs::write( + directory.join(SEALED_STORE_RECEIPT_FILE), + b"invalid receipt", + ) + .unwrap(); } let result = build_or_open_sealed_store( SealedRowSource::Manifest(&manifest), @@ -3506,8 +3509,14 @@ mod build_tests { &database_path, check, ); - assert!(result.is_err(), "a symlinked sealed root must refuse writes"); - assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"not owned"); + assert!( + result.is_err(), + "a symlinked sealed root must refuse writes" + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"not owned" + ); assert_eq!( std::fs::read_dir(&foreign).unwrap().count(), 1, From c5b89807d057724812f700c8078ec082c1f4cfdd Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 06/70] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From c22406e1840698932f7e5f9f1fc7b6117865f88d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:02:17 -0800 Subject: [PATCH 07/70] fix(graph): verify failed-adoption cleanup before unlink Refs #4616 Use the strict generation/digest-bound retirement authority on failed recovery adoption and post-install reopen failures. Preserve unverified artifacts instead of path-deleting them. Cover missing, corrupt, wrong- digest and owned receipts plus unrelated sibling preservation. --- .../tracedecay-graph-db/src/sealed_store.rs | 84 +++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7f063d87f50..2422c4f5d6d 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -858,6 +858,24 @@ fn retire_sealed_directory_checked( /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted /// replay instead uses the checked path, so a failed unlink cannot be reported /// as a completed retirement and lose its durable retry ownership. +/// Best-effort disposal after failed adoption/reopen. Only an exact durable +/// generation receipt authorizes removal; foreign bytes remain for diagnosis. +fn discard_failed_sealed_adoption( + directory: &Path, + locator: &GenerationLocator, + expected_digest: &str, +) { + if let Err(error) = + retire_sealed_directory_checked(directory, locator, Some(expected_digest)) + { + tracing::warn!( + %error, + path = %directory.display(), + "failed sealed adoption left an unverified artifact for inspection" + ); + } +} + fn remove_sealed_directory(directory: &Path) { if let Err(error) = remove_sealed_directory_checked(directory) { tracing::warn!( @@ -1644,10 +1662,16 @@ impl GraphDb { Ok(Some(store)) => self.install_sealed_generation_store(locator, store), Ok(None) => Ok(()), Err(error @ (GraphDbError::Cancelled | GraphDbError::DeadlineExceeded)) => Err(error), - Err(_) => { - // A stale or corrupt artifact never outranks the verified - // staging rows; discard it so a later seal can rebuild. - remove_sealed_directory(&directory); + Err(error) => { + // A failed adoption does not prove this directory belongs to + // the generation. Preserve unknown bytes rather than removing + // an occupied path without a matching durable receipt. + discard_failed_sealed_adoption(&directory, &locator, expected.as_str()); + tracing::warn!( + %error, + path = %directory.display(), + "sealed adoption failed; serving falls back to staging rows" + ); Ok(()) } } @@ -1954,13 +1978,13 @@ fn build_or_open_sealed_store( Ok((store, staging_proof)) } Ok(None) => { - remove_sealed_directory(&directory); + discard_failed_sealed_adoption(&directory, &locator, expected.as_str()); Err(GraphDbError::unavailable( "sealed generation store disappeared between install and reopen".to_owned(), )) } Err(error) => { - remove_sealed_directory(&directory); + discard_failed_sealed_adoption(&directory, &locator, expected.as_str()); Err(error) } } @@ -3062,6 +3086,54 @@ mod receipt_budget_tests { } } + #[test] + fn failed_adoption_cannot_discard_unverified_or_foreign_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new("generation:retirement").unwrap(), + ); + let directory = + sealed_generation_directory(&root, &locator.physical_namespace().unwrap()); + std::fs::create_dir_all(&directory).unwrap(); + let name = directory.file_name().unwrap().to_str().unwrap(); + let sentinel = directory.join("foreign-sentinel"); + std::fs::write(&sentinel, b"preserve").unwrap(); + let sibling = root.join("unrelated"); + std::fs::create_dir(&sibling).unwrap(); + std::fs::write(sibling.join("sentinel"), b"other").unwrap(); + + // Neither a missing nor a malformed receipt proves ownership. + discard_failed_sealed_adoption(&directory, &locator, "digest"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); + let receipt_path = directory.join(SEALED_STORE_RECEIPT_FILE); + std::fs::write(&receipt_path, b"invalid receipt").unwrap(); + discard_failed_sealed_adoption(&directory, &locator, "digest"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); + + // An otherwise recognizable receipt still cannot retire a different + // expected digest. The exact matching receipt allows cleanup. + std::fs::write( + &receipt_path, + receipt( + GraphFormatVersion::current().get(), + name, + "generation:retirement", + ), + ) + .unwrap(); + discard_failed_sealed_adoption(&directory, &locator, "other-digest"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); + discard_failed_sealed_adoption(&directory, &locator, "digest"); + assert!(!directory.exists(), "owned invalid artifacts can be reclaimed"); + assert_eq!(std::fs::read(sibling.join("sentinel")).unwrap(), b"other"); + } + #[test] fn retired_sealed_generation_requires_its_exact_receipt_identity() { let temp = tempfile::tempdir().unwrap(); From 7c9a6f479c7804d70520d2b7313921a28a494eb5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:22:45 -0800 Subject: [PATCH 08/70] style(graph): format failed adoption cleanup --- crates/tracedecay-graph-db/src/sealed_store.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 2422c4f5d6d..46137d5a209 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -865,9 +865,7 @@ fn discard_failed_sealed_adoption( locator: &GenerationLocator, expected_digest: &str, ) { - if let Err(error) = - retire_sealed_directory_checked(directory, locator, Some(expected_digest)) - { + if let Err(error) = retire_sealed_directory_checked(directory, locator, Some(expected_digest)) { tracing::warn!( %error, path = %directory.display(), @@ -3098,8 +3096,7 @@ mod receipt_budget_tests { ), crate::GraphGenerationId::new("generation:retirement").unwrap(), ); - let directory = - sealed_generation_directory(&root, &locator.physical_namespace().unwrap()); + let directory = sealed_generation_directory(&root, &locator.physical_namespace().unwrap()); std::fs::create_dir_all(&directory).unwrap(); let name = directory.file_name().unwrap().to_str().unwrap(); let sentinel = directory.join("foreign-sentinel"); @@ -3130,7 +3127,10 @@ mod receipt_budget_tests { discard_failed_sealed_adoption(&directory, &locator, "other-digest"); assert_eq!(std::fs::read(&sentinel).unwrap(), b"preserve"); discard_failed_sealed_adoption(&directory, &locator, "digest"); - assert!(!directory.exists(), "owned invalid artifacts can be reclaimed"); + assert!( + !directory.exists(), + "owned invalid artifacts can be reclaimed" + ); assert_eq!(std::fs::read(sibling.join("sentinel")).unwrap(), b"other"); } From 882d64297c3a98c0a0d032957aed817bab84ed4f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 15:39:50 -0800 Subject: [PATCH 09/70] fix(graph): refuse occupied sealed generation installs Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 7 +- .../src/capability_dir.rs | 70 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 46137d5a209..207d54fe378 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -54,6 +54,7 @@ use grafeo_core::graph::compact::IncrementalCompactStoreBuilder; use grafeo_engine::GrafeoDB; use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nofollow; use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ @@ -1948,7 +1949,11 @@ fn build_or_open_sealed_store( std::fs::write(staging.join(SEALED_STORE_RECEIPT_FILE), encoded) .inspect_err(|_| remove_sealed_directory(&staging)) .map_err(|error| sealed_store_io_failure("receipt write failed", error))?; - if let Err(error) = std::fs::rename(&staging, &directory) { + let artifact_name = directory + .file_name() + .ok_or_else(|| GraphDbError::unavailable("sealed artifact name missing"))?; + let install = install_child_directory_noreplace_nofollow(&root, staging_name, artifact_name); + if let Err(error) = install { remove_sealed_directory(&staging); // A concurrent seal of the same generation may have installed the // directory first; adopting it below keeps this path idempotent. diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 4b4013345fb..56983fb4eaf 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -238,6 +238,28 @@ fn open_root_nofollow(root_path: &Path) -> io::Result { parent.open_dir_nofollow(root_name) } +/// Install a staged generation without replacing even an empty destination +/// directory. The final component of the sealed root and the staging entry +/// must be real directories; a link or an occupied generation is not a +/// recoverable staging collision. +/// +/// The caller still owns the responsibility for the identity of the store +/// ancestors, which this pathname API cannot establish. +pub fn install_child_directory_noreplace_nofollow( + root_path: &Path, + staging_name: &OsStr, + generation_name: &OsStr, +) -> io::Result<()> { + single_component(staging_name)?; + single_component(generation_name)?; + let root = open_root_nofollow(root_path)?; + let staging = root.open_dir_nofollow(staging_name)?; + // Reject a linked staging entry, then close the extra directory handle + // before publication for platforms that disallow renaming open children. + drop(staging); + rename_noreplace(&root, staging_name, &root, generation_name) +} + /// Remove exactly one held child directory without following a substituted /// sealed-root or child symlink. Missing entries are idempotent; other failures /// remain errors so callers can retain retry ownership. @@ -490,6 +512,54 @@ mod tests { Dir::open_ambient_dir(path, ambient_authority()).expect("open test directory") } + #[test] + fn staged_generation_install_preserves_an_occupied_empty_directory() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + std::fs::create_dir(&root).unwrap(); + let staged_name = OsStr::new(".staging-generation"); + let installed_name = OsStr::new("generation"); + let staged = root.join(staged_name); + let installed = root.join(installed_name); + std::fs::create_dir(&staged).unwrap(); + std::fs::write(staged.join("sealed.json"), b"built").unwrap(); + std::fs::create_dir(&installed).unwrap(); + + let result = + install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::AlreadyExists); + assert!(installed.is_dir(), "an empty foreign target is not expendable"); + assert_eq!(std::fs::read(staged.join("sealed.json")).unwrap(), b"built"); + + std::fs::remove_dir(&installed).unwrap(); + install_child_directory_noreplace_nofollow(&root, staged_name, installed_name).unwrap(); + assert!(!staged.exists()); + assert_eq!(std::fs::read(installed.join("sealed.json")).unwrap(), b"built"); + } + + #[cfg(unix)] + #[test] + fn staged_generation_install_refuses_linked_root_and_staging_entries() { + let temp = tempfile::tempdir().unwrap(); + let foreign = temp.path().join("foreign"); + let real = temp.path().join("real"); + let link = temp.path().join("link"); + std::fs::create_dir(&foreign).unwrap(); + std::fs::create_dir(&real).unwrap(); + std::fs::write(foreign.join("sentinel"), b"preserve").unwrap(); + std::fs::create_dir(foreign.join("staged")).unwrap(); + std::os::unix::fs::symlink(&foreign, &link).unwrap(); + let stage = OsStr::new("staged"); + let published = OsStr::new("installed"); + assert!(install_child_directory_noreplace_nofollow(&link, stage, published).is_err()); + + std::os::unix::fs::symlink(&foreign, real.join("staged")).unwrap(); + assert!(install_child_directory_noreplace_nofollow(&real, stage, published).is_err()); + assert!(!foreign.join("installed").exists()); + assert!(!real.join("installed").exists()); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"preserve"); + } + #[test] fn rename_noreplace_moves_between_parents_and_refuses_occupied_destinations() { let root = tempfile::tempdir().expect("create rename fixture"); From dacd4a0e1e86f79300ac3c92c79e773ea66f8dbe Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:05:18 -0800 Subject: [PATCH 10/70] style(storage): format no-replace directory installation --- .../src/capability_dir.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 56983fb4eaf..d04bea7d3fe 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -525,16 +525,21 @@ mod tests { std::fs::write(staged.join("sealed.json"), b"built").unwrap(); std::fs::create_dir(&installed).unwrap(); - let result = - install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); + let result = install_child_directory_noreplace_nofollow(&root, staged_name, installed_name); assert_eq!(result.unwrap_err().kind(), io::ErrorKind::AlreadyExists); - assert!(installed.is_dir(), "an empty foreign target is not expendable"); + assert!( + installed.is_dir(), + "an empty foreign target is not expendable" + ); assert_eq!(std::fs::read(staged.join("sealed.json")).unwrap(), b"built"); std::fs::remove_dir(&installed).unwrap(); install_child_directory_noreplace_nofollow(&root, staged_name, installed_name).unwrap(); assert!(!staged.exists()); - assert_eq!(std::fs::read(installed.join("sealed.json")).unwrap(), b"built"); + assert_eq!( + std::fs::read(installed.join("sealed.json")).unwrap(), + b"built" + ); } #[cfg(unix)] @@ -557,7 +562,10 @@ mod tests { assert!(install_child_directory_noreplace_nofollow(&real, stage, published).is_err()); assert!(!foreign.join("installed").exists()); assert!(!real.join("installed").exists()); - assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"preserve"); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"preserve" + ); } #[test] From e2f3371c632323e580be465ffa5fee27f4b4b7cf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:31:08 -0800 Subject: [PATCH 11/70] test(graph): preserve unverified sealed artifacts on recovery Refs #4616 --- .../verified_generation_contract/sealed_store.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 7d360753e71..c2f11b96dc8 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1187,8 +1187,8 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { assert!(registered.close().unwrap()); drop(registered); - // Tamper with the receipt: recovery must discard the artifact and serve - // from the staging database. + // Tamper with the receipt: recovery must refuse adoption and serve from + // staging, without deleting bytes that no longer prove our ownership. let root = sealed_store_root(temp.path()); let mut tampered = None; for entry in std::fs::read_dir(&root).unwrap().map(Result::unwrap) { @@ -1200,6 +1200,7 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { } } let tampered = tampered.expect("the sealed artifact directory must exist"); + let tampered_receipt = std::fs::read(tampered.join("sealed.json")).unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let (control, probe) = control_and_probe(); let context = GraphPublicationOperationContextV1::new(&control, &probe).unwrap(); @@ -1217,8 +1218,13 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { "a tampered artifact must not be adopted" ); assert!( - !tampered.exists(), - "a tampered artifact must be discarded from disk" + tampered.is_dir(), + "recovery must preserve an artifact with an unverified receipt" + ); + assert_eq!( + std::fs::read(tampered.join("sealed.json")).unwrap(), + tampered_receipt, + "recovery must not modify the unverified receipt" ); assert_snapshot_reads(&snapshot, &identity, "durable"); } From b568770d301cabd1337549834e57b92b6aa62a8d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:36:00 -0800 Subject: [PATCH 12/70] fix(graph): refuse occupied sealed staging on rebuild Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 63 ++++++++++++++++--- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 207d54fe378..6d3903d7165 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1922,8 +1922,8 @@ fn build_or_open_sealed_store( .and_then(|name| name.to_str()) .unwrap_or("sealed") )); - // A failed cleanup must never be followed by create_dir_all: it would - // accept a pre-existing staging symlink and write into its foreign target. + // create_dir_all may accept a symlinked sealed root, so verify the actual + // root before creating the staging entry or writing a container. std::fs::create_dir_all(&root) .map_err(|error| sealed_store_io_failure("sealed root create failed", error))?; // create_dir_all accepts symlinked directory components. Refuse a @@ -1938,8 +1938,9 @@ fn build_or_open_sealed_store( let staging_name = staging .file_name() .ok_or_else(|| GraphDbError::unavailable("sealed staging name missing"))?; - tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(&root, staging_name) - .map_err(|error| sealed_store_io_failure("staging directory cleanup refused", error))?; + // A deterministic staging name is not proof of ownership. An earlier + // interrupted build or a foreign directory must not be deleted on retry. + // create_dir is exclusive, so an occupied name fails closed. std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check) @@ -3612,9 +3613,8 @@ mod build_tests { assert_eq!(store.recovered_digest(), expected.as_str()); } - /// A leftover staging symlink must not become a writable artifact root. - /// Refusing it preserves the foreign target rather than following it after - /// a best-effort cleanup that did not actually unlink the entry. + /// A pre-existing staging symlink must not become a writable artifact + /// root or be unlinked as disposable scratch space. #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_staging_symlink() { @@ -3679,6 +3679,55 @@ mod build_tests { ); } + /// A pre-existing real staging directory is not disposable merely + /// because its name matches the generation being built. + #[test] + fn sealed_build_preserves_a_foreign_real_staging_directory() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + std::fs::create_dir_all(&staging).unwrap(); + let sentinel = staging.join("foreign-sentinel"); + std::fs::write(&sentinel, b"must survive").unwrap(); + + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!(result.is_err(), "an occupied staging entry must refuse sealing"); + assert_eq!(std::fs::read(&sentinel).unwrap(), b"must survive"); + assert!( + !staging.join(SEALED_STORE_DATABASE_FILE).exists() + && !staging.join(SEALED_STORE_RECEIPT_FILE).exists(), + "refused sealing must not modify foreign staging data" + ); + assert!(!directory.exists(), "no generation may be installed"); + + std::fs::remove_dir_all(&staging).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .expect("clearing the unknown stage allows a verified seal"); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves From 5de9d92cb05f93e4c3324e291ed5ac387bca0897 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:42:34 -0800 Subject: [PATCH 13/70] fix(graph): require receipts for abandoned staging cleanup Refs #4616 Do not treat a staging-shaped name as deletion authority. Retire only staged children with bounded matching sealed-generation receipts through held no-follow capabilities, preserving unknown or substituted contents. Exercise real store reopen for owned cleanup and unverified preservation. --- .../tracedecay-graph-db/src/sealed_store.rs | 102 +++++++++--------- .../sealed_store.rs | 92 ++++++++++++---- 2 files changed, 121 insertions(+), 73 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 6d3903d7165..08b192f268b 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -690,62 +690,64 @@ fn sealed_staging_name(name: &str) -> bool { }) } -/// Removes every `.staging-*` directory a seal left behind under the store's -/// sealed root. +/// Reclaims interrupted seals only when the bounded receipt from the same +/// opened staging directory proves its exact generation identity. The staging +/// filename alone is not deletion authority: a missing, malformed, oversized, +/// or foreign receipt leaves the directory intact for inspection. /// -/// A seal builds into `.staging-` and installs by rename, so a staging -/// directory that survives to the next open of the same store belongs to a -/// build the process never finished, a crash, a kill, or an OOM between -/// container write and rename. Nothing reads it and the next seal of that -/// generation starts over, so on the live profile these accumulated to 7.4 GB -/// under one store. Run at open: the exclusive store lock means no seal of -/// this store is in flight. +/// Store open owns exclusive maintenance admission. Capability-relative +/// traversal holds the root and child across verification and recursive removal, +/// so a pathname replacement cannot redirect deletion into a different tree. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, - Err(error) => { - tracing::warn!( - event = "sealed_staging_sweep_unreadable", - root = %root.display(), - error = %error, - "sealed store root could not be enumerated; abandoned staging stays" - ); - return; - } - }; - let mut removed = 0usize; - for entry in entries.flatten() { - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - if !sealed_staging_name(name) { - continue; - } - let path = entry.path(); - match tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( - &root, - &entry.file_name(), - ) { - Ok(true) => removed += 1, - Ok(false) => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - path = %path.display(), - error = %error, - "abandoned sealed staging directory could not be removed" - ), - } - } - if removed > 0 { - tracing::info!( + let reclaimed = tracedecay_private_fs::capability_dir::remove_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), + MAX_SEALED_RECEIPT_BYTES, + |name, bytes| { + let Some(name) = name.to_str() else { + return false; + }; + if !sealed_staging_name(name) { + return false; + } + let Some(generation_hex) = name.strip_prefix(".staging-") else { + return false; + }; + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return false; + }; + let recognized_form = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + recognized_form + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.graph_format > 0 + && receipt.graph_format <= GraphFormatVersion::current().get() + && receipt.physical_namespace == format!("generation:{generation_hex}") + && !receipt.namespace.is_empty() + && !receipt.projection.is_empty() + && !receipt.generation.is_empty() + && !receipt.recovered_digest.is_empty() + }, + ); + match reclaimed { + Ok(removed) if removed > 0 => tracing::info!( event = "sealed_staging_swept", root = %root.display(), removed, - "removed abandoned sealed staging directories left by interrupted seals" - ); + "removed receipt-verified staging from interrupted seals" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned staging stays for inspection" + ), } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index c2f11b96dc8..a965d9f9a15 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1229,12 +1229,11 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { assert_snapshot_reads(&snapshot, &identity, "durable"); } -/// A seal that died between container write and rename leaves -/// `.staging-` under the sealed root. Nothing reads it, so the next -/// open of the store removes every such directory while leaving installed -/// artifacts untouched. +/// A completed interrupted seal carries an exact generation receipt and is +/// reclaimed when reopening the store. Unreceipted, mismatched, oversized and +/// linked lookalikes must survive instead of being deleted by filename. #[test] -fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { +fn reopening_the_store_reclaims_only_receipted_staging() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let mut authority = RelationalAuthority::default(); @@ -1260,28 +1259,47 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { assert!(registered.close().unwrap()); drop(registered); - // Two interrupted seals: one of a generation that never installed and one - // whose digest matches the installed artifact but was started again. let root = sealed_store_root(temp.path()); let installed = std::fs::read_dir(&root) .unwrap() .map(Result::unwrap) .map(|entry| entry.path()) .find(|path| path.join("sealed.json").is_file()) - .expect("the sealed artifact directory must exist"); + .expect("the installed sealed artifact directory must exist"); + let receipt = std::fs::read(installed.join("sealed.json")).unwrap(); + let installed_name = installed.file_name().unwrap().to_str().unwrap(); + let verified = root.join(format!(".staging-{installed_name}")); + std::fs::create_dir(&verified).unwrap(); + std::fs::copy( + installed.join("generation.grafeo"), + verified.join("generation.grafeo"), + ) + .unwrap(); + std::fs::write(verified.join("sealed.json"), &receipt).unwrap(); + + let unowned = root.join(format!(".staging-{}", "c".repeat(64))); + std::fs::create_dir(&unowned).unwrap(); + std::fs::write(unowned.join("generation.grafeo"), b"foreign bytes").unwrap(); + let mismatched = root.join(format!(".staging-{}", "d".repeat(64))); + std::fs::create_dir(&mismatched).unwrap(); + std::fs::write(mismatched.join("sealed.json"), &receipt).unwrap(); + std::fs::write(mismatched.join("sentinel"), b"wrong identity").unwrap(); + let oversized = root.join(format!(".staging-{}", "e".repeat(64))); + std::fs::create_dir(&oversized).unwrap(); + std::fs::write(oversized.join("sealed.json"), vec![b'x'; 128 * 1024]).unwrap(); + std::fs::write(oversized.join("sentinel"), b"too large").unwrap(); let unrelated = root.join(".staging-notes"); std::fs::create_dir(&unrelated).unwrap(); std::fs::write(unrelated.join("notes"), b"keep").unwrap(); - let abandoned = [ - root.join(format!(".staging-{}", "c".repeat(64))), - root.join(format!( - ".staging-{}", - installed.file_name().unwrap().to_str().unwrap() - )), - ]; - for staging in &abandoned { - std::fs::create_dir_all(staging).unwrap(); - std::fs::write(staging.join("generation.grafeo"), b"partial container").unwrap(); + + #[cfg(unix)] + let outside = TempDir::new().unwrap(); + #[cfg(unix)] + let linked = root.join(format!(".staging-{}", "f".repeat(64))); + #[cfg(unix)] + { + std::fs::write(outside.path().join("sentinel"), b"outside").unwrap(); + std::os::unix::fs::symlink(outside.path(), &linked).unwrap(); } let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); @@ -1296,17 +1314,45 @@ fn reopening_the_store_sweeps_staging_left_by_an_interrupted_seal() { &record.publication.key.projection, ) .unwrap(); + + assert!(!verified.exists(), "a verified abandoned seal must be reclaimed"); + assert_eq!( + std::fs::read(unowned.join("generation.grafeo")).unwrap(), + b"foreign bytes", + "missing receipt is not deletion authority" + ); + assert_eq!( + std::fs::read(mismatched.join("sealed.json")).unwrap(), + receipt, + "a receipt for a different generation cannot authorize cleanup" + ); + assert_eq!( + std::fs::read(mismatched.join("sentinel")).unwrap(), + b"wrong identity" + ); + assert_eq!( + std::fs::read(oversized.join("sentinel")).unwrap(), + b"too large", + "oversized receipts must be refused without recursive removal" + ); assert_eq!(std::fs::read(unrelated.join("notes")).unwrap(), b"keep"); - for staging in &abandoned { + #[cfg(unix)] + { assert!( - !staging.exists(), - "opening the store removes abandoned staging {}", - staging.display() + std::fs::symlink_metadata(&linked) + .unwrap() + .file_type() + .is_symlink(), + "a linked staging entry must not be followed or removed" + ); + assert_eq!( + std::fs::read(outside.path().join("sentinel")).unwrap(), + b"outside" ); } assert!( installed.join("sealed.json").is_file(), - "the installed sealed artifact is untouched by the sweep" + "the installed generation must remain untouched" ); assert!(snapshot.serves_from_sealed_store()); assert_snapshot_reads(&snapshot, &identity, "durable"); From 0bf8a86853f66920ef8544b5cee26b773ac53a92 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:45:12 -0800 Subject: [PATCH 14/70] fix(graph): bind abandoned receipts to canonical generation identity Refs #4616 A copied receipt with rewritten physical namespace must not authorize recursive cleanup of an unrelated staging directory. Exercise the forgery and malformed-receipt refusals through real store reopen. --- .../tracedecay-graph-db/src/sealed_store.rs | 21 ++++++++++++-- .../sealed_store.rs | 28 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 08b192f268b..0f10cedbf37 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -717,6 +717,23 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Ok(receipt) = serde_json::from_slice::(bytes) else { return false; }; + // Bind the receipt's logical identity back to the canonical + // physical namespace. A copied receipt with a rewritten physical + // name is not proof of ownership of another staged generation. + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace.clone()), + GraphProjectionId::new(receipt.projection.clone()), + crate::GraphGenerationId::new(receipt.generation.clone()), + ) else { + return false; + }; + let locator = GenerationLocator::new( + GraphProjectionIdentity::new(namespace, projection), + generation, + ); + let Ok(physical_namespace) = locator.physical_namespace() else { + return false; + }; let recognized_form = match receipt.form.as_str() { SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), @@ -727,9 +744,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { && receipt.graph_format > 0 && receipt.graph_format <= GraphFormatVersion::current().get() && receipt.physical_namespace == format!("generation:{generation_hex}") - && !receipt.namespace.is_empty() - && !receipt.projection.is_empty() - && !receipt.generation.is_empty() + && receipt.physical_namespace == physical_namespace.as_str() && !receipt.recovered_digest.is_empty() }, ); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index a965d9f9a15..f983483cd4b 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1284,6 +1284,21 @@ fn reopening_the_store_reclaims_only_receipted_staging() { std::fs::create_dir(&mismatched).unwrap(); std::fs::write(mismatched.join("sealed.json"), &receipt).unwrap(); std::fs::write(mismatched.join("sentinel"), b"wrong identity").unwrap(); + // A copied receipt whose physical namespace was rewritten to fit the + // staging filename still does not bind the original logical generation. + let forged_name = "a".repeat(64); + let forged = root.join(format!(".staging-{forged_name}")); + std::fs::create_dir(&forged).unwrap(); + let mut forged_receipt: serde_json::Value = serde_json::from_slice(&receipt).unwrap(); + forged_receipt["physical_namespace"] = + serde_json::Value::String(format!("generation:{forged_name}")); + let forged_bytes = serde_json::to_vec(&forged_receipt).unwrap(); + std::fs::write(forged.join("sealed.json"), &forged_bytes).unwrap(); + std::fs::write(forged.join("sentinel"), b"forged receipt").unwrap(); + let malformed = root.join(format!(".staging-{}", "b".repeat(64))); + std::fs::create_dir(&malformed).unwrap(); + std::fs::write(malformed.join("sealed.json"), b"not-json").unwrap(); + std::fs::write(malformed.join("sentinel"), b"malformed receipt").unwrap(); let oversized = root.join(format!(".staging-{}", "e".repeat(64))); std::fs::create_dir(&oversized).unwrap(); std::fs::write(oversized.join("sealed.json"), vec![b'x'; 128 * 1024]).unwrap(); @@ -1330,6 +1345,19 @@ fn reopening_the_store_reclaims_only_receipted_staging() { std::fs::read(mismatched.join("sentinel")).unwrap(), b"wrong identity" ); + assert_eq!( + std::fs::read(forged.join("sealed.json")).unwrap(), + forged_bytes, + "a rewritten physical namespace is not generation ownership" + ); + assert_eq!( + std::fs::read(forged.join("sentinel")).unwrap(), + b"forged receipt" + ); + assert_eq!( + std::fs::read(malformed.join("sentinel")).unwrap(), + b"malformed receipt" + ); assert_eq!( std::fs::read(oversized.join("sentinel")).unwrap(), b"too large", From bd4f39eb6b85516919d9cbb3686f1a4dd7e4cea8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:58:20 -0800 Subject: [PATCH 15/70] test(graph): preserve unreceipted staging in crash regression Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 0f10cedbf37..687b3ce09f3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3977,11 +3977,32 @@ mod build_tests { "an interrupted direct build must leave neither its staging directory nor an artifact" ); - // A crash image: the process died after creating the staging - // directory and part of a container. The next attempt owns that - // directory and rebuilds from the manifest. + // A crash before the receipt exists leaves no deletion authority. + // A retry must preserve the unverified staging bytes, not silently + // treat a name collision as proof that the directory belongs to us. std::fs::create_dir_all(&staging).unwrap(); - std::fs::write(staging.join(SEALED_STORE_DATABASE_FILE), b"torn container").unwrap(); + let torn_container = staging.join(SEALED_STORE_DATABASE_FILE); + std::fs::write(&torn_container, b"torn container").unwrap(); + let refused = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ); + assert!( + matches!(&refused, Err(GraphDbError::Unavailable { .. })), + "an unreceipted staging collision must fail closed: {refused:?}" + ); + assert_eq!(std::fs::read(&torn_container).unwrap(), b"torn container"); + assert!( + !directory.exists(), + "refused build must not publish a generation" + ); + + // The test fixture explicitly removes its unowned crash image. + // Only then may the same build succeed and prove its normal payload. + std::fs::remove_dir_all(&staging).unwrap(); let (direct, staging_proof) = build_or_open_sealed_store( SealedRowSource::Manifest(&manifest), &identity, @@ -3998,7 +4019,7 @@ mod build_tests { assert_eq!((direct.entity_count, direct.relation_count), (2_000, 3_000)); assert!( !staging.exists(), - "the crash image must be replaced, not kept" + "successful build must consume its staging directory" ); let _ = direct.database().close(); let direct_bytes = std::fs::read(&artifact).unwrap(); From 35abfc8c0b3a6e19d6b5a57ddb207ec400bc4b3c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:04:01 -0800 Subject: [PATCH 16/70] test(graph-db): import the canonical sealed receipt filename --- crates/tracedecay-graph-db/src/sealed_store.rs | 11 +++++++---- .../verified_generation_contract/sealed_store.rs | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 687b3ce09f3..8396576fabe 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3270,9 +3270,9 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - SEALED_STORE_DATABASE_FILE, SealedRowSource, build_or_open_sealed_store, - build_sealed_container, sealed_artifact_database_options, sealed_copy_proof, - sealed_generation_directory, sealed_store_root, + SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, + build_or_open_sealed_store, build_sealed_container, sealed_artifact_database_options, + sealed_copy_proof, sealed_generation_directory, sealed_store_root, }; use crate::location::PersistentGraphStoreState; use crate::{ @@ -3723,7 +3723,10 @@ mod build_tests { &database_path, check, ); - assert!(result.is_err(), "an occupied staging entry must refuse sealing"); + assert!( + result.is_err(), + "an occupied staging entry must refuse sealing" + ); assert_eq!(std::fs::read(&sentinel).unwrap(), b"must survive"); assert!( !staging.join(SEALED_STORE_DATABASE_FILE).exists() diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index f983483cd4b..05bd84f29a2 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1330,7 +1330,10 @@ fn reopening_the_store_reclaims_only_receipted_staging() { ) .unwrap(); - assert!(!verified.exists(), "a verified abandoned seal must be reclaimed"); + assert!( + !verified.exists(), + "a verified abandoned seal must be reclaimed" + ); assert_eq!( std::fs::read(unowned.join("generation.grafeo")).unwrap(), b"foreign bytes", From f3d8f4eecdabc42115ce89e1cba4e8087d2a8598 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:30:10 -0800 Subject: [PATCH 17/70] fix(graph): verify row spill ownership before reclaim Refs #4616 Preserve foreign row-spill directories on store-open sweeping and drop. Persist bounded owner receipts before writing rows; reclaim only from receipt-verified held directories, with native crash/replacement tests. --- crates/tracedecay-graph-db/src/generation.rs | 5 +- .../src/generation/spill.rs | 94 ++++++++++++++++++- .../tracedecay-graph-db/src/sealed_store.rs | 16 +++- 3 files changed, 106 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation.rs b/crates/tracedecay-graph-db/src/generation.rs index 7a78013cdc9..8f8b912aa49 100644 --- a/crates/tracedecay-graph-db/src/generation.rs +++ b/crates/tracedecay-graph-db/src/generation.rs @@ -59,7 +59,10 @@ pub use replay::{ pub(crate) use replay::{ checked_decode_replay_source, metadata_manifest_from_source, validate_supplied_rows_binding, }; -pub(crate) use spill::{ENTITIES_FILE, read_spilled_entity}; +pub(crate) use spill::{ + ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, + read_spilled_entity, row_spill_owner_receipt, +}; pub use spill::{ GRAPH_ROW_SPILL_RUN_BYTES, GraphGenerationRowSpill, GraphGenerationRows, GraphSpillRowFootprint, SpilledGraphGeneration, diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 92ff10db944..3ffdb687859 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -17,6 +17,7 @@ use std::cmp::{Ordering as CmpOrdering, Reverse}; use std::collections::BinaryHeap; +use std::ffi::OsStr; use std::fs::File; use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; @@ -59,13 +60,41 @@ const RELATIONS_FILE: &str = "relations.rows"; /// A producer-owned file that travels with the sealed generation, see /// [`GraphGenerationRowSpill::attachment_path`]. const ATTACHMENT_FILE: &str = "attachment"; +/// An abandoned row-spill name is not authority to erase an occupied directory. +/// Persist this marker before any rows; a crash before the marker leaves an +/// unverified directory for inspection rather than authorizing deletion. +pub(crate) const ROW_SPILL_OWNER_RECEIPT_FILE: &str = ".row-spill-owner-v1"; +pub(crate) const MAX_ROW_SPILL_OWNER_RECEIPT_BYTES: usize = 512; + +pub(crate) fn row_spill_owner_receipt(name: &OsStr) -> Option> { + let name = name.to_str()?; + let receipt = format!("tracedecay.graph-row-spill-v1:{name}").into_bytes(); + (receipt.len() <= MAX_ROW_SPILL_OWNER_RECEIPT_BYTES).then_some(receipt) +} /// Scratch directory owned by one spill; removed with its owner. struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { + let name = path + .file_name() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; + let receipt = row_spill_owner_receipt(name) + .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; std::fs::create_dir(&path).map_err(|error| spill_io("directory create", error))?; + // Exclusive marker creation refuses a substituted preexisting file. + // The marker is synced before row writes so crash recovery can verify + // this exact directory instead of deleting by its filename. + let marker = path.join(ROW_SPILL_OWNER_RECEIPT_FILE); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker) + .map_err(|error| spill_io("owner marker create", error))?; + file.write_all(&receipt) + .and_then(|()| file.sync_all()) + .map_err(|error| spill_io("owner marker persist", error))?; Ok(Self(path)) } @@ -76,14 +105,27 @@ impl SpillDirectory { impl Drop for SpillDirectory { fn drop(&mut self) { - match std::fs::remove_dir_all(&self.0) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + let Some((root, name)) = self.0.parent().zip(self.0.file_name()) else { + return; + }; + let Some(receipt) = row_spill_owner_receipt(name) else { + return; + }; + // The pathname may have been replaced since this spill was created. + // Read the proof and remove relative to the same held child capability. + match tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |bytes| bytes == receipt, + ) { + Ok(_) => {} Err(error) => tracing::warn!( event = "graph_row_spill_remove_failed", path = %self.0.display(), error = %error, - "graph row spill directory could not be removed; the next store open sweeps it" + "unverified graph row spill preserved for inspection" ), } } @@ -1037,3 +1079,47 @@ impl GraphGenerationManifestIdentity { .map_err(|error| GraphDbError::invalid(error.to_string())) } } + +#[cfg(test)] +mod owner_receipt_tests { + use super::{SpillDirectory, ROW_SPILL_OWNER_RECEIPT_FILE}; + + #[test] + fn row_spill_recovery_reclaims_only_proven_owners_and_drop_is_fail_closed() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let other_pid = std::process::id().wrapping_add(1); + let owned = root.join(format!(".rows-{other_pid}-1")); + let completed = SpillDirectory::create(owned.clone()).unwrap(); + std::fs::write(owned.join("entities-0.run"), b"owned").unwrap(); + std::mem::forget(completed); // Simulate a producer killed before Drop. + + let unreceipted = root.join(format!(".rows-{other_pid}-2")); + std::fs::create_dir(&unreceipted).unwrap(); + std::fs::write(unreceipted.join("foreign"), b"unowned").unwrap(); + let forged = root.join(format!(".rows-{other_pid}-3")); + let interrupted = SpillDirectory::create(forged.clone()).unwrap(); + std::fs::write(forged.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"wrong owner").unwrap(); + std::fs::write(forged.join("foreign"), b"preserve").unwrap(); + std::mem::forget(interrupted); + + crate::sealed_store::sweep_abandoned_row_spills(&database); + assert!(!owned.exists(), "the completed, marker-bound spill is reclaimed"); + assert_eq!(std::fs::read(unreceipted.join("foreign")).unwrap(), b"unowned"); + assert_eq!(std::fs::read(forged.join("foreign")).unwrap(), b"preserve"); + + let active = root.join(format!(".rows-{}-4", std::process::id())); + let spill = SpillDirectory::create(active.clone()).unwrap(); + drop(spill); + assert!(!active.exists(), "a current, proven spill cleans up on drop"); + + let substituted = root.join(format!(".rows-{}-5", std::process::id())); + let spill = SpillDirectory::create(substituted.clone()).unwrap(); + std::fs::write(substituted.join("foreign"), b"still here").unwrap(); + std::fs::write(substituted.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"substituted").unwrap(); + drop(spill); + assert_eq!(std::fs::read(substituted.join("foreign")).unwrap(), b"still here"); + } +} diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 8396576fabe..ec70c1fab78 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -58,7 +58,8 @@ use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nof use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ - GraphRowDigestSum, physical_namespace_projection_map, recovered_digest_from_row_sum, + GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, + physical_namespace_projection_map, recovered_digest_from_row_sum, row_spill_owner_receipt, verify_sealed_copy_generation, }; use crate::lease::GenerationLocator; @@ -793,18 +794,25 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { owner != own_pid && owner.to_string() == pid && counter.to_string() == sequence }) { + let name = entry.file_name(); + let Some(receipt) = row_spill_owner_receipt(&name) else { + continue; + }; let path = entry.path(); if let Err(error) = - tracedecay_private_fs::capability_dir::remove_child_directory_nofollow( + tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( &root, - &entry.file_name(), + &name, + std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |bytes| bytes == receipt, ) { tracing::warn!( event = "graph_row_spill_sweep_failed", path = %path.display(), error = %error, - "abandoned graph row spill could not be removed" + "unverified abandoned graph row spill preserved for inspection" ); } } From 05c18a3c708eaa6c4b9d295c5bd57f480535a55d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:50:37 -0800 Subject: [PATCH 18/70] fix(graph): publish row spills only with a complete owner receipt --- .../src/generation/spill.rs | 93 ++++++++++++++++--- .../sealed_store.rs | 16 ++-- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 3ffdb687859..c83a5f36341 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -82,11 +82,14 @@ impl SpillDirectory { .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; let receipt = row_spill_owner_receipt(name) .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; - std::fs::create_dir(&path).map_err(|error| spill_io("directory create", error))?; - // Exclusive marker creation refuses a substituted preexisting file. - // The marker is synced before row writes so crash recovery can verify - // this exact directory instead of deleting by its filename. - let marker = path.join(ROW_SPILL_OWNER_RECEIPT_FILE); + let parent = path + .parent() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; + // Receipt failures must leave the requested name available to retry. + // Keep the partial marker private until the complete spill can be installed. + let mut staging = + tempfile::tempdir_in(parent).map_err(|error| spill_io("directory create", error))?; + let marker = staging.path().join(ROW_SPILL_OWNER_RECEIPT_FILE); let mut file = std::fs::OpenOptions::new() .write(true) .create_new(true) @@ -95,6 +98,10 @@ impl SpillDirectory { file.write_all(&receipt) .and_then(|()| file.sync_all()) .map_err(|error| spill_io("owner marker persist", error))?; + drop(file); + tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) + .map_err(|error| spill_io("directory install", error))?; + staging.disable_cleanup(true); Ok(Self(path)) } @@ -1082,7 +1089,55 @@ impl GraphGenerationManifestIdentity { #[cfg(test)] mod owner_receipt_tests { - use super::{SpillDirectory, ROW_SPILL_OWNER_RECEIPT_FILE}; + use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory}; + + #[cfg(unix)] + #[test] + fn receipt_write_failure_leaves_no_spill_directory() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_LIMITED_SPILL_ROOT"; + const WRITE_REFUSED: i32 = 42; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let destination = std::path::PathBuf::from(root).join(".rows-1-0"); + let error = SpillDirectory::create(destination) + .err() + .expect("zero file-size limit"); + assert!( + error.to_string().contains("owner marker persist"), + "{error}" + ); + std::process::exit(WRITE_REFUSED); + } + + // Limit only the child process. Its directory creation succeeds and + // the real filesystem refuses the first receipt write with EFBIG. + let root = tempfile::tempdir().unwrap(); + let child = std::process::Command::new("/bin/sh") + .args(["-c", r#"trap "" XFSZ && ulimit -f 0 && exec "$@""#, "sh"]) + .arg(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, root.path()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(WRITE_REFUSED), "{child:?}"); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } + + #[test] + fn occupied_spill_publication_preserves_destination_and_discards_staging() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join(".rows-1-0"); + std::fs::create_dir(&destination).unwrap(); + assert!(SpillDirectory::create(destination.clone()).is_err()); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 1); + assert_eq!(std::fs::read_dir(&destination).unwrap().count(), 0); + + std::fs::remove_dir(&destination).unwrap(); + let spill = SpillDirectory::create(destination.clone()).unwrap(); + assert!(destination.join(ROW_SPILL_OWNER_RECEIPT_FILE).is_file()); + drop(spill); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } #[test] fn row_spill_recovery_reclaims_only_proven_owners_and_drop_is_fail_closed() { @@ -1106,20 +1161,36 @@ mod owner_receipt_tests { std::mem::forget(interrupted); crate::sealed_store::sweep_abandoned_row_spills(&database); - assert!(!owned.exists(), "the completed, marker-bound spill is reclaimed"); - assert_eq!(std::fs::read(unreceipted.join("foreign")).unwrap(), b"unowned"); + assert!( + !owned.exists(), + "the completed, marker-bound spill is reclaimed" + ); + assert_eq!( + std::fs::read(unreceipted.join("foreign")).unwrap(), + b"unowned" + ); assert_eq!(std::fs::read(forged.join("foreign")).unwrap(), b"preserve"); let active = root.join(format!(".rows-{}-4", std::process::id())); let spill = SpillDirectory::create(active.clone()).unwrap(); drop(spill); - assert!(!active.exists(), "a current, proven spill cleans up on drop"); + assert!( + !active.exists(), + "a current, proven spill cleans up on drop" + ); let substituted = root.join(format!(".rows-{}-5", std::process::id())); let spill = SpillDirectory::create(substituted.clone()).unwrap(); std::fs::write(substituted.join("foreign"), b"still here").unwrap(); - std::fs::write(substituted.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"substituted").unwrap(); + std::fs::write( + substituted.join(ROW_SPILL_OWNER_RECEIPT_FILE), + b"substituted", + ) + .unwrap(); drop(spill); - assert_eq!(std::fs::read(substituted.join("foreign")).unwrap(), b"still here"); + assert_eq!( + std::fs::read(substituted.join("foreign")).unwrap(), + b"still here" + ); } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 05bd84f29a2..0f43b56e3a3 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -2453,10 +2453,14 @@ fn spilled_rows_refuse_conflicting_repeats_and_dangling_endpoints() { fn store_open_sweeps_row_spills_abandoned_by_another_process() { let temp = TempDir::new().unwrap(); let root = sealed_store_root(temp.path()); - std::fs::create_dir_all(root.join(".rows-4194305-0")).unwrap(); - std::fs::write(root.join(".rows-4194305-0/entities-0.run"), b"abandoned").unwrap(); - let own = format!(".rows-{}-999999", std::process::id()); - std::fs::create_dir_all(root.join(&own)).unwrap(); + std::fs::create_dir_all(&root).unwrap(); + let abandoned = root.join(".rows-4194305-0"); + let identity = projection("code", "store-open-spill"); + let spill = GraphGenerationRowSpill::create(abandoned.clone(), identity.clone()).unwrap(); + std::fs::write(abandoned.join("entities-0.run"), b"abandoned").unwrap(); + std::mem::forget(spill); // Simulate a producer that exited before Drop. + let own = root.join(format!(".rows-{}-999999", std::process::id())); + let _live_spill = GraphGenerationRowSpill::create(own.clone(), identity).unwrap(); let foreign = ".rows-notes"; std::fs::create_dir(root.join(foreign)).unwrap(); @@ -2467,6 +2471,6 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::read(root.join(foreign).join("notes")).unwrap(), b"keep" ); - assert!(root.join(own).is_dir()); - assert!(!root.join(".rows-4194305-0").exists()); + assert!(own.is_dir()); + assert!(!abandoned.exists()); } From d13622cb45887b7d8688a7b87185ec0217b4e2f4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:07:28 -0800 Subject: [PATCH 19/70] fix(graph): recover pending row spills after producer exit --- crates/tracedecay-graph-db/src/generation.rs | 4 +- .../src/generation/spill.rs | 130 ++++++++++++++++-- .../tracedecay-graph-db/src/sealed_store.rs | 29 +--- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation.rs b/crates/tracedecay-graph-db/src/generation.rs index 8f8b912aa49..2db4c376fb7 100644 --- a/crates/tracedecay-graph-db/src/generation.rs +++ b/crates/tracedecay-graph-db/src/generation.rs @@ -60,8 +60,8 @@ pub(crate) use replay::{ checked_decode_replay_source, metadata_manifest_from_source, validate_supplied_rows_binding, }; pub(crate) use spill::{ - ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, - read_spilled_entity, row_spill_owner_receipt, + AbandonedRowSpill, ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, read_spilled_entity, }; pub use spill::{ GRAPH_ROW_SPILL_RUN_BYTES, GraphGenerationRowSpill, GraphGenerationRows, diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index c83a5f36341..90949aef3f5 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -34,6 +34,8 @@ use tracedecay_store::runtime::{ StoreShardIdV1, }; +use tracedecay_domain::canonical_text::{canonical_framed_sha256, is_lowercase_hex}; + use crate::limits::{MAX_VERIFIED_GENERATION_ENTITIES, MAX_VERIFIED_GENERATION_RELATIONS}; use crate::row_index::{ ENTITY_ROW_OFFSETS_FILE, EntityRowOffsetsWriter, ROW_INDEX_FILE, RowIndexBuilder, @@ -66,17 +68,78 @@ const ATTACHMENT_FILE: &str = "attachment"; pub(crate) const ROW_SPILL_OWNER_RECEIPT_FILE: &str = ".row-spill-owner-v1"; pub(crate) const MAX_ROW_SPILL_OWNER_RECEIPT_BYTES: usize = 512; -pub(crate) fn row_spill_owner_receipt(name: &OsStr) -> Option> { +pub(crate) const ROW_SPILL_PREFIX: &str = ".rows-"; +const RECEIPT_PREFIX: &str = "tracedecay.graph-row-spill-v1:"; + +fn row_spill_owner_receipt(name: &OsStr) -> Option> { let name = name.to_str()?; - let receipt = format!("tracedecay.graph-row-spill-v1:{name}").into_bytes(); + let receipt = format!("{RECEIPT_PREFIX}{name}").into_bytes(); (receipt.len() <= MAX_ROW_SPILL_OWNER_RECEIPT_BYTES).then_some(receipt) } +fn staging_name(owner: u32, receipt: &[u8]) -> String { + let digest = canonical_framed_sha256( + b"tracedecay.graph-row-spill.pending.v1", + &[&owner.to_be_bytes(), receipt], + ); + format!("{ROW_SPILL_PREFIX}{owner}-pending-{digest}") +} + +/// A foreign publisher may have stopped immediately before or after rename. +/// Reclaim either name only when the receipt on the held child proves it. +pub(crate) enum AbandonedRowSpill<'a> { + Published(Vec), + Pending { owner: u32, name: &'a str }, +} + +impl<'a> AbandonedRowSpill<'a> { + pub(crate) fn from_name(name: &'a OsStr, current: u32) -> Option { + let text = name.to_str()?; + let (pid, suffix) = text.strip_prefix(ROW_SPILL_PREFIX)?.split_once('-')?; + let owner = pid.parse::().ok()?; + if current == owner || owner.to_string() != pid { + return None; + } + if let Some(digest) = suffix.strip_prefix("pending-") { + return is_lowercase_hex(digest, 64).then_some(Self::Pending { owner, name: text }); + } + let counter = suffix.parse::().ok()?; + if counter.to_string() != suffix { + return None; + } + row_spill_owner_receipt(name).map(Self::Published) + } + + pub(crate) fn owns(&self, receipt: &[u8]) -> bool { + match self { + Self::Published(expected) => receipt == expected, + Self::Pending { owner, name } => { + let Some(bound_name) = receipt + .strip_prefix(RECEIPT_PREFIX.as_bytes()) + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + else { + return false; + }; + Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) + && staging_name(*owner, receipt) == *name + } + } + } +} + /// Scratch directory owned by one spill; removed with its owner. struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { + let mut staging = Self::prepare(&path)?; + tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) + .map_err(|error| spill_io("directory install", error))?; + staging.disable_cleanup(true); + Ok(Self(path)) + } + + fn prepare(path: &Path) -> Result { let name = path .file_name() .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; @@ -87,8 +150,12 @@ impl SpillDirectory { .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; // Receipt failures must leave the requested name available to retry. // Keep the partial marker private until the complete spill can be installed. - let mut staging = - tempfile::tempdir_in(parent).map_err(|error| spill_io("directory create", error))?; + let temporary = staging_name(std::process::id(), &receipt); + let staging = tempfile::Builder::new() + .prefix(&temporary) + .rand_bytes(0) + .tempdir_in(parent) + .map_err(|error| spill_io("directory create", error))?; let marker = staging.path().join(ROW_SPILL_OWNER_RECEIPT_FILE); let mut file = std::fs::OpenOptions::new() .write(true) @@ -99,10 +166,7 @@ impl SpillDirectory { .and_then(|()| file.sync_all()) .map_err(|error| spill_io("owner marker persist", error))?; drop(file); - tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) - .map_err(|error| spill_io("directory install", error))?; - staging.disable_cleanup(true); - Ok(Self(path)) + Ok(staging) } fn path(&self) -> &Path { @@ -1123,6 +1187,56 @@ mod owner_receipt_tests { assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); } + #[test] + fn store_open_reclaims_receipted_staging_after_producer_exits_before_rename() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_PENDING_SPILL_ROOT"; + const PREPARED: i32 = 42; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let name = format!(".rows-{}-1", std::process::id()); + let _staged = + SpillDirectory::prepare(&std::path::PathBuf::from(root).join(name)).unwrap(); + std::process::exit(PREPARED); + } + + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, &root) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(PREPARED), "{child:?}"); + let interrupted = std::fs::read_dir(&root) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let (owner, _) = interrupted + .file_name() + .unwrap() + .to_str() + .unwrap() + .split_once("-pending-") + .unwrap(); + let mismatched = root.join(format!("{owner}-pending-{}", "0".repeat(64))); + std::fs::create_dir(&mismatched).unwrap(); + let receipt = std::fs::read(interrupted.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(); + std::fs::write(mismatched.join(ROW_SPILL_OWNER_RECEIPT_FILE), &receipt).unwrap(); + + let active = SpillDirectory::prepare(&root.join(".rows-active")).unwrap(); + crate::sealed_store::sweep_abandoned_row_spills(&database); + assert!(!interrupted.exists()); + assert!(active.path().is_dir()); + assert_eq!( + std::fs::read(mismatched.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(), + receipt + ); + } + #[test] fn occupied_spill_publication_preserves_destination_and_discards_staging() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ec70c1fab78..86330fcc7e4 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -58,9 +58,9 @@ use tracedecay_private_fs::capability_dir::install_child_directory_noreplace_nof use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ - GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, - physical_namespace_projection_map, recovered_digest_from_row_sum, row_spill_owner_receipt, - verify_sealed_copy_generation, + AbandonedRowSpill, GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, physical_namespace_projection_map, + recovered_digest_from_row_sum, verify_sealed_copy_generation, }; use crate::lease::GenerationLocator; use crate::location::PersistentGraphStoreState; @@ -767,9 +767,6 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { } } -/// Prefix of a batch producer's row spill under the sealed root. -const ROW_SPILL_PREFIX: &str = ".rows-"; - /// Removes every row spill another process left under the store's sealed /// root. Run at the eager open, whose exclusive store lock means no other /// process is producing rows for this store. Spills named for this process @@ -782,22 +779,8 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { }; let own_pid = std::process::id(); for entry in entries.flatten() { - if entry - .file_name() - .to_str() - .and_then(|name| name.strip_prefix(ROW_SPILL_PREFIX)) - .and_then(|suffix| suffix.split_once('-')) - .is_some_and(|(pid, sequence)| { - let (Ok(owner), Ok(counter)) = (pid.parse::(), sequence.parse::()) else { - return false; - }; - owner != own_pid && owner.to_string() == pid && counter.to_string() == sequence - }) - { - let name = entry.file_name(); - let Some(receipt) = row_spill_owner_receipt(&name) else { - continue; - }; + let name = entry.file_name(); + if let Some(proof) = AbandonedRowSpill::from_name(&name, own_pid) { let path = entry.path(); if let Err(error) = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( @@ -805,7 +788,7 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { &name, std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, - |bytes| bytes == receipt, + |bytes| proof.owns(bytes), ) { tracing::warn!( From ded88e87c54cde9c989ef3edd035242dcd1b8b6e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:28:29 -0800 Subject: [PATCH 20/70] fix(graph): distinguish spill owners across reused process IDs --- crates/tracedecay-graph-db/src/generation.rs | 2 +- .../src/generation/spill.rs | 98 +++++++++++++------ .../tracedecay-graph-db/src/sealed_store.rs | 8 +- .../sealed_store.rs | 47 +++++++-- .../tracedecay-private-fs/src/coarse_time.rs | 6 +- crates/tracedecay-private-fs/src/lib.rs | 11 +++ 6 files changed, 124 insertions(+), 48 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation.rs b/crates/tracedecay-graph-db/src/generation.rs index 2db4c376fb7..ba64685fd60 100644 --- a/crates/tracedecay-graph-db/src/generation.rs +++ b/crates/tracedecay-graph-db/src/generation.rs @@ -61,7 +61,7 @@ pub(crate) use replay::{ }; pub(crate) use spill::{ AbandonedRowSpill, ENTITIES_FILE, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, - ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, read_spilled_entity, + ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, read_spilled_entity, row_spill_owner, }; pub use spill::{ GRAPH_ROW_SPILL_RUN_BYTES, GraphGenerationRowSpill, GraphGenerationRows, diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 90949aef3f5..57a76eb1238 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -71,13 +71,20 @@ pub(crate) const MAX_ROW_SPILL_OWNER_RECEIPT_BYTES: usize = 512; pub(crate) const ROW_SPILL_PREFIX: &str = ".rows-"; const RECEIPT_PREFIX: &str = "tracedecay.graph-row-spill-v1:"; -fn row_spill_owner_receipt(name: &OsStr) -> Option> { +fn row_spill_owner_receipt(name: &OsStr, owner: u128) -> Option> { let name = name.to_str()?; - let receipt = format!("{RECEIPT_PREFIX}{name}").into_bytes(); + let receipt = format!("{RECEIPT_PREFIX}{owner}:{name}").into_bytes(); (receipt.len() <= MAX_ROW_SPILL_OWNER_RECEIPT_BYTES).then_some(receipt) } -fn staging_name(owner: u32, receipt: &[u8]) -> String { +/// The PID changes at fork. The process nonce changes on restart, even when +/// the kernel reuses that PID and the allocation counter starts over. +pub(crate) fn row_spill_owner() -> u128 { + (u128::from(std::process::id()) << 64) + | u128::from(tracedecay_private_fs::process_instance_nonce()) +} + +fn staging_name(owner: u128, receipt: &[u8]) -> String { let digest = canonical_framed_sha256( b"tracedecay.graph-row-spill.pending.v1", &[&owner.to_be_bytes(), receipt], @@ -89,15 +96,15 @@ fn staging_name(owner: u32, receipt: &[u8]) -> String { /// Reclaim either name only when the receipt on the held child proves it. pub(crate) enum AbandonedRowSpill<'a> { Published(Vec), - Pending { owner: u32, name: &'a str }, + Pending { owner: u128, name: &'a str }, } impl<'a> AbandonedRowSpill<'a> { - pub(crate) fn from_name(name: &'a OsStr, current: u32) -> Option { + pub(crate) fn from_name(name: &'a OsStr, current: u128) -> Option { let text = name.to_str()?; - let (pid, suffix) = text.strip_prefix(ROW_SPILL_PREFIX)?.split_once('-')?; - let owner = pid.parse::().ok()?; - if current == owner || owner.to_string() != pid { + let (instance, suffix) = text.strip_prefix(ROW_SPILL_PREFIX)?.split_once('-')?; + let owner = instance.parse::().ok()?; + if current == owner || owner.to_string() != instance { return None; } if let Some(digest) = suffix.strip_prefix("pending-") { @@ -107,7 +114,7 @@ impl<'a> AbandonedRowSpill<'a> { if counter.to_string() != suffix { return None; } - row_spill_owner_receipt(name).map(Self::Published) + row_spill_owner_receipt(name, owner).map(Self::Published) } pub(crate) fn owns(&self, receipt: &[u8]) -> bool { @@ -120,7 +127,11 @@ impl<'a> AbandonedRowSpill<'a> { else { return false; }; - Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) + let Some((declared_owner, bound_name)) = bound_name.split_once(':') else { + return false; + }; + declared_owner == owner.to_string() + && Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) && staging_name(*owner, receipt) == *name } } @@ -132,25 +143,25 @@ struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { - let mut staging = Self::prepare(&path)?; + let mut staging = Self::prepare(&path, row_spill_owner())?; tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) .map_err(|error| spill_io("directory install", error))?; staging.disable_cleanup(true); Ok(Self(path)) } - fn prepare(path: &Path) -> Result { + fn prepare(path: &Path, owner: u128) -> Result { let name = path .file_name() .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; - let receipt = row_spill_owner_receipt(name) + let receipt = row_spill_owner_receipt(name, owner) .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; let parent = path .parent() .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; // Receipt failures must leave the requested name available to retry. // Keep the partial marker private until the complete spill can be installed. - let temporary = staging_name(std::process::id(), &receipt); + let temporary = staging_name(owner, &receipt); let staging = tempfile::Builder::new() .prefix(&temporary) .rand_bytes(0) @@ -179,7 +190,7 @@ impl Drop for SpillDirectory { let Some((root, name)) = self.0.parent().zip(self.0.file_name()) else { return; }; - let Some(receipt) = row_spill_owner_receipt(name) else { + let Some(receipt) = row_spill_owner_receipt(name, row_spill_owner()) else { return; }; // The pathname may have been replaced since this spill was created. @@ -1153,7 +1164,7 @@ impl GraphGenerationManifestIdentity { #[cfg(test)] mod owner_receipt_tests { - use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory}; + use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory, row_spill_owner}; #[cfg(unix)] #[test] @@ -1192,9 +1203,12 @@ mod owner_receipt_tests { const CHILD_ROOT: &str = "TRACEDECAY_TEST_PENDING_SPILL_ROOT"; const PREPARED: i32 = 42; if let Some(root) = std::env::var_os(CHILD_ROOT) { - let name = format!(".rows-{}-1", std::process::id()); - let _staged = - SpillDirectory::prepare(&std::path::PathBuf::from(root).join(name)).unwrap(); + let name = format!(".rows-{}-1", row_spill_owner()); + let _staged = SpillDirectory::prepare( + &std::path::PathBuf::from(root).join(name), + row_spill_owner(), + ) + .unwrap(); std::process::exit(PREPARED); } @@ -1227,7 +1241,8 @@ mod owner_receipt_tests { let receipt = std::fs::read(interrupted.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(); std::fs::write(mismatched.join(ROW_SPILL_OWNER_RECEIPT_FILE), &receipt).unwrap(); - let active = SpillDirectory::prepare(&root.join(".rows-active")).unwrap(); + let active = + SpillDirectory::prepare(&root.join(".rows-active"), row_spill_owner()).unwrap(); crate::sealed_store::sweep_abandoned_row_spills(&database); assert!(!interrupted.exists()); assert!(active.path().is_dir()); @@ -1237,6 +1252,32 @@ mod owner_receipt_tests { ); } + #[test] + fn a_reused_pid_reclaims_the_prior_incarnation_and_preserves_live_spills() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let current = row_spill_owner(); + let previous = current ^ 1; + assert_eq!(current >> 64, previous >> 64, "same numeric process ID"); + let final_path = root.join(format!(".rows-{current}-0")); + let stale = SpillDirectory::prepare(&final_path, previous) + .unwrap() + .keep(); + let live = SpillDirectory::prepare(&final_path, current).unwrap(); + let peer_path = root.join(format!(".rows-{current}-1")); + let _peer = SpillDirectory::create(peer_path.clone()).unwrap(); + + crate::sealed_store::sweep_abandoned_row_spills(&database); + assert!(!stale.exists()); + assert!(live.path().is_dir()); + assert!(peer_path.is_dir()); + drop(live); + let _published = SpillDirectory::create(final_path.clone()).unwrap(); + assert!(final_path.is_dir()); + } + #[test] fn occupied_spill_publication_preserves_destination_and_discards_staging() { let root = tempfile::tempdir().unwrap(); @@ -1259,16 +1300,17 @@ mod owner_receipt_tests { let database = temp.path().join("graph.grafeo"); let root = temp.path().join("graph.sealed"); std::fs::create_dir(&root).unwrap(); - let other_pid = std::process::id().wrapping_add(1); - let owned = root.join(format!(".rows-{other_pid}-1")); - let completed = SpillDirectory::create(owned.clone()).unwrap(); + let other_owner = row_spill_owner() ^ 1; + let owned = root.join(format!(".rows-{other_owner}-1")); + let mut completed = SpillDirectory::prepare(&owned, other_owner).unwrap(); + tracedecay_private_fs::framed_log::rename_noreplace(completed.path(), &owned).unwrap(); + completed.disable_cleanup(true); std::fs::write(owned.join("entities-0.run"), b"owned").unwrap(); - std::mem::forget(completed); // Simulate a producer killed before Drop. - let unreceipted = root.join(format!(".rows-{other_pid}-2")); + let unreceipted = root.join(format!(".rows-{other_owner}-2")); std::fs::create_dir(&unreceipted).unwrap(); std::fs::write(unreceipted.join("foreign"), b"unowned").unwrap(); - let forged = root.join(format!(".rows-{other_pid}-3")); + let forged = root.join(format!(".rows-{other_owner}-3")); let interrupted = SpillDirectory::create(forged.clone()).unwrap(); std::fs::write(forged.join(ROW_SPILL_OWNER_RECEIPT_FILE), b"wrong owner").unwrap(); std::fs::write(forged.join("foreign"), b"preserve").unwrap(); @@ -1285,7 +1327,7 @@ mod owner_receipt_tests { ); assert_eq!(std::fs::read(forged.join("foreign")).unwrap(), b"preserve"); - let active = root.join(format!(".rows-{}-4", std::process::id())); + let active = root.join(format!(".rows-{}-4", row_spill_owner())); let spill = SpillDirectory::create(active.clone()).unwrap(); drop(spill); assert!( @@ -1293,7 +1335,7 @@ mod owner_receipt_tests { "a current, proven spill cleans up on drop" ); - let substituted = root.join(format!(".rows-{}-5", std::process::id())); + let substituted = root.join(format!(".rows-{}-5", row_spill_owner())); let spill = SpillDirectory::create(substituted.clone()).unwrap(); std::fs::write(substituted.join("foreign"), b"still here").unwrap(); std::fs::write( diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 86330fcc7e4..e22b590885d 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -60,7 +60,7 @@ use tracedecay_store::runtime::GraphRecoveredGenerationDigestV1; use crate::generation::{ AbandonedRowSpill, GraphRowDigestSum, MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, physical_namespace_projection_map, - recovered_digest_from_row_sum, verify_sealed_copy_generation, + recovered_digest_from_row_sum, row_spill_owner, verify_sealed_copy_generation, }; use crate::lease::GenerationLocator; use crate::location::PersistentGraphStoreState; @@ -777,10 +777,10 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let Ok(entries) = std::fs::read_dir(&root) else { return; }; - let own_pid = std::process::id(); + let current_owner = row_spill_owner(); for entry in entries.flatten() { let name = entry.file_name(); - if let Some(proof) = AbandonedRowSpill::from_name(&name, own_pid) { + if let Some(proof) = AbandonedRowSpill::from_name(&name, current_owner) { let path = entry.path(); if let Err(error) = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( @@ -1469,7 +1469,7 @@ impl GraphDb { .map_err(|error| sealed_store_io_failure("row spill root create failed", error))?; Ok(root.join(format!( "{ROW_SPILL_PREFIX}{}-{}", - std::process::id(), + row_spill_owner(), NEXT_SPILL.fetch_add(1, AtomicOrdering::Relaxed) ))) } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 0f43b56e3a3..046861645a8 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -2451,16 +2451,44 @@ fn spilled_rows_refuse_conflicting_repeats_and_dangling_endpoints() { /// own live publisher's and stays. #[test] fn store_open_sweeps_row_spills_abandoned_by_another_process() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_ABANDONED_GRAPH_SPILL"; + const SPILLED: i32 = 42; + let identity = projection("code", "store-open-spill"); + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let root = PathBuf::from(root); + let registered = RegisteredGraph::new_mounted(&root).unwrap(); + let spill = registered + .registry + .generation_row_spill(registration(registered.binding.clone(), &root), identity) + .unwrap(); + std::fs::write(spill.attachment_path(), b"abandoned").unwrap(); + std::process::exit(SPILLED); + } + let temp = TempDir::new().unwrap(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, temp.path()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(SPILLED), "{child:?}"); let root = sealed_store_root(temp.path()); - std::fs::create_dir_all(&root).unwrap(); - let abandoned = root.join(".rows-4194305-0"); - let identity = projection("code", "store-open-spill"); - let spill = GraphGenerationRowSpill::create(abandoned.clone(), identity.clone()).unwrap(); - std::fs::write(abandoned.join("entities-0.run"), b"abandoned").unwrap(); - std::mem::forget(spill); // Simulate a producer that exited before Drop. - let own = root.join(format!(".rows-{}-999999", std::process::id())); - let _live_spill = GraphGenerationRowSpill::create(own.clone(), identity).unwrap(); + let abandoned = row_spills(temp.path()); + assert_eq!(abandoned.len(), 1); + let abandoned = root.join(&abandoned[0]); + let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); + assert!(!abandoned.exists()); + let live_spill = registered + .registry + .generation_row_spill( + registration(registered.binding.clone(), temp.path()), + identity, + ) + .unwrap(); + let attachment = live_spill.attachment_path(); + std::fs::write(&attachment, b"live peer attachment").unwrap(); + drop(registered); let foreign = ".rows-notes"; std::fs::create_dir(root.join(foreign)).unwrap(); @@ -2471,6 +2499,5 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::read(root.join(foreign).join("notes")).unwrap(), b"keep" ); - assert!(own.is_dir()); - assert!(!abandoned.exists()); + assert_eq!(std::fs::read(attachment).unwrap(), b"live peer attachment"); } diff --git a/crates/tracedecay-private-fs/src/coarse_time.rs b/crates/tracedecay-private-fs/src/coarse_time.rs index 8978f1b928a..8b8ccc1f0a3 100644 --- a/crates/tracedecay-private-fs/src/coarse_time.rs +++ b/crates/tracedecay-private-fs/src/coarse_time.rs @@ -6,13 +6,10 @@ //! timestamp quantum can share a stamp, so a change time is proof the bytes //! are stable only once the clock has left the quantum it was stamped in. -use std::collections::hash_map::RandomState; use std::fs::Metadata; -use std::hash::BuildHasher; #[cfg(unix)] use std::os::unix::fs::MetadataExt; use std::path::Path; -use std::sync::LazyLock; use std::sync::atomic::{AtomicU64, Ordering}; const NANOS_PER_SECOND: i128 = 1_000_000_000; @@ -120,10 +117,9 @@ impl ChangeStamp { fn unvouched() -> Self { // The random half keeps samples from different processes apart where // an identity is persisted as a digest. - static PROCESS: LazyLock = LazyLock::new(|| RandomState::new().hash_one(0_u8)); static SAMPLE: AtomicU64 = AtomicU64::new(0); let sample = SAMPLE.fetch_add(1, Ordering::Relaxed); - Self::Unvouched((u128::from(*PROCESS) << 64) | u128::from(sample)) + Self::Unvouched((u128::from(crate::process_instance_nonce()) << 64) | u128::from(sample)) } #[must_use] diff --git a/crates/tracedecay-private-fs/src/lib.rs b/crates/tracedecay-private-fs/src/lib.rs index 7e7298a08db..52c99ebe108 100644 --- a/crates/tracedecay-private-fs/src/lib.rs +++ b/crates/tracedecay-private-fs/src/lib.rs @@ -2,7 +2,10 @@ //! plus the Windows file-handle identity primitive that shares this crate's //! cold, dependency-light position. +use std::collections::hash_map::RandomState; +use std::hash::BuildHasher; use std::io; +use std::sync::LazyLock; pub mod capability_dir; mod coarse_time; @@ -15,6 +18,14 @@ mod rename_noreplace; pub use coarse_time::{ChangeClockReading, ChangeStamp, RewriteWitness, change_time_settled}; pub use inode_generation::inode_generation; +/// A process-local random witness for persisted identities that must distinguish +/// successive processes reusing a PID. Pair it with the PID for forked children. +#[must_use] +pub fn process_instance_nonce() -> u64 { + static NONCE: LazyLock = LazyLock::new(|| RandomState::new().hash_one(0_u8)); + *NONCE +} + pub use file_lease::FileLease; pub use lock_admission::{LockAdmissionError, lock_shared_until, lock_until}; #[cfg(windows)] From 59b5cd33c56ea30c730a12303b95d2261b15cb60 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:35:16 -0800 Subject: [PATCH 21/70] fix(graph): keep row-spill sweep bound to one sealed-root handle Refs #4616 Reuse the receipt-verified held-root collector for abandoned row spills. Filter candidate names before receipt IO so sealed generations are not probed as spills. Cover root path replacement during verified collection and preserve wrong-receipt foreign rows on a real store reopen. --- .../tracedecay-graph-db/src/sealed_store.rs | 52 +++++++------- .../sealed_store.rs | 10 +++ .../src/capability_dir.rs | 67 +++++++++++++++++++ 3 files changed, 105 insertions(+), 24 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index e22b590885d..9c32619154e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -774,31 +774,35 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { /// whose spill removes itself when it drops. pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { let root = sealed_store_root(database_path); - let Ok(entries) = std::fs::read_dir(&root) else { - return; - }; let current_owner = row_spill_owner(); - for entry in entries.flatten() { - let name = entry.file_name(); - if let Some(proof) = AbandonedRowSpill::from_name(&name, current_owner) { - let path = entry.path(); - if let Err(error) = - tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( - &root, - &name, - std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), - MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, - |bytes| proof.owns(bytes), - ) - { - tracing::warn!( - event = "graph_row_spill_sweep_failed", - path = %path.display(), - error = %error, - "unverified abandoned graph row spill preserved for inspection" - ); - } - } + // Enumerate, verify and unlink through one held sealed-root capability: + // a root rename between discovery and cleanup must not redirect deletion. + let result = + tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |name| AbandonedRowSpill::from_name(name, current_owner).is_some(), + |name, receipt| { + AbandonedRowSpill::from_name(name, current_owner) + .is_some_and(|proof| proof.owns(receipt)) + }, + ); + match result { + Ok(removed) if removed > 0 => tracing::info!( + event = "graph_row_spill_swept", + root = %root.display(), + removed, + "reclaimed receipt-verified abandoned graph row spills" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "graph_row_spill_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned graph row spills preserved for inspection" + ), } } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 046861645a8..1fd3f0ada90 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -2490,6 +2490,12 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::write(&attachment, b"live peer attachment").unwrap(); drop(registered); + // A different producer's name is not authority to reclaim foreign bytes. + // Even a well-shaped abandoned-spill entry with a wrong receipt survives + // store opening, while the authentic abandoned spill above was reclaimed. + std::fs::create_dir(&abandoned).unwrap(); + std::fs::write(abandoned.join(".row-spill-owner-v1"), b"wrong owner").unwrap(); + std::fs::write(abandoned.join("foreign"), b"preserve").unwrap(); let foreign = ".rows-notes"; std::fs::create_dir(root.join(foreign)).unwrap(); std::fs::write(root.join(foreign).join("notes"), b"keep").unwrap(); @@ -2499,5 +2505,9 @@ fn store_open_sweeps_row_spills_abandoned_by_another_process() { std::fs::read(root.join(foreign).join("notes")).unwrap(), b"keep" ); + assert_eq!( + std::fs::read(abandoned.join("foreign")).unwrap(), + b"preserve" + ); assert_eq!(std::fs::read(attachment).unwrap(), b"live peer attachment"); } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index d04bea7d3fe..a504c587af0 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -324,6 +324,26 @@ pub fn remove_receipted_directories_nofollow( root_path: &Path, receipt_name: &OsStr, maximum: usize, + authorized: impl FnMut(&OsStr, &[u8]) -> bool, +) -> io::Result { + remove_matching_receipted_directories_nofollow( + root_path, + receipt_name, + maximum, + |_| true, + authorized, + ) +} + +/// Scan and reclaim selected children beneath one held root. Filter names +/// before receipt I/O so unrelated installed generations are not treated as +/// abandoned staging or spills. Receipt verification and removal remain +/// anchored to the same opened child, even if the root pathname changes. +pub fn remove_matching_receipted_directories_nofollow( + root_path: &Path, + receipt_name: &OsStr, + maximum: usize, + mut candidate: impl FnMut(&OsStr) -> bool, mut authorized: impl FnMut(&OsStr, &[u8]) -> bool, ) -> io::Result { // Receipt names are always a single sibling: a slash-bearing name could @@ -347,6 +367,9 @@ pub fn remove_receipted_directories_nofollow( continue; } let name = entry.file_name(); + if !candidate(&name) { + continue; + } // Do not trust the DirEntry's former pathname kind after a rename. let child = match root.open_dir_nofollow(&name) { Ok(child) => child, @@ -625,6 +648,50 @@ mod tests { ); } + /// Candidate filtering must not reopen a potentially substituted root: + /// only the receipted owned child goes away; unrelated and foreign trees survive. + #[cfg(unix)] + #[test] + fn filtered_receipt_sweep_holds_root_through_path_substitution() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("owned-displaced"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join(".rows-old")).unwrap(); + std::fs::create_dir_all(root.join("generation")).unwrap(); + std::fs::create_dir_all(foreign.join(".rows-old")).unwrap(); + std::fs::write(root.join(".rows-old/.owner"), b"verified").unwrap(); + std::fs::write(root.join("generation/live"), b"live").unwrap(); + std::fs::write(foreign.join(".rows-old/.owner"), b"verified").unwrap(); + std::fs::write(foreign.join(".rows-old/sentinel"), b"foreign").unwrap(); + + let removed = remove_matching_receipted_directories_nofollow( + &root, + OsStr::new(".owner"), + 512, + |name| name == OsStr::new(".rows-old"), + |name, receipt| { + assert_eq!(name, OsStr::new(".rows-old")); + assert_eq!(receipt, b"verified"); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + true + }, + ) + .unwrap(); + + assert_eq!(removed, 1); + assert!(!displaced.join(".rows-old").exists()); + assert_eq!( + std::fs::read(displaced.join("generation/live")).unwrap(), + b"live" + ); + assert_eq!( + std::fs::read(foreign.join(".rows-old/sentinel")).unwrap(), + b"foreign" + ); + } + /// A parent-path swap after receipt validation must not redirect cleanup: /// the already-open child is removed, and matching foreign bytes survive. #[cfg(unix)] From 552b5c7fc6e82ff9248f5d0743f6ee7866eb2eb7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:51:59 -0800 Subject: [PATCH 22/70] fix(storage): continue cleanup after a verified child unlink fails --- .../src/capability_dir.rs | 66 ++++++++++++++++--- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index a504c587af0..72b9eda88b3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -339,6 +339,7 @@ pub fn remove_receipted_directories_nofollow( /// before receipt I/O so unrelated installed generations are not treated as /// abandoned staging or spills. Receipt verification and removal remain /// anchored to the same opened child, even if the root pathname changes. +/// A failure on one entry is reported after its independent siblings are processed. pub fn remove_matching_receipted_directories_nofollow( root_path: &Path, receipt_name: &OsStr, @@ -360,37 +361,50 @@ pub fn remove_matching_receipted_directories_nofollow( )); } let root = open_root_nofollow(root_path)?; - let mut removed = 0usize; - for entry in root.read_dir(".")? { + let outcomes = root.read_dir(".")?.map(|entry| -> io::Result { let entry = entry?; if !entry.file_type()?.is_dir() { - continue; + return Ok(false); } let name = entry.file_name(); if !candidate(&name) { - continue; + return Ok(false); } // Do not trust the DirEntry's former pathname kind after a rename. let child = match root.open_dir_nofollow(&name) { Ok(child) => child, Err(error) => { tracing::warn!(%error, entry = ?name, "sealed child could not be verified"); - continue; + return Ok(false); } }; let receipt = match read_bounded_child_receipt(&child, receipt_name, maximum) { Ok(receipt) => receipt, Err(error) => { tracing::warn!(%error, entry = ?name, "unverified sealed receipt preserved"); - continue; + return Ok(false); } }; - if authorized(&name, &receipt) { - remove_open_dir_all_nofollow(child)?; - removed = removed.saturating_add(1); + if !authorized(&name, &receipt) { + return Ok(false); + } + remove_open_dir_all_nofollow(child).inspect_err(|error| { + tracing::warn!(%error, entry = ?name, "verified child directory could not be removed"); + })?; + Ok(true) + }); + let mut removed = 0; + let mut first_error = None; + for outcome in outcomes { + match outcome { + Ok(true) => removed += 1, + Ok(false) => {} + Err(error) => { + first_error.get_or_insert(error); + } } } - Ok(removed) + first_error.map_or(Ok(removed), Err) } #[cfg(unix)] @@ -692,6 +706,38 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn a_failed_verified_unlink_still_reclaims_independent_siblings() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + for name in ["a", "b"] { + std::fs::create_dir_all(root.join(name)).unwrap(); + std::fs::write(root.join(name).join("owner"), name).unwrap(); + } + let mut verified = Vec::new(); + let error = remove_matching_receipted_directories_nofollow( + &root, + OsStr::new("owner"), + 64, + |_| true, + |name, receipt| { + assert_eq!(name.as_encoded_bytes(), receipt); + if verified.is_empty() { + // A concurrent cleanup wins after this child was verified. + // Its now-unlinked capability cannot be removed a second time. + std::fs::remove_dir_all(root.join(name)).unwrap(); + } + verified.push(name.to_owned()); + true + }, + ) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::NotFound, "{error}"); + assert_eq!(verified.len(), 2); + assert!(!root.join(&verified[1]).exists()); + } + /// A parent-path swap after receipt validation must not redirect cleanup: /// the already-open child is removed, and matching foreign bytes survive. #[cfg(unix)] From 6eadebb7e39913b0a1b1355ef3f9bd41510184c5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 22:14:06 -0800 Subject: [PATCH 23/70] fix(graph): verify logical owner before old-format cleanup --- .../tracedecay-graph-db/src/sealed_store.rs | 83 ++++++++++++++----- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 9c32619154e..fca6cb2ccc9 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -667,7 +667,8 @@ pub(crate) fn discard_superseded_sealed_generations( }; recognized_form && receipt.version == SEALED_STORE_RECEIPT_VERSION - && receipt.physical_namespace == format!("generation:{name}") + && sealed_receipt_matches_generation_entry(&receipt, name) + && !receipt.recovered_digest.is_empty() && receipt.graph_format > 0 && receipt.graph_format < GraphFormatVersion::current().get() }, @@ -682,6 +683,30 @@ pub(crate) fn discard_superseded_sealed_generations( } } +/// A physical generation entry is deletable only if the receipt's logical +/// locator derives that exact entry name. A copied receipt with a fabricated +/// physical namespace is not evidence of ownership. +fn sealed_receipt_matches_generation_entry( + receipt: &SealedStoreReceiptV1, + generation_hex: &str, +) -> bool { + let (Ok(namespace), Ok(projection), Ok(generation)) = ( + GraphNamespace::new(receipt.namespace.clone()), + GraphProjectionId::new(receipt.projection.clone()), + crate::GraphGenerationId::new(receipt.generation.clone()), + ) else { + return false; + }; + let locator = GenerationLocator::new( + GraphProjectionIdentity::new(namespace, projection), + generation, + ); + locator.physical_namespace().is_ok_and(|physical| { + physical.as_str() == receipt.physical_namespace + && receipt.physical_namespace == format!("generation:{generation_hex}") + }) +} + fn sealed_staging_name(name: &str) -> bool { name.strip_prefix(".staging-").is_some_and(|suffix| { suffix.len() == 64 @@ -718,23 +743,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let Ok(receipt) = serde_json::from_slice::(bytes) else { return false; }; - // Bind the receipt's logical identity back to the canonical - // physical namespace. A copied receipt with a rewritten physical - // name is not proof of ownership of another staged generation. - let (Ok(namespace), Ok(projection), Ok(generation)) = ( - GraphNamespace::new(receipt.namespace.clone()), - GraphProjectionId::new(receipt.projection.clone()), - crate::GraphGenerationId::new(receipt.generation.clone()), - ) else { - return false; - }; - let locator = GenerationLocator::new( - GraphProjectionIdentity::new(namespace, projection), - generation, - ); - let Ok(physical_namespace) = locator.physical_namespace() else { - return false; - }; + // Reuse the same identity proof as obsolete-format retirement. let recognized_form = match receipt.form.as_str() { SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), @@ -744,8 +753,7 @@ pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { && receipt.version == SEALED_STORE_RECEIPT_VERSION && receipt.graph_format > 0 && receipt.graph_format <= GraphFormatVersion::current().get() - && receipt.physical_namespace == format!("generation:{generation_hex}") - && receipt.physical_namespace == physical_namespace.as_str() + && sealed_receipt_matches_generation_entry(&receipt, generation_hex) && !receipt.recovered_digest.is_empty() }, ); @@ -3035,7 +3043,29 @@ mod receipt_budget_tests { std::fs::create_dir_all(&root).unwrap(); let current = GraphFormatVersion::current().get(); assert!(current > 0); - let old = generation(&root, 'a', &receipt(current - 1, &"a".repeat(64), "old")); + let old_locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new("generation:obsolete").unwrap(), + ); + let old_physical = old_locator.physical_namespace().unwrap(); + let old_name = old_physical.as_str().strip_prefix("generation:").unwrap(); + let old = root.join(old_name); + std::fs::create_dir(&old).unwrap(); + std::fs::write( + old.join(SEALED_STORE_RECEIPT_FILE), + receipt(current - 1, old_name, "generation:obsolete"), + ) + .unwrap(); + // A syntactically plausible old receipt with a different logical + // generation cannot authorize deletion of this occupied directory. + let forged = generation( + &root, + 'a', + &receipt(current - 1, &"a".repeat(64), "generation:foreign"), + ); let live = generation(&root, 'b', &receipt(current, &"b".repeat(64), "live")); let invalid = generation(&root, 'c', b"not a receipt"); // Zero is not a valid GraphFormatVersion, even when the JSON is well formed. @@ -3070,6 +3100,7 @@ mod receipt_budget_tests { }; for dir in [ &old, + &forged, &live, &invalid, &zero_format, @@ -3082,7 +3113,15 @@ mod receipt_budget_tests { discard_superseded_sealed_generations(&database).unwrap(); assert!(!old.exists(), "verified old-format receipt is reclaimable"); - for dir in [&live, &invalid, &zero_format, &oversized, &missing, &future] { + for dir in [ + &forged, + &live, + &invalid, + &zero_format, + &oversized, + &missing, + &future, + ] { assert_eq!(std::fs::read(dir.join("sentinel")).unwrap(), b"owned"); } assert_eq!(std::fs::read(&unrelated).unwrap(), b"foreign"); From 53457e4bead9be2fb4ad80811e8cb3897451adc2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:31:13 -0800 Subject: [PATCH 24/70] fix(graph): bind pending spill receipts to canonical names Refs #4616 --- .../src/generation/spill.rs | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 57a76eb1238..c62d9873ebb 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -130,8 +130,20 @@ impl<'a> AbandonedRowSpill<'a> { let Some((declared_owner, bound_name)) = bound_name.split_once(':') else { return false; }; + // A pending receipt must bind a real published spill name. + // Hashing arbitrary receipt text is not proof that the + // directory was ever produced by a graph row-spill writer. + let Some((published_owner, counter)) = bound_name + .strip_prefix(ROW_SPILL_PREFIX) + .and_then(|suffix| suffix.split_once('-')) + else { + return false; + }; declared_owner == owner.to_string() - && Path::new(bound_name).file_name() == Some(OsStr::new(bound_name)) + && published_owner == declared_owner + && counter + .parse::() + .is_ok_and(|parsed| parsed.to_string() == counter) && staging_name(*owner, receipt) == *name } } @@ -1164,7 +1176,10 @@ impl GraphGenerationManifestIdentity { #[cfg(test)] mod owner_receipt_tests { - use super::{ROW_SPILL_OWNER_RECEIPT_FILE, SpillDirectory, row_spill_owner}; + use super::{ + RECEIPT_PREFIX, ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, SpillDirectory, + row_spill_owner, staging_name, + }; #[cfg(unix)] #[test] @@ -1252,6 +1267,58 @@ mod owner_receipt_tests { ); } + #[test] + fn store_open_preserves_pending_spills_with_noncanonical_published_receipts() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = temp.path().join("graph.sealed"); + std::fs::create_dir(&root).unwrap(); + let previous_owner = row_spill_owner() ^ 1; + let canonical = root.join(format!("{ROW_SPILL_PREFIX}{previous_owner}-7")); + let owned = SpillDirectory::prepare(&canonical, previous_owner) + .unwrap() + .keep(); + + let forged_bound_names = [ + "unrelated".to_owned(), + format!("{ROW_SPILL_PREFIX}{previous_owner}-00"), + format!("{ROW_SPILL_PREFIX}{previous_owner}-not-a-counter"), + format!("{ROW_SPILL_PREFIX}{previous_owner}-18446744073709551616"), + format!("{ROW_SPILL_PREFIX}{}-7", previous_owner ^ 2), + ]; + let mut foreign = Vec::new(); + for (index, bound_name) in forged_bound_names.iter().enumerate() { + // Even a digest-consistent staging name is not ownership proof + // when its claimed published row-spill name is not canonical. + let receipt = format!("{RECEIPT_PREFIX}{previous_owner}:{bound_name}").into_bytes(); + let directory = root.join(staging_name(previous_owner, &receipt)); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join(ROW_SPILL_OWNER_RECEIPT_FILE), &receipt).unwrap(); + let sentinel = format!("foreign-{index}").into_bytes(); + std::fs::write(directory.join("sentinel"), &sentinel).unwrap(); + foreign.push((directory, receipt, sentinel)); + } + let unrelated = root.join("live-generation"); + std::fs::create_dir(&unrelated).unwrap(); + std::fs::write(unrelated.join("sentinel"), b"live").unwrap(); + + crate::sealed_store::sweep_abandoned_row_spills(&database); + + assert!( + !owned.exists(), + "a genuine interrupted pending spill must still be reclaimed" + ); + for (directory, receipt, sentinel) in foreign { + assert_eq!( + std::fs::read(directory.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(), + receipt, + "a forged receipt cannot authorize pending-spill cleanup" + ); + assert_eq!(std::fs::read(directory.join("sentinel")).unwrap(), sentinel); + } + assert_eq!(std::fs::read(unrelated.join("sentinel")).unwrap(), b"live"); + } + #[test] fn a_reused_pid_reclaims_the_prior_incarnation_and_preserves_live_spills() { let temp = tempfile::tempdir().unwrap(); From e60c8940c5ea215548f60275529dfdae1ad7295f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:59:08 -0800 Subject: [PATCH 25/70] test(storage): use genuine spill and seal ownership identities --- crates/tracedecay-graph-db/src/generation/spill.rs | 3 ++- .../tests/graph_db_suite/runtime_contract.rs | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index c62d9873ebb..8add23cd36d 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -1329,7 +1329,8 @@ mod owner_receipt_tests { let previous = current ^ 1; assert_eq!(current >> 64, previous >> 64, "same numeric process ID"); let final_path = root.join(format!(".rows-{current}-0")); - let stale = SpillDirectory::prepare(&final_path, previous) + let previous_path = root.join(format!(".rows-{previous}-0")); + let stale = SpillDirectory::prepare(&previous_path, previous) .unwrap() .keep(); let live = SpillDirectory::prepare(&final_path, current).unwrap(); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index c69dbf3222a..a34e05465fd 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -5,6 +5,7 @@ use std::time::Duration; use grafeo_common::types::Value; use tempfile::TempDir; +use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_graph_db::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphDbLeaseV1, GraphDbOwner, GraphEntity, GraphEntityId, GraphFormatVersion, GraphIdempotencyKey, GraphLabel, GraphMutation, @@ -1063,7 +1064,8 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() ) .unwrap(); raw.close().unwrap(); - let sealed_generation_name = "0".repeat(64); + // The on-disk name is the digest of the receipt's JSON identity tuple. + let sealed_generation_name = sha256_hex(br#"["project","code","g0"]"#); let sealed_generation = temp .path() .join("graph.sealed") @@ -1106,7 +1108,8 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() // A generation this build sealed before the staging container was // replaced, as a publication racing the rebuild leaves it. - let current_generation = temp.path().join("graph.sealed").join("1".repeat(64)); + let current_name = sha256_hex(br#"["project","code","g1"]"#); + let current_generation = temp.path().join("graph.sealed").join(¤t_name); std::fs::create_dir_all(¤t_generation).unwrap(); std::fs::write( current_generation.join("sealed.json"), @@ -1116,7 +1119,7 @@ fn superseded_format_store_rebuild_removes_proven_old_seals_but_keeps_unproven() "namespace": "project", "projection": "code", "generation": "g1", - "physical_namespace": format!("generation:{}", "1".repeat(64)), + "physical_namespace": format!("generation:{current_name}"), "recovered_digest": "sha256:1", "entities": 1, "relations": 0, From 6287274abb5f6ac46160d5c628e3962de6506654 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:32:46 -0800 Subject: [PATCH 26/70] fix(graph): reclaim crashed sealed stages with owner receipts Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 342 +++++++++++++++--- 1 file changed, 290 insertions(+), 52 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index fca6cb2ccc9..a95a32ffe2a 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -44,6 +44,7 @@ //! digest before the store serves a read. use std::collections::{BTreeMap, HashMap}; +use std::io::Write; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -228,6 +229,9 @@ fn read_sealed_receipt_bytes(path: &Path) -> std::io::Result> { pub(crate) const SEALED_STORE_DATABASE_FILE: &str = "generation.grafeo"; const SEALED_STORE_RECEIPT_FILE: &str = "sealed.json"; +/// Small, generation-bound deletion authority persisted before any staged rows. +/// Unlike sealed.json, this is never evidence that a container is complete. +const SEALED_STAGING_OWNER_FILE: &str = ".sealed-staging-owner-v1"; /// Digest recorded only after a successful post-reopen proof. A `sealed.json` /// written before that proof is not release authority. const SEALED_STORE_CHECKED_FILE: &str = "sealed.checked"; @@ -716,6 +720,103 @@ fn sealed_staging_name(name: &str) -> bool { }) } +fn initial_sealed_staging_owner( + identity: &GraphGenerationManifestIdentity, + expected: &GraphRecoveredGenerationDigestV1, + physical_namespace: &str, +) -> SealedStoreReceiptV1 { + // This receipt authorizes only abandoned-staging cleanup, never serving. + // The completed container receives a separate sealed.json after writing. + SealedStoreReceiptV1 { + version: SEALED_STORE_RECEIPT_VERSION, + form: SEALED_STORE_FORM_COMPACT.to_owned(), + namespace: identity.projection.namespace.as_str().to_owned(), + projection: identity.projection.projection.as_str().to_owned(), + generation: identity.generation.as_str().to_owned(), + physical_namespace: physical_namespace.to_owned(), + recovered_digest: expected.as_str().to_owned(), + entities: 0, + relations: 0, + graph_format: GraphFormatVersion::current().get(), + row_sum: None, + base: None, + source_generation: None, + watermark: None, + } +} + +fn verified_staging_receipt( + name: &std::ffi::OsStr, + bytes: &[u8], +) -> Option { + let name = name.to_str()?; + if !sealed_staging_name(name) { + return None; + } + let generation_hex = name.strip_prefix(".staging-")?; + let receipt = serde_json::from_slice::(bytes).ok()?; + let recognized_form = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + (recognized_form + && receipt.version == SEALED_STORE_RECEIPT_VERSION + && receipt.graph_format > 0 + && receipt.graph_format <= GraphFormatVersion::current().get() + && sealed_receipt_matches_generation_entry(&receipt, generation_hex) + && !receipt.recovered_digest.is_empty()) + .then_some(receipt) +} + +/// A failed builder must never recursively delete a substituted directory +/// merely because its pathname is the deterministic staging name. The early +/// owner marker covers unfinished builds; sealed.json covers the short window +/// after final receipt publication and before installation. +fn discard_failed_sealed_staging( + staging: &Path, + locator: &GenerationLocator, + expected_digest: &str, +) { + let Some((root, name)) = staging.parent().zip(staging.file_name()) else { + return; + }; + for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { + let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + std::ffi::OsStr::new(marker), + MAX_SEALED_RECEIPT_BYTES, + |bytes| { + verified_staging_receipt(name, bytes).is_some_and(|receipt| { + receipt.namespace == locator.projection.namespace.as_str() + && receipt.projection == locator.projection.projection.as_str() + && receipt.generation == locator.generation.as_str() + && receipt.recovered_digest == expected_digest + }) + }, + ); + match result { + Ok(_) => return, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + tracing::warn!( + event = "sealed_staging_cleanup_deferred", + path = %staging.display(), + %error, + "unverified staging preserved instead of deleting an unknown directory" + ); + return; + } + } + } + tracing::warn!( + event = "sealed_staging_cleanup_deferred", + path = %staging.display(), + "staging has no verified owner receipt; preserving unknown bytes" + ); +} + /// Reclaims interrupted seals only when the bounded receipt from the same /// opened staging directory proves its exact generation identity. The staging /// filename alone is not deletion authority: a missing, malformed, oversized, @@ -726,52 +827,33 @@ fn sealed_staging_name(name: &str) -> bool { /// so a pathname replacement cannot redirect deletion into a different tree. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - let reclaimed = tracedecay_private_fs::capability_dir::remove_receipted_directories_nofollow( - &root, - std::ffi::OsStr::new(SEALED_STORE_RECEIPT_FILE), - MAX_SEALED_RECEIPT_BYTES, - |name, bytes| { - let Some(name) = name.to_str() else { - return false; - }; - if !sealed_staging_name(name) { - return false; - } - let Some(generation_hex) = name.strip_prefix(".staging-") else { - return false; - }; - let Ok(receipt) = serde_json::from_slice::(bytes) else { - return false; - }; - // Reuse the same identity proof as obsolete-format retirement. - let recognized_form = match receipt.form.as_str() { - SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), - SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), - _ => false, - }; - recognized_form - && receipt.version == SEALED_STORE_RECEIPT_VERSION - && receipt.graph_format > 0 - && receipt.graph_format <= GraphFormatVersion::current().get() - && sealed_receipt_matches_generation_entry(&receipt, generation_hex) - && !receipt.recovered_digest.is_empty() - }, - ); - match reclaimed { - Ok(removed) if removed > 0 => tracing::info!( - event = "sealed_staging_swept", - root = %root.display(), - removed, - "removed receipt-verified staging from interrupted seals" - ), - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - root = %root.display(), - %error, - "unverified abandoned staging stays for inspection" - ), + // A crash during row copying has no completed sealed.json yet, but it + // does have the initial owner marker. A crash just before installation + // can be reclaimed using either marker. Never sweep an unreceipted name. + for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { + let reclaimed = tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( + &root, + std::ffi::OsStr::new(marker), + MAX_SEALED_RECEIPT_BYTES, + |name| name.to_str().is_some_and(sealed_staging_name), + |name, bytes| verified_staging_receipt(name, bytes).is_some(), + ); + match reclaimed { + Ok(removed) if removed > 0 => tracing::info!( + event = "sealed_staging_swept", + root = %root.display(), + removed, + "removed receipt-verified staging from interrupted seals" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned staging stays for inspection" + ), + } } } @@ -1963,19 +2045,46 @@ fn build_or_open_sealed_store( // create_dir is exclusive, so an occupied name fails closed. std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; + let owner = initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()); + let owner_bytes = serde_json::to_vec(&owner) + .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; + // Exclusive creation refuses a pre-existing marker. Flush before writing + // rows so a killed producer leaves verifiable deletion authority. + let marker_path = staging.join(SEALED_STAGING_OWNER_FILE); + let mut marker = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker_path) + .map_err(|error| sealed_store_io_failure("staging owner create failed", error))?; + marker + .write_all(&owner_bytes) + .and_then(|()| marker.sync_all()) + .map_err(|error| sealed_store_io_failure("staging owner persist failed", error))?; + drop(marker); let receipt = stage_sealed_artifact(rows, identity, expected, &staging, check) - .inspect_err(|_| remove_sealed_directory(&staging))?; + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; let encoded = serde_json::to_vec_pretty(&receipt) - .map_err(|error| GraphDbError::unavailable(format!("sealed receipt encode: {error}")))?; - std::fs::write(staging.join(SEALED_STORE_RECEIPT_FILE), encoded) - .inspect_err(|_| remove_sealed_directory(&staging)) - .map_err(|error| sealed_store_io_failure("receipt write failed", error))?; + .map_err(|error| GraphDbError::unavailable(format!("sealed receipt encode: {error}"))) + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; + let mut receipt_file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(staging.join(SEALED_STORE_RECEIPT_FILE)) + .map_err(|error| sealed_store_io_failure("receipt create failed", error)) + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; + let receipt_write = receipt_file + .write_all(&encoded) + .and_then(|()| receipt_file.sync_all()); + drop(receipt_file); + receipt_write + .map_err(|error| sealed_store_io_failure("receipt persist failed", error)) + .inspect_err(|_| discard_failed_sealed_staging(&staging, &locator, expected.as_str()))?; let artifact_name = directory .file_name() .ok_or_else(|| GraphDbError::unavailable("sealed artifact name missing"))?; let install = install_child_directory_noreplace_nofollow(&root, staging_name, artifact_name); if let Err(error) = install { - remove_sealed_directory(&staging); + discard_failed_sealed_staging(&staging, &locator, expected.as_str()); // A concurrent seal of the same generation may have installed the // directory first; adopting it below keeps this path idempotent. if !directory.exists() { @@ -3782,6 +3891,135 @@ mod build_tests { assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); } + /// Kill the real builder during row preparation, before sealed.json is + /// published. Its early receipt must let a later store-open sweep reclaim + /// the crashed stage without touching foreign staging data. + #[test] + fn crashed_seal_before_final_receipt_is_reclaimed_and_can_retry() { + const CHILD_ROOT: &str = "TRACEDECAY_TEST_CRASHED_SEAL_ROOT"; + const CRASHED: i32 = 43; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let database_path = std::path::PathBuf::from(root).join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let exit_during_copy = || -> Result<(), GraphDbError> { + std::process::exit(CRASHED); + }; + let _ = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &exit_during_copy, + ); + panic!("the row stream did not invoke its cancellation callback"); + } + + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("--exact") + .arg(std::thread::current().name().unwrap()) + .env(CHILD_ROOT, temp.path()) + .output() + .unwrap(); + assert_eq!(child.status.code(), Some(CRASHED), "{child:?}"); + assert!(staging.join(SEALED_STAGING_OWNER_FILE).is_file()); + assert!(!staging.join(SEALED_STORE_RECEIPT_FILE).exists()); + + let foreign = root.join(format!(".staging-{}", "0".repeat(64))); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sentinel"), b"foreign").unwrap(); + sweep_abandoned_sealed_staging(&database_path); + assert!( + !staging.exists(), + "the crashed but proven stage must be reclaimed" + ); + assert_eq!( + std::fs::read(foreign.join("sentinel")).unwrap(), + b"foreign" + ); + + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + } + + /// The failed-builder cleanup must not delete a foreign replacement of a + /// staging pathname after the owner marker was written. + #[cfg(unix)] + #[test] + fn interrupted_seal_preserves_a_substituted_staging_directory() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(24, 12); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let moved = root.join("original-stage"); + let swapped = std::cell::Cell::new(false); + let interrupt = || { + if !swapped.replace(true) { + std::fs::rename(&staging, &moved).unwrap(); + std::fs::create_dir(&staging).unwrap(); + std::fs::write(staging.join("sentinel"), b"foreign replacement").unwrap(); + } + Err(GraphDbError::Cancelled) + }; + let result = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &interrupt, + ); + assert!( + matches!(&result, Err(GraphDbError::Cancelled)), + "{result:?}" + ); + assert_eq!( + std::fs::read(staging.join("sentinel")).unwrap(), + b"foreign replacement", + "cleanup must not erase the substituted directory" + ); + assert!(moved.join(SEALED_STAGING_OWNER_FILE).is_file()); + assert!(!directory.exists()); + + std::fs::remove_dir_all(&staging).unwrap(); + std::fs::remove_dir_all(&moved).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + assert_eq!(store.recovered_digest(), expected.as_str()); + } + /// The sealed container is written once, complete, after every row has /// been encoded: no engine, WAL, or partial container exists under the /// staging directory while rows stream. A build cancelled mid-copy leaves From 76ac5bd5145f7847c76d40276a46ee160a9d5b7c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:34:34 -0800 Subject: [PATCH 27/70] test(graph): import sealed staging recovery helpers Refs #4616 --- crates/tracedecay-graph-db/src/sealed_store.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a95a32ffe2a..8f908e65603 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3413,9 +3413,10 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, - build_or_open_sealed_store, build_sealed_container, sealed_artifact_database_options, - sealed_copy_proof, sealed_generation_directory, sealed_store_root, + SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, + SealedRowSource, build_or_open_sealed_store, build_sealed_container, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ From 95a66960f8cd451385b1e25f049b76d2992451fb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 02:03:47 -0800 Subject: [PATCH 28/70] fix(graph): preserve live sealed staging across remounts Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 178 +++++++++++------- 1 file changed, 115 insertions(+), 63 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 8f908e65603..3b221f75fe5 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -277,6 +277,14 @@ struct SealedStoreReceiptV1 { watermark: Option, } +/// An incomplete sealed container belongs to one producer incarnation. +/// A remount in that process must not reclaim its in-flight row stream. +#[derive(Debug, Deserialize, Serialize)] +struct SealedStagingOwnerV1 { + producer: String, + receipt: SealedStoreReceiptV1, +} + impl SealedStoreReceiptV1 { fn binds( &self, @@ -748,13 +756,18 @@ fn initial_sealed_staging_owner( fn verified_staging_receipt( name: &std::ffi::OsStr, bytes: &[u8], -) -> Option { +) -> Option { let name = name.to_str()?; if !sealed_staging_name(name) { return None; } let generation_hex = name.strip_prefix(".staging-")?; - let receipt = serde_json::from_slice::(bytes).ok()?; + let owner = serde_json::from_slice::(bytes).ok()?; + let instance = owner.producer.parse::().ok()?; + if instance.to_string() != owner.producer { + return None; + } + let receipt = &owner.receipt; let recognized_form = match receipt.form.as_str() { SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), @@ -764,15 +777,15 @@ fn verified_staging_receipt( && receipt.version == SEALED_STORE_RECEIPT_VERSION && receipt.graph_format > 0 && receipt.graph_format <= GraphFormatVersion::current().get() - && sealed_receipt_matches_generation_entry(&receipt, generation_hex) + && sealed_receipt_matches_generation_entry(receipt, generation_hex) && !receipt.recovered_digest.is_empty()) - .then_some(receipt) + .then_some(owner) } /// A failed builder must never recursively delete a substituted directory -/// merely because its pathname is the deterministic staging name. The early -/// owner marker covers unfinished builds; sealed.json covers the short window -/// after final receipt publication and before installation. +/// merely because its pathname is the deterministic staging name. Only the +/// producer-bound early marker authorizes cleanup, including after sealed.json +/// publication and before installation. fn discard_failed_sealed_staging( staging: &Path, locator: &GenerationLocator, @@ -781,40 +794,30 @@ fn discard_failed_sealed_staging( let Some((root, name)) = staging.parent().zip(staging.file_name()) else { return; }; - for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { - let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( - root, - name, - std::ffi::OsStr::new(marker), - MAX_SEALED_RECEIPT_BYTES, - |bytes| { - verified_staging_receipt(name, bytes).is_some_and(|receipt| { - receipt.namespace == locator.projection.namespace.as_str() - && receipt.projection == locator.projection.projection.as_str() - && receipt.generation == locator.generation.as_str() - && receipt.recovered_digest == expected_digest - }) - }, + let result = tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + name, + std::ffi::OsStr::new(SEALED_STAGING_OWNER_FILE), + MAX_SEALED_RECEIPT_BYTES, + |bytes| { + verified_staging_receipt(name, bytes).is_some_and(|owner| { + let receipt = &owner.receipt; + owner.producer == row_spill_owner().to_string() + && receipt.namespace == locator.projection.namespace.as_str() + && receipt.projection == locator.projection.projection.as_str() + && receipt.generation == locator.generation.as_str() + && receipt.recovered_digest == expected_digest + }) + }, + ); + if !matches!(&result, Ok(true)) { + tracing::warn!( + event = "sealed_staging_cleanup_deferred", + path = %staging.display(), + ?result, + "unverified staging preserved instead of deleting an unknown directory" ); - match result { - Ok(_) => return, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - tracing::warn!( - event = "sealed_staging_cleanup_deferred", - path = %staging.display(), - %error, - "unverified staging preserved instead of deleting an unknown directory" - ); - return; - } - } } - tracing::warn!( - event = "sealed_staging_cleanup_deferred", - path = %staging.display(), - "staging has no verified owner receipt; preserving unknown bytes" - ); } /// Reclaims interrupted seals only when the bounded receipt from the same @@ -827,33 +830,35 @@ fn discard_failed_sealed_staging( /// so a pathname replacement cannot redirect deletion into a different tree. pub(crate) fn sweep_abandoned_sealed_staging(database_path: &Path) { let root = sealed_store_root(database_path); - // A crash during row copying has no completed sealed.json yet, but it - // does have the initial owner marker. A crash just before installation - // can be reclaimed using either marker. Never sweep an unreceipted name. - for marker in [SEALED_STAGING_OWNER_FILE, SEALED_STORE_RECEIPT_FILE] { - let reclaimed = tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( + // Every new sealed build writes this marker before rows and retains it + // through installation. The final sealed.json is not a separate cleanup + // authority: it cannot identify whether a concurrent build is still live. + let reclaimed = + tracedecay_private_fs::capability_dir::remove_matching_receipted_directories_nofollow( &root, - std::ffi::OsStr::new(marker), + std::ffi::OsStr::new(SEALED_STAGING_OWNER_FILE), MAX_SEALED_RECEIPT_BYTES, |name| name.to_str().is_some_and(sealed_staging_name), - |name, bytes| verified_staging_receipt(name, bytes).is_some(), + |name, bytes| { + verified_staging_receipt(name, bytes) + .is_some_and(|owner| owner.producer != row_spill_owner().to_string()) + }, ); - match reclaimed { - Ok(removed) if removed > 0 => tracing::info!( - event = "sealed_staging_swept", - root = %root.display(), - removed, - "removed receipt-verified staging from interrupted seals" - ), - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!( - event = "sealed_staging_sweep_failed", - root = %root.display(), - %error, - "unverified abandoned staging stays for inspection" - ), - } + match reclaimed { + Ok(removed) if removed > 0 => tracing::info!( + event = "sealed_staging_swept", + root = %root.display(), + removed, + "removed receipt-verified staging from exited producers" + ), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!( + event = "sealed_staging_sweep_failed", + root = %root.display(), + %error, + "unverified abandoned staging stays for inspection" + ), } } @@ -2045,7 +2050,10 @@ fn build_or_open_sealed_store( // create_dir is exclusive, so an occupied name fails closed. std::fs::create_dir(&staging) .map_err(|error| sealed_store_io_failure("staging directory create failed", error))?; - let owner = initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()); + let owner = SealedStagingOwnerV1 { + producer: row_spill_owner().to_string(), + receipt: initial_sealed_staging_owner(identity, expected, physical_namespace.as_str()), + }; let owner_bytes = serde_json::to_vec(&owner) .map_err(|error| GraphDbError::unavailable(format!("staging owner encode: {error}")))?; // Exclusive creation refuses a pre-existing marker. Flush before writing @@ -3892,6 +3900,50 @@ mod build_tests { assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); } + /// A store remount invokes the real startup sweep while a producer is + /// still streaming rows. The sweep must preserve its owner-bound staging + /// directory, or the builder can no longer install the verified artifact. + #[test] + fn remount_preserves_live_sealed_staging_and_seal_completes() { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(48, 24); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let root = sealed_store_root(&database_path); + let directory = + sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let staging = root.join(format!( + ".staging-{}", + directory.file_name().unwrap().to_str().unwrap() + )); + let remounted = std::cell::Cell::new(false); + let check = || { + if !remounted.replace(true) { + assert!(staging.join(SEALED_STAGING_OWNER_FILE).is_file()); + let remount = open_source(&database_path); + drop(remount); + assert!( + staging.join(SEALED_STAGING_OWNER_FILE).is_file(), + "remount must not reclaim a live producer's staged container" + ); + } + Ok(()) + }; + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + &check, + ) + .expect("live seal must finish after a store remount"); + assert!(remounted.get(), "remount must run inside the row-copy callback"); + assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); + assert_eq!(store.recovered_digest(), expected.as_str()); + assert!(!staging.exists()); + } + /// Kill the real builder during row preparation, before sealed.json is /// published. Its early receipt must let a later store-open sweep reclaim /// the crashed stage without touching foreign staging data. From 8d97999f422480e7dc6baefecfc71c3544290fbe Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:07:12 -0800 Subject: [PATCH 29/70] fix(graph): reject unowned receipts in sealed census Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 114 ++++++++++++++++-- 1 file changed, 102 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 3b221f75fe5..84327defcf7 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -591,13 +591,16 @@ pub fn census_sealed_store( continue; } let path = entry.path(); - let bytes = directory_bytes(&path, &mut counted)?; let name = entry.file_name(); if name.to_str().is_some_and(sealed_staging_name) { + let bytes = directory_bytes(&path, &mut counted)?; census.abandoned_staging_count += 1; census.abandoned_staging_bytes += bytes; continue; } + // Receipt existence is not sufficient: a copied receipt can name a + // verified head while residing in an unrelated physical directory. + // Classify unproven entries before walking or charging their contents. let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) .ok() .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); @@ -605,12 +608,32 @@ pub fn census_sealed_store( census.unrecognized_count += 1; continue; }; + let Some(name) = name.to_str() else { + census.unrecognized_count += 1; + continue; + }; + let form_valid = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + if !form_valid + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.graph_format == 0 + || receipt.graph_format > GraphFormatVersion::current().get() + || receipt.recovered_digest.is_empty() + || !sealed_receipt_matches_generation_entry(&receipt, name) + { + census.unrecognized_count += 1; + continue; + } + let bytes = directory_bytes(&path, &mut counted)?; if head_generations.contains(&receipt.generation) { census.head_count += 1; - census.head_bytes += bytes; + census.head_bytes = census.head_bytes.saturating_add(bytes); } else { census.superseded_count += 1; - census.superseded_bytes += bytes; + census.superseded_bytes = census.superseded_bytes.saturating_add(bytes); } } Ok(census) @@ -3370,17 +3393,26 @@ mod receipt_budget_tests { let root = sealed_store_root(&database); std::fs::create_dir_all(&root).unwrap(); const FILE_BYTES: u64 = 220 * 1024 * 1024; + let mut oldest = None; for index in 0..70 { - let name = format!("{index:064x}"); - let dir = root.join(&name); + let generation = format!("generation:g{index}"); + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new(generation.clone()).unwrap(), + ); + let physical = locator.physical_namespace().unwrap(); + let name = physical.as_str().strip_prefix("generation:").unwrap(); + let dir = root.join(name); + if index == 0 { + oldest = Some(dir.clone()); + } std::fs::create_dir_all(&dir).unwrap(); std::fs::write( dir.join(SEALED_STORE_RECEIPT_FILE), - receipt( - GraphFormatVersion::current().get(), - &name, - &format!("g{index}"), - ), + receipt(GraphFormatVersion::current().get(), name, &generation), ) .unwrap(); std::fs::File::create(dir.join(SEALED_STORE_DATABASE_FILE)) @@ -3396,10 +3428,14 @@ mod receipt_budget_tests { .unwrap() .set_len(20 * 1024 * 1024 * 1024) .unwrap(); - let oldest = root.join("0".repeat(64)); + let oldest = oldest.unwrap(); std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); - let census = census_sealed_store(&database, &BTreeSet::from(["g69".to_owned()])).unwrap(); + let census = census_sealed_store( + &database, + &BTreeSet::from(["generation:g69".to_owned()]), + ) + .unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 69); assert_eq!(census.unrecognized_count, 0); @@ -3410,6 +3446,60 @@ mod receipt_budget_tests { "foreign link targets must not contribute to sealed generation bytes" ); } + + #[test] + fn census_does_not_classify_copied_or_foreign_receipts_as_live_heads() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("graph.grafeo"); + let root = sealed_store_root(&database); + std::fs::create_dir_all(&root).unwrap(); + let generation = "generation:head"; + let locator = GenerationLocator::new( + GraphProjectionIdentity::new( + GraphNamespace::new("test").unwrap(), + GraphProjectionId::new("code").unwrap(), + ), + crate::GraphGenerationId::new(generation).unwrap(), + ); + let physical = locator.physical_namespace().unwrap(); + let owned_name = physical.as_str().strip_prefix("generation:").unwrap(); + let owned = root.join(owned_name); + std::fs::create_dir(&owned).unwrap(); + let owned_receipt = receipt( + GraphFormatVersion::current().get(), + owned_name, + generation, + ); + std::fs::write(owned.join(SEALED_STORE_RECEIPT_FILE), &owned_receipt).unwrap(); + std::fs::write(owned.join(SEALED_STORE_DATABASE_FILE), b"owned payload").unwrap(); + + // The foreign directory carries the *exact* valid head receipt, but + // its physical entry is not the one derived from the logical identity. + let foreign_name = if owned_name == "f".repeat(64) { + "e".repeat(64) + } else { + "f".repeat(64) + }; + let foreign = root.join(foreign_name); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join(SEALED_STORE_RECEIPT_FILE), &owned_receipt).unwrap(); + std::fs::File::create(foreign.join(SEALED_STORE_DATABASE_FILE)) + .unwrap() + .set_len(5 * 1024 * 1024 * 1024) + .unwrap(); + + let census = census_sealed_store( + &database, + &BTreeSet::from([generation.to_owned()]), + ) + .unwrap(); + assert_eq!(census.head_count, 1); + assert_eq!(census.superseded_count, 0); + assert_eq!(census.unrecognized_count, 1); + assert_eq!(census.head_bytes, owned_receipt.len() as u64 + 13); + assert!(foreign.join(SEALED_STORE_DATABASE_FILE).exists()); + } + } #[cfg(test)] From c87aa2fecbbbe6d10cbfb31f8dc250c9a6317d41 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:08:29 -0800 Subject: [PATCH 30/70] test(graph): keep sparse census regression Unix-only Refs #4616 --- crates/tracedecay-graph-db/src/sealed_store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 84327defcf7..d8de2bbacfe 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3447,6 +3447,7 @@ mod receipt_budget_tests { ); } + #[cfg(unix)] #[test] fn census_does_not_classify_copied_or_foreign_receipts_as_live_heads() { let temp = tempfile::tempdir().unwrap(); @@ -3499,7 +3500,6 @@ mod receipt_budget_tests { assert_eq!(census.head_bytes, owned_receipt.len() as u64 + 13); assert!(foreign.join(SEALED_STORE_DATABASE_FILE).exists()); } - } #[cfg(test)] From 54b3b0a1aded24fcd5b404780d70455114ef33a7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 04:16:06 -0800 Subject: [PATCH 31/70] fix(graph): exclude unproven staging from sealed census Refs #4616 Only a receipt-verified staging directory from an exited producer may count as abandoned. Preserve unknown, copied, oversized and current-producer staging without traversing its contents. Cover positive and negative census behavior with native Rust regressions. --- .../tracedecay-graph-db/src/sealed_store.rs | 104 ++++++++++++++++-- .../sealed_store.rs | 15 ++- 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index d8de2bbacfe..8127870b838 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -556,11 +556,12 @@ pub struct SealedStoreCensusV1 { /// retirement has not run since the last publication. pub superseded_count: u64, pub superseded_bytes: u64, - /// `.staging-*` directories left by seals that never installed. + /// Receipt-verified `.staging-*` directories belonging to earlier producer + /// incarnations; current or unproven staging cannot be called abandoned. pub abandoned_staging_count: u64, pub abandoned_staging_bytes: u64, - /// Entries that are neither a readable sealed receipt nor staging: an - /// unrecognized layout is reported, never silently classed as dead. + /// Unrecognized or unproven entries (including live staging) are reported + /// without walking their contents or claiming that they can be reclaimed. pub unrecognized_count: u64, } @@ -593,9 +594,25 @@ pub fn census_sealed_store( let path = entry.path(); let name = entry.file_name(); if name.to_str().is_some_and(sealed_staging_name) { + // A matching filename is not proof that the directory is + // abandoned: remounts can overlap a seal in this process, and + // foreign or interrupted pre-receipt entries may occupy its name. + // Do not walk unverified trees (which may be arbitrarily large). + let verified = tracedecay_private_fs::framed_log::read_bounded( + &path.join(SEALED_STAGING_OWNER_FILE), + MAX_SEALED_RECEIPT_BYTES, + ) + .ok() + .flatten() + .and_then(|bytes| verified_staging_receipt(&name, &bytes)) + .is_some_and(|owner| owner.producer != row_spill_owner().to_string()); + if !verified { + census.unrecognized_count += 1; + continue; + } let bytes = directory_bytes(&path, &mut counted)?; census.abandoned_staging_count += 1; - census.abandoned_staging_bytes += bytes; + census.abandoned_staging_bytes = census.abandoned_staging_bytes.saturating_add(bytes); continue; } // Receipt existence is not sufficient: a copied receipt can name a @@ -3511,10 +3528,11 @@ mod build_tests { use rayon::ThreadPoolBuilder; use super::{ - SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, - SealedRowSource, build_or_open_sealed_store, build_sealed_container, - sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, - sealed_store_root, sweep_abandoned_sealed_staging, + MAX_SEALED_RECEIPT_BYTES, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, + SEALED_STORE_RECEIPT_FILE, SealedRowSource, SealedStagingOwnerV1, + build_or_open_sealed_store, build_sealed_container, census_sealed_store, + initial_sealed_staging_owner, sealed_artifact_database_options, sealed_copy_proof, + sealed_generation_directory, sealed_store_root, sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ @@ -3582,6 +3600,76 @@ mod build_tests { .unwrap() } + #[cfg(unix)] + #[test] + fn sealed_census_only_attributes_verified_abandoned_staging() { + let temp = tempfile::tempdir().unwrap(); + let database = temp.path().join("source.grafeo"); + let root = sealed_store_root(&database); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let physical = identity.physical_namespace().unwrap(); + let name = physical.as_str().strip_prefix("generation:").unwrap(); + let staging = root.join(format!(".staging-{name}")); + std::fs::create_dir_all(&staging).unwrap(); + + let exited = SealedStagingOwnerV1 { + producer: (crate::generation::row_spill_owner() ^ 1).to_string(), + receipt: initial_sealed_staging_owner(&identity, &expected, physical.as_str()), + }; + let bytes = serde_json::to_vec(&exited).unwrap(); + let owner_path = staging.join(SEALED_STAGING_OWNER_FILE); + std::fs::write(&owner_path, &bytes).unwrap(); + std::fs::write(staging.join("payload"), vec![7; 4096]).unwrap(); + + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 1); + assert_eq!(census.abandoned_staging_bytes, bytes.len() as u64 + 4096); + assert_eq!(census.unrecognized_count, 0); + + // This same completed marker may belong to a seal currently running + // in this process. A remount must not report its bytes as garbage. + let live = SealedStagingOwnerV1 { + producer: crate::generation::row_spill_owner().to_string(), + receipt: initial_sealed_staging_owner(&identity, &expected, physical.as_str()), + }; + std::fs::write(&owner_path, serde_json::to_vec(&live).unwrap()).unwrap(); + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.abandoned_staging_bytes, 0); + assert_eq!(census.unrecognized_count, 1); + + // A copied authentic receipt in a different physical staging name + // does not authorize either reclamation or an unbounded size walk. + let foreign_name = if name == "f".repeat(64) { + "e".repeat(64) + } else { + "f".repeat(64) + }; + let foreign = root.join(format!(".staging-{foreign_name}")); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join(SEALED_STAGING_OWNER_FILE), &bytes).unwrap(); + std::fs::File::create(foreign.join("sparse-foreign-payload")) + .unwrap() + .set_len(5 * 1024 * 1024 * 1024) + .unwrap(); + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.abandoned_staging_bytes, 0); + assert_eq!(census.unrecognized_count, 2); + assert!(foreign.join("sparse-foreign-payload").is_file()); + + // Even an owned directory whose marker exceeds the fixed bound must + // not cause its contents to be traversed or counted as abandoned. + let oversized = vec![b'x'; MAX_SEALED_RECEIPT_BYTES + 1]; + std::fs::write(&owner_path, &oversized).unwrap(); + let census = census_sealed_store(&database, &BTreeSet::new()).unwrap(); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.unrecognized_count, 2); + assert_eq!(std::fs::read(&owner_path).unwrap(), oversized); + } + fn open_source(database_path: &std::path::Path) -> crate::GraphDbLeaseV1 { let owner = GraphDbOwner::open(GraphDbOpenOptions { location: GraphDbLocation::Persistent(database_path.to_path_buf()), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 1fd3f0ada90..e0371137939 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1389,12 +1389,11 @@ fn reopening_the_store_reclaims_only_receipted_staging() { assert_snapshot_reads(&snapshot, &identity, "durable"); } -/// The sealed census splits the sealed root into what serves and what does -/// not: the head's artifact, a superseded generation's artifact, and staging -/// an interrupted seal left behind are each counted where they belong, and a -/// directory without a readable receipt is reported rather than classed. +/// The sealed census splits the sealed root into verified head and +/// superseded artifacts. Unreceipted staging and unrelated directories remain +/// unrecognized rather than falsely counted as reclaimable garbage. #[test] -fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { +fn sealed_census_separates_heads_and_preserves_unproven_staging() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let mut authority = RelationalAuthority::default(); @@ -1450,9 +1449,9 @@ fn sealed_census_separates_heads_from_superseded_and_abandoned_bytes() { census.superseded_count, 1, "g1 is sealed but no longer the head" ); - assert_eq!(census.abandoned_staging_count, 1); - assert_eq!(census.abandoned_staging_bytes, 4096); - assert_eq!(census.unrecognized_count, 1); + assert_eq!(census.abandoned_staging_count, 0); + assert_eq!(census.abandoned_staging_bytes, 0); + assert_eq!(census.unrecognized_count, 2); assert!(census.head_bytes > 0 && census.superseded_bytes > 0); // With no heads journaled every sealed artifact reads as superseded, and From f192a01ab3efb448275c2d10fbf25212b6101901 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:54:29 -0800 Subject: [PATCH 32/70] style(graph): format sealed census and recovery regressions --- .../tracedecay-graph-db/src/sealed_store.rs | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 8127870b838..6f6ecc4af64 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -793,10 +793,7 @@ fn initial_sealed_staging_owner( } } -fn verified_staging_receipt( - name: &std::ffi::OsStr, - bytes: &[u8], -) -> Option { +fn verified_staging_receipt(name: &std::ffi::OsStr, bytes: &[u8]) -> Option { let name = name.to_str()?; if !sealed_staging_name(name) { return None; @@ -3448,11 +3445,8 @@ mod receipt_budget_tests { let oldest = oldest.unwrap(); std::os::unix::fs::symlink(&foreign, oldest.join("foreign")).unwrap(); std::os::unix::fs::symlink(temp.path().join("missing"), oldest.join("dangling")).unwrap(); - let census = census_sealed_store( - &database, - &BTreeSet::from(["generation:g69".to_owned()]), - ) - .unwrap(); + let census = + census_sealed_store(&database, &BTreeSet::from(["generation:g69".to_owned()])).unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 69); assert_eq!(census.unrecognized_count, 0); @@ -3483,11 +3477,7 @@ mod receipt_budget_tests { let owned_name = physical.as_str().strip_prefix("generation:").unwrap(); let owned = root.join(owned_name); std::fs::create_dir(&owned).unwrap(); - let owned_receipt = receipt( - GraphFormatVersion::current().get(), - owned_name, - generation, - ); + let owned_receipt = receipt(GraphFormatVersion::current().get(), owned_name, generation); std::fs::write(owned.join(SEALED_STORE_RECEIPT_FILE), &owned_receipt).unwrap(); std::fs::write(owned.join(SEALED_STORE_DATABASE_FILE), b"owned payload").unwrap(); @@ -3506,11 +3496,8 @@ mod receipt_budget_tests { .set_len(5 * 1024 * 1024 * 1024) .unwrap(); - let census = census_sealed_store( - &database, - &BTreeSet::from([generation.to_owned()]), - ) - .unwrap(); + let census = + census_sealed_store(&database, &BTreeSet::from([generation.to_owned()])).unwrap(); assert_eq!(census.head_count, 1); assert_eq!(census.superseded_count, 0); assert_eq!(census.unrecognized_count, 1); @@ -4089,8 +4076,7 @@ mod build_tests { let identity = manifest.identity(); let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); let root = sealed_store_root(&database_path); - let directory = - sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); + let directory = sealed_generation_directory(&root, &identity.physical_namespace().unwrap()); let staging = root.join(format!( ".staging-{}", directory.file_name().unwrap().to_str().unwrap() @@ -4116,7 +4102,10 @@ mod build_tests { &check, ) .expect("live seal must finish after a store remount"); - assert!(remounted.get(), "remount must run inside the row-copy callback"); + assert!( + remounted.get(), + "remount must run inside the row-copy callback" + ); assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); assert_eq!(store.recovered_digest(), expected.as_str()); assert!(!staging.exists()); @@ -4176,10 +4165,7 @@ mod build_tests { !staging.exists(), "the crashed but proven stage must be reclaimed" ); - assert_eq!( - std::fs::read(foreign.join("sentinel")).unwrap(), - b"foreign" - ); + assert_eq!(std::fs::read(foreign.join("sentinel")).unwrap(), b"foreign"); let (store, _) = build_or_open_sealed_store( SealedRowSource::Manifest(&manifest), From e006c620df35ce6c4048f5f16ff28034467ec7af Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:56:58 -0800 Subject: [PATCH 33/70] test(graph): retire five sealed replay generations --- .../sealed_store.rs | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index e0371137939..94e656c34db 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1603,6 +1603,116 @@ fn retirement_deletes_the_superseded_sealed_artifact() { ); } +/// A real sequence of five published sealed replay generations must retire +/// all four superseded artifacts independently, not merely the oldest one. +/// Each retirement goes through the same production registry API used by +/// code-generation retention, and the newest reader stays usable throughout. +#[test] +fn five_sealed_replay_generations_retire_to_the_live_head() { + let temp = TempDir::new().unwrap(); + let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); + let mut authority = RelationalAuthority::default(); + let identity = projection("sealed-store:five-generation-retire", "code"); + let mut prior_head = None; + let mut superseded = Vec::new(); + let mut serving = None; + + for (ordinal, digit) in (1..=5).zip(['1', '2', '3', '4', '5']) { + let generation = format!("five-retire-g{ordinal}"); + let manifest = rich_manifest( + identity.clone(), + &generation, + &format!("marker-{ordinal}"), + ); + let record = stage_sealed_manifest( + &mut authority, + ®istered.binding, + &manifest, + &format!("publish:{generation}"), + prior_head.take(), + digit, + ); + let commit = publish_sealed( + ®istered, + temp.path(), + &mut authority, + &record, + &manifest, + ); + prior_head = Some(commit.head.clone()); + assert!( + receipt_for_generation(temp.path(), &generation).is_some(), + "each published generation must have its own sealed artifact" + ); + if ordinal == 5 { + serving = Some(commit); + } else { + superseded.push(( + generation, + manifest.identity(), + CodeGenerationId::new(format!( + "code-generation.{}", + manifest.generation.as_str() + )) + .unwrap(), + SealedGraphStateDigest::try_from(format!( + "sha256:{}", + digit.to_string().repeat(64) + )) + .unwrap(), + )); + drop(commit); + } + } + + assert_eq!(superseded.len(), 4); + assert_eq!( + sealed_receipts(temp.path()).len(), + 5, + "five distinct generations must exist before retirement" + ); + let serving = serving.unwrap(); + assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); + + let database = registered + .registry + .resolve(registration(registered.binding.clone(), temp.path())) + .unwrap(); + let (control, probe) = control_and_probe(); + let context = GraphPublicationOperationContextV1::new(&control, &probe).unwrap(); + for (generation, manifest_identity, code_generation, sealed_digest) in superseded { + assert!(matches!( + registered + .registry + .retire_one_code_generation_replay( + registration(registered.binding.clone(), temp.path()), + &mut authority, + &context, + &code_generation, + &sealed_digest, + ) + .unwrap(), + GraphReplayCollectionOutcome::Retired(_) + )); + assert!( + receipt_for_generation(temp.path(), &generation).is_none(), + "retired generation {generation} must be absent from disk" + ); + assert!( + !database.is_generation_sealed_only(&manifest_identity).unwrap(), + "retired generation {generation} must leave sealed-only state" + ); + assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); + } + assert_eq!( + sealed_receipts(temp.path()).len(), + 1, + "only the live sealed generation should remain" + ); + assert!(receipt_for_generation(temp.path(), "five-retire-g5").is_some()); + assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); +} + /// A live direct-sealed reader (recovered through /// `recover_verified_sealed_snapshot`, which bypasses the staging database's /// verified-generation state) must gate retirement of its generation exactly From 79c53bb732139d98aadd3a1c9188794faac34f04 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:57:44 -0800 Subject: [PATCH 34/70] fix(graph): anchor sealed proof marker writes to held directories Refs #4616. Publish sealed.checked through an exclusively staged sibling on a held no-follow generation capability. Refuse linked generation directories and verify foreign marker bytes survive a directory swap. Partial safety repair; automatic retention acceptance remains outstanding. --- .../tracedecay-graph-db/src/sealed_store.rs | 44 +++++++-- .../src/capability_dir.rs | 96 ++++++++++++++++++- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 6f6ecc4af64..1e50ccf204c 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -2789,11 +2789,10 @@ fn persist_sealed_store_check(directory: &Path, expected_digest: &str) -> Result } // Never truncate an existing symlink target: the marker is a rebuildable // cache, so stage a regular file and publish it by atomic replacement. - tracedecay_private_fs::framed_log::atomic_write( - &directory.join(SEALED_STORE_CHECKED_FILE), - "sealed-check", + tracedecay_private_fs::capability_dir::atomic_replace_child_file_nofollow( + directory, + std::ffi::OsStr::new(SEALED_STORE_CHECKED_FILE), expected_digest.as_bytes(), - tracedecay_private_fs::framed_log::DirectorySyncPolicy::TolerateUnsupported, ) .map_err(|error| sealed_store_io_failure("sealed check persist failed", error)) } @@ -3518,8 +3517,9 @@ mod build_tests { MAX_SEALED_RECEIPT_BYTES, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, SealedStagingOwnerV1, build_or_open_sealed_store, build_sealed_container, census_sealed_store, - initial_sealed_staging_owner, sealed_artifact_database_options, sealed_copy_proof, - sealed_generation_directory, sealed_store_root, sweep_abandoned_sealed_staging, + initial_sealed_staging_owner, persist_sealed_store_check, sealed_artifact_database_options, + sealed_copy_proof, sealed_generation_directory, sealed_store_root, + sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ @@ -3530,6 +3530,38 @@ mod build_tests { GraphRelationId, GraphRelationKind, GraphWatermark, NeverCancelled, SourceGeneration, }; + #[cfg(unix)] + #[test] + fn sealed_check_persistence_refuses_a_substituted_generation_directory() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned-generation"); + let foreign = temp.path().join("foreign-generation"); + std::fs::create_dir(&owned).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sealed.checked"), b"foreign marker").unwrap(); + std::fs::remove_dir(&owned).unwrap(); + symlink(&foreign, &owned).unwrap(); + + assert!(persist_sealed_store_check(&owned, "accepted-digest").is_err()); + assert_eq!( + std::fs::read(foreign.join("sealed.checked")).unwrap(), + b"foreign marker", + ); + std::fs::remove_file(&owned).unwrap(); + std::fs::create_dir(&owned).unwrap(); + persist_sealed_store_check(&owned, "accepted-digest").unwrap(); + assert_eq!( + std::fs::read(owned.join("sealed.checked")).unwrap(), + b"accepted-digest", + ); + assert_eq!( + std::fs::read(foreign.join("sealed.checked")).unwrap(), + b"foreign marker", + ); + } + fn entity_identity(index: usize) -> GraphEntityId { GraphEntityId::new(format!("symbol:{index:05}")).unwrap() } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 72b9eda88b3..1da1852bec3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -7,8 +7,9 @@ #[cfg(unix)] use std::ffi::CString; -use std::ffi::OsStr; -use std::io; +use std::ffi::{OsStr, OsString}; +use std::io::{self, Write}; +use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(unix)] use std::os::fd::{AsRawFd, FromRawFd}; #[cfg(unix)] @@ -227,6 +228,61 @@ fn single_component(name: &OsStr) -> io::Result<()> { Ok(()) } +/// Publish a rebuildable marker through one held directory. Replacing the +/// pathname after acquisition cannot redirect its bytes into a foreign tree. +pub fn atomic_replace_child_file_nofollow( + directory_path: &Path, + filename: &OsStr, + contents: &[u8], +) -> io::Result<()> { + single_component(filename)?; + let directory = open_root_nofollow(directory_path)?; + atomic_replace_file_in_dir(&directory, filename, contents) +} + +fn atomic_replace_file_in_dir( + directory: &Dir, + filename: &OsStr, + contents: &[u8], +) -> io::Result<()> { + static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); + let (temporary, mut file) = (0..8) + .find_map(|_| { + let sequence = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); + let temporary = OsString::from(format!( + ".sealed-proof-{}-{:016x}-{sequence:016x}.tmp", + std::process::id(), + crate::process_instance_nonce(), + )); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + match directory.open_with(&temporary, &options) { + Ok(file) => Some(Ok((temporary, file))), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => None, + Err(error) => Some(Err(error)), + } + }) + .transpose()? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::AlreadyExists, + "could not exclusively stage a sealed proof marker", + ) + })?; + let persisted = file.write_all(contents).and_then(|()| file.sync_all()); + drop(file); + let result = persisted + .and_then(|()| replace_file(directory, &temporary, filename)) + .and_then(|()| match sync_directory(directory) { + Err(error) if error.kind() == io::ErrorKind::InvalidInput => Ok(()), + result => result, + }); + if result.is_err() { + let _ = directory.remove_file(&temporary); + } + result +} + fn open_root_nofollow(root_path: &Path) -> io::Result { let parent_path = root_path .parent() @@ -474,6 +530,42 @@ mod replacement_tests { use super::*; + #[cfg(unix)] + #[test] + fn marker_publication_does_not_follow_a_substituted_directory() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let owned = temp.path().join("owned"); + let moved = temp.path().join("moved"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir(&owned).unwrap(); + std::fs::create_dir(&foreign).unwrap(); + std::fs::write(foreign.join("sealed.checked"), b"foreign bytes").unwrap(); + + let held = open_root_nofollow(&owned).unwrap(); + std::fs::rename(&owned, &moved).unwrap(); + symlink(&foreign, &owned).unwrap(); + + assert!(atomic_replace_child_file_nofollow( + &owned, + OsStr::new("sealed.checked"), + b"new proof", + ) + .is_err()); + atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof") + .unwrap(); + assert_eq!( + std::fs::read(foreign.join("sealed.checked")).unwrap(), + b"foreign bytes", + ); + assert_eq!( + std::fs::read(moved.join("sealed.checked")).unwrap(), + b"owned proof", + ); + } + + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From 43b1ac43bc650615634815d819a3710c947ff140 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 06:59:08 -0800 Subject: [PATCH 35/70] style(fs): normalize sealed proof regression formatting Refs #4616. Keep the exact-head Bazel rustfmt gate unchanged. --- crates/tracedecay-private-fs/src/capability_dir.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 1da1852bec3..b4712db2ed7 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -553,8 +553,7 @@ mod replacement_tests { b"new proof", ) .is_err()); - atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof") - .unwrap(); + atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof").unwrap(); assert_eq!( std::fs::read(foreign.join("sealed.checked")).unwrap(), b"foreign bytes", @@ -565,7 +564,6 @@ mod replacement_tests { ); } - #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From fa9ae40206db0ea3b78a30ffc88c786cdf95d761 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:30:16 -0800 Subject: [PATCH 36/70] fix(graph): preserve foreign sealed proof marker entries Refs #4616 Refuse nonregular proof destinations through the held directory before and after staging. Verify the symlink entry and foreign target survive in native graph and private-filesystem regressions. --- .../tracedecay-graph-db/src/sealed_store.rs | 7 ++ .../src/capability_dir.rs | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 1e50ccf204c..1cfe5642820 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3185,6 +3185,13 @@ mod receipt_budget_tests { "publishing a proof marker must refuse a foreign symlink" ); assert_eq!(std::fs::read(&foreign).unwrap(), b"foreign sentinel"); + assert!( + std::fs::symlink_metadata(&marker) + .unwrap() + .file_type() + .is_symlink(), + "the foreign marker entry itself must survive" + ); } } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index b4712db2ed7..ae599dc4eb3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -245,6 +245,22 @@ fn atomic_replace_file_in_dir( filename: &OsStr, contents: &[u8], ) -> io::Result<()> { + // An accelerator may replace a regular cache file, but cannot take + // ownership of an unrelated symlink, directory, FIFO or device entry. + // Check relative to the held parent, never through an ambient pathname. + fn regular_or_missing(directory: &Dir, filename: &OsStr) -> io::Result<()> { + match directory.symlink_metadata(filename) { + Ok(metadata) if metadata.file_type().is_file() => Ok(()), + Ok(_) => Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "sealed proof destination is not a regular file", + )), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } + } + + regular_or_missing(directory, filename)?; static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); let (temporary, mut file) = (0..8) .find_map(|_| { @@ -272,6 +288,8 @@ fn atomic_replace_file_in_dir( let persisted = file.write_all(contents).and_then(|()| file.sync_all()); drop(file); let result = persisted + // Recheck before publishing in case the entry changed during staging. + .and_then(|()| regular_or_missing(directory, filename)) .and_then(|()| replace_file(directory, &temporary, filename)) .and_then(|()| match sync_directory(directory) { Err(error) if error.kind() == io::ErrorKind::InvalidInput => Ok(()), @@ -564,6 +582,54 @@ mod replacement_tests { ); } + #[cfg(unix)] + #[test] + fn proof_publication_preserves_nonregular_marker_entries() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("sealed.checked"); + let foreign = temp.path().join("foreign"); + std::fs::write(&foreign, b"foreign").unwrap(); + symlink(&foreign, &marker).unwrap(); + + assert_eq!( + atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied + ); + assert!( + std::fs::symlink_metadata(&marker) + .unwrap() + .file_type() + .is_symlink(), + "the link itself, not merely its target, must survive" + ); + assert_eq!(std::fs::read(&foreign).unwrap(), b"foreign"); + + std::fs::remove_file(&marker).unwrap(); + std::fs::create_dir(&marker).unwrap(); + assert_eq!( + atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied + ); + assert!(marker.is_dir()); + + std::fs::remove_dir(&marker).unwrap(); + atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") + .unwrap(); + assert_eq!(std::fs::read(&marker).unwrap(), b"proof"); + assert!( + std::fs::read_dir(temp.path()) + .unwrap() + .all(|entry| !entry.unwrap().file_name().to_string_lossy().starts_with(".sealed-proof-")), + "rejected publications must not leave staging files" + ); + } + #[test] fn replacement_publishes_complete_bytes_and_refuses_parent_traversal() { let root = tempfile::tempdir().unwrap(); From 280928f7b02bd82858bab44741491bdd6284a9e1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:32:20 -0800 Subject: [PATCH 37/70] style(fs): format sealed marker preservation regression Refs #4616 --- .../tracedecay-private-fs/src/capability_dir.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index ae599dc4eb3..76b66c298e0 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -622,12 +622,16 @@ mod replacement_tests { atomic_replace_child_file_nofollow(temp.path(), OsStr::new("sealed.checked"), b"proof") .unwrap(); assert_eq!(std::fs::read(&marker).unwrap(), b"proof"); - assert!( - std::fs::read_dir(temp.path()) - .unwrap() - .all(|entry| !entry.unwrap().file_name().to_string_lossy().starts_with(".sealed-proof-")), - "rejected publications must not leave staging files" - ); + for entry in std::fs::read_dir(temp.path()).unwrap() { + assert!( + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".sealed-proof-"), + "rejected publications must not leave staging files" + ); + } } #[test] From 261682f1643ac634db26276a5fa21e070446edde Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 08:40:54 -0800 Subject: [PATCH 38/70] style(graph): fix Bazel rustfmt failures in sealed regressions Refs #4616 --- .../sealed_store.rs | 25 ++++++------------- .../src/capability_dir.rs | 12 ++++----- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 94e656c34db..61cbf81297e 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1619,11 +1619,7 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { for (ordinal, digit) in (1..=5).zip(['1', '2', '3', '4', '5']) { let generation = format!("five-retire-g{ordinal}"); - let manifest = rich_manifest( - identity.clone(), - &generation, - &format!("marker-{ordinal}"), - ); + let manifest = rich_manifest(identity.clone(), &generation, &format!("marker-{ordinal}")); let record = stage_sealed_manifest( &mut authority, ®istered.binding, @@ -1632,13 +1628,7 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { prior_head.take(), digit, ); - let commit = publish_sealed( - ®istered, - temp.path(), - &mut authority, - &record, - &manifest, - ); + let commit = publish_sealed(®istered, temp.path(), &mut authority, &record, &manifest); prior_head = Some(commit.head.clone()); assert!( receipt_for_generation(temp.path(), &generation).is_some(), @@ -1650,11 +1640,8 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { superseded.push(( generation, manifest.identity(), - CodeGenerationId::new(format!( - "code-generation.{}", - manifest.generation.as_str() - )) - .unwrap(), + CodeGenerationId::new(format!("code-generation.{}", manifest.generation.as_str())) + .unwrap(), SealedGraphStateDigest::try_from(format!( "sha256:{}", digit.to_string().repeat(64) @@ -1699,7 +1686,9 @@ fn five_sealed_replay_generations_retire_to_the_live_head() { "retired generation {generation} must be absent from disk" ); assert!( - !database.is_generation_sealed_only(&manifest_identity).unwrap(), + !database + .is_generation_sealed_only(&manifest_identity) + .unwrap(), "retired generation {generation} must leave sealed-only state" ); assert_snapshot_reads(&serving.snapshot, &identity, "marker-5"); diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 76b66c298e0..1dbb76aaefe 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -9,12 +9,12 @@ use std::ffi::CString; use std::ffi::{OsStr, OsString}; use std::io::{self, Write}; -use std::sync::atomic::{AtomicU64, Ordering}; #[cfg(unix)] use std::os::fd::{AsRawFd, FromRawFd}; #[cfg(unix)] use std::os::unix::ffi::OsStrExt; use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; use cap_fs_ext::DirExt; #[cfg(not(windows))] @@ -565,12 +565,10 @@ mod replacement_tests { std::fs::rename(&owned, &moved).unwrap(); symlink(&foreign, &owned).unwrap(); - assert!(atomic_replace_child_file_nofollow( - &owned, - OsStr::new("sealed.checked"), - b"new proof", - ) - .is_err()); + assert!( + atomic_replace_child_file_nofollow(&owned, OsStr::new("sealed.checked"), b"new proof",) + .is_err() + ); atomic_replace_file_in_dir(&held, OsStr::new("sealed.checked"), b"owned proof").unwrap(); assert_eq!( std::fs::read(foreign.join("sealed.checked")).unwrap(), From 424ef247942680d46a532f72cc56fe7bd3ac1040 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:45:23 -0800 Subject: [PATCH 39/70] fix(graph): bind sealed census size walks to held directories Refs #4616 Inspect receipts and charge bytes beneath the same held no-follow root and child capabilities, not pathnames that can be swapped. Preserve bounded reads, hardlink accounting and unrecognized entries. Add a native root-substitution regression. --- .../tracedecay-graph-db/src/sealed_store.rs | 178 +++++++----------- .../src/capability_dir.rs | 122 ++++++++++++ 2 files changed, 194 insertions(+), 106 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 1cfe5642820..ba4cf6be8e8 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -571,123 +571,89 @@ pub struct SealedStoreCensusV1 { /// Metadata only: every entry is one receipt read plus a directory size walk, /// never a container open. An unreadable receipt counts as unrecognized so the /// census stays truthful when a receipt is mid-write or corrupt. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum SealedCensusEntry { + Head, + Superseded, + AbandonedStaging, + Unrecognized, +} + pub fn census_sealed_store( database_path: &Path, head_generations: &std::collections::BTreeSet, ) -> std::io::Result { let root = sealed_store_root(database_path); - let entries = match std::fs::read_dir(&root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(SealedStoreCensusV1::default()); - } - Err(error) => return Err(error), - }; - let mut census = SealedStoreCensusV1::default(); - let mut counted = std::collections::HashSet::new(); - for entry in entries { - let entry = entry?; - if !entry.file_type()?.is_dir() { - census.unrecognized_count += 1; - continue; - } - let path = entry.path(); - let name = entry.file_name(); - if name.to_str().is_some_and(sealed_staging_name) { - // A matching filename is not proof that the directory is - // abandoned: remounts can overlap a seal in this process, and - // foreign or interrupted pre-receipt entries may occupy its name. - // Do not walk unverified trees (which may be arbitrarily large). - let verified = tracedecay_private_fs::framed_log::read_bounded( - &path.join(SEALED_STAGING_OWNER_FILE), - MAX_SEALED_RECEIPT_BYTES, - ) - .ok() - .flatten() - .and_then(|bytes| verified_staging_receipt(&name, &bytes)) - .is_some_and(|owner| owner.producer != row_spill_owner().to_string()); - if !verified { - census.unrecognized_count += 1; - continue; + let current_producer = row_spill_owner().to_string(); + let totals = tracedecay_private_fs::capability_dir::census_receipted_directory_bytes( + &root, + SealedCensusEntry::Unrecognized, + |name| { + if name.to_str().is_some_and(sealed_staging_name) { + (SEALED_STAGING_OWNER_FILE, MAX_SEALED_RECEIPT_BYTES) + } else { + (SEALED_STORE_RECEIPT_FILE, MAX_SEALED_RECEIPT_BYTES) } - let bytes = directory_bytes(&path, &mut counted)?; - census.abandoned_staging_count += 1; - census.abandoned_staging_bytes = census.abandoned_staging_bytes.saturating_add(bytes); - continue; - } - // Receipt existence is not sufficient: a copied receipt can name a - // verified head while residing in an unrelated physical directory. - // Classify unproven entries before walking or charging their contents. - let receipt = read_sealed_receipt_bytes(&path.join(SEALED_STORE_RECEIPT_FILE)) - .ok() - .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); - let Some(receipt) = receipt else { - census.unrecognized_count += 1; - continue; - }; - let Some(name) = name.to_str() else { - census.unrecognized_count += 1; - continue; - }; - let form_valid = match receipt.form.as_str() { - SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), - SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), - _ => false, - }; - if !form_valid - || receipt.version != SEALED_STORE_RECEIPT_VERSION - || receipt.graph_format == 0 - || receipt.graph_format > GraphFormatVersion::current().get() - || receipt.recovered_digest.is_empty() - || !sealed_receipt_matches_generation_entry(&receipt, name) - { - census.unrecognized_count += 1; - continue; - } - let bytes = directory_bytes(&path, &mut counted)?; - if head_generations.contains(&receipt.generation) { - census.head_count += 1; - census.head_bytes = census.head_bytes.saturating_add(bytes); - } else { - census.superseded_count += 1; - census.superseded_bytes = census.superseded_bytes.saturating_add(bytes); - } - } - Ok(census) -} - -/// Bytes under `directory` not already counted in `counted`: a layered -/// generation's base container is a hard link to bytes another directory -/// already holds, so each file is charged once per census. -fn directory_bytes( - directory: &Path, - counted: &mut std::collections::HashSet<(u64, u64)>, -) -> std::io::Result { - let mut total = 0u64; - let mut pending = vec![directory.to_path_buf()]; - while let Some(directory) = pending.pop() { - for entry in std::fs::read_dir(&directory)? { - let entry = entry?; - // Count only the link itself: following a foreign directory link - // can mischarge its entire tree or revisit an ancestor forever. - let metadata = std::fs::symlink_metadata(entry.path())?; - if metadata.is_dir() { - pending.push(entry.path()); - continue; + }, + |name, bytes| { + let Some(bytes) = bytes else { + return SealedCensusEntry::Unrecognized; + }; + if name.to_str().is_some_and(sealed_staging_name) { + return if verified_staging_receipt(name, bytes) + .is_some_and(|owner| owner.producer != current_producer) + { + SealedCensusEntry::AbandonedStaging + } else { + SealedCensusEntry::Unrecognized + }; } - #[cfg(unix)] + let Some(name) = name.to_str() else { + return SealedCensusEntry::Unrecognized; + }; + let Ok(receipt) = serde_json::from_slice::(bytes) else { + return SealedCensusEntry::Unrecognized; + }; + let form_valid = match receipt.form.as_str() { + SEALED_STORE_FORM_COMPACT => receipt.base.is_none(), + SEALED_STORE_FORM_LAYERED => receipt.base.is_some(), + _ => false, + }; + if !form_valid + || receipt.version != SEALED_STORE_RECEIPT_VERSION + || receipt.graph_format == 0 + || receipt.graph_format > GraphFormatVersion::current().get() + || receipt.recovered_digest.is_empty() + || !sealed_receipt_matches_generation_entry(&receipt, name) { - use std::os::unix::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; - } + return SealedCensusEntry::Unrecognized; } - #[cfg(not(unix))] - let _ = &counted; - total = total.saturating_add(metadata.len()); + if head_generations.contains(&receipt.generation) { + SealedCensusEntry::Head + } else { + SealedCensusEntry::Superseded + } + }, + )?; + let mut census = SealedStoreCensusV1::default(); + for (kind, (count, bytes)) in totals { + match kind { + SealedCensusEntry::Head => { + census.head_count = count; + census.head_bytes = bytes; + } + SealedCensusEntry::Superseded => { + census.superseded_count = count; + census.superseded_bytes = bytes; + } + SealedCensusEntry::AbandonedStaging => { + census.abandoned_staging_count = count; + census.abandoned_staging_bytes = bytes; + } + SealedCensusEntry::Unrecognized => census.unrecognized_count = count, } } - Ok(total) + Ok(census) } /// Removes every sealed generation beside `database_path` whose receipt does diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 1dbb76aaefe..1a60b1184fc 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -334,6 +334,84 @@ pub fn install_child_directory_noreplace_nofollow( rename_noreplace(&root, staging_name, &root, generation_name) } +/// Census proof-bound children through held root/child capabilities, not +/// ambient paths. Unknown entries are counted but never traversed; only +/// multiply linked inodes consume deduplication memory. Receipts are bounded +/// and read from the same child whose bytes are subsequently charged. +pub fn census_receipted_directory_bytes( + root_path: &Path, + unrecognized: K, + mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, +) -> io::Result> { + let root = match open_root_nofollow(root_path) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Default::default()), + Err(error) => return Err(error), + }; + let mut totals = std::collections::BTreeMap::::new(); + let mut counted = std::collections::HashSet::<(u64, u64)>::new(); + for entry in root.read_dir(".")? { + let entry = entry?; + let name = entry.file_name(); + let child = if entry.file_type()?.is_dir() { + root.open_dir_nofollow(&name).ok() + } else { + None + }; + let (category, bytes) = if let Some(child) = child { + let (receipt_name, maximum) = receipt_for(&name); + let receipt = + read_bounded_child_receipt(&child, OsStr::new(receipt_name), maximum).ok(); + let category = classify(&name, receipt.as_deref()); + let bytes = if category == unrecognized { + 0 + } else { + directory_bytes_from_held_dir(child, &mut counted)? + }; + (category, bytes) + } else { + (unrecognized, 0) + }; + let totals = totals.entry(category).or_default(); + totals.0 = totals.0.saturating_add(1); + totals.1 = totals.1.saturating_add(bytes); + } + Ok(totals) +} + +/// Walk no-follow entries relative to the same held child, even after its +/// old pathname has been renamed. The stack scales with directory depth. +fn directory_bytes_from_held_dir( + directory: Dir, + counted: &mut std::collections::HashSet<(u64, u64)>, +) -> io::Result { + let mut total = 0_u64; + let mut pending = vec![directory]; + while let Some(directory) = pending.pop() { + for entry in directory.read_dir(".")? { + let entry = entry?; + let name = entry.file_name(); + let metadata = directory.symlink_metadata(&name)?; + if metadata.is_dir() { + pending.push(directory.open_dir_nofollow(&name)?); + continue; + } + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { + continue; + } + } + #[cfg(not(unix))] + let _ = &counted; + total = total.saturating_add(metadata.len()); + } + } + Ok(total) +} + /// Remove exactly one held child directory without following a substituted /// sealed-root or child symlink. Missing entries are idempotent; other failures /// remain errors so callers can retain retry ownership. @@ -820,6 +898,50 @@ mod tests { ); } + /// An ambient-path census would charge the 100,041 foreign bytes after + /// a root swap. This census must charge only its held 41-byte payload. + #[cfg(unix)] + #[test] + fn receipted_census_keeps_opened_child_after_root_substitution() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("graph.sealed"); + let displaced = temp.path().join("displaced"); + let foreign = temp.path().join("foreign"); + std::fs::create_dir_all(root.join("generation")).unwrap(); + std::fs::create_dir_all(foreign.join("generation")).unwrap(); + std::fs::write(root.join("generation/owner"), b"owned").unwrap(); + std::fs::write(root.join("generation/payload"), vec![b'a'; 41]).unwrap(); + std::fs::write(foreign.join("generation/owner"), b"owned").unwrap(); + std::fs::write(foreign.join("generation/payload"), vec![b'f'; 100_041]).unwrap(); + #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] + enum Entry { + Owned, + Unknown, + } + let totals = census_receipted_directory_bytes( + &root, + Entry::Unknown, + |_| ("owner", 16), + |name, receipt| { + assert_eq!(name, OsStr::new("generation")); + assert_eq!(receipt, Some(b"owned".as_slice())); + std::fs::rename(&root, &displaced).unwrap(); + std::os::unix::fs::symlink(&foreign, &root).unwrap(); + Entry::Owned + }, + ) + .unwrap(); + assert_eq!(totals.get(&Entry::Owned), Some(&(1, 46))); + assert_eq!( + std::fs::read(foreign.join("generation/payload")).unwrap(), + vec![b'f'; 100_041], + ); + assert_eq!( + std::fs::read(displaced.join("generation/payload")).unwrap(), + vec![b'a'; 41], + ); + } + /// Candidate filtering must not reopen a potentially substituted root: /// only the receipted owned child goes away; unrelated and foreign trees survive. #[cfg(unix)] From db26ea9948b0829fb15ea92e931f20fb477cc59b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:54:11 -0800 Subject: [PATCH 40/70] fix(graph): bound sealed census directory descriptors Walk verified generation children depth-first through held parent capabilities, not with one open handle per sibling. Add a native resource-limited regression with 128 sibling directories and exact byte accounting. Refs #4616 --- .../src/capability_dir.rs | 101 ++++++++++++++---- 1 file changed, 83 insertions(+), 18 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 1a60b1184fc..17501a7b29b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -387,27 +387,33 @@ fn directory_bytes_from_held_dir( counted: &mut std::collections::HashSet<(u64, u64)>, ) -> io::Result { let mut total = 0_u64; - let mut pending = vec![directory]; - while let Some(directory) = pending.pop() { - for entry in directory.read_dir(".")? { - let entry = entry?; - let name = entry.file_name(); - let metadata = directory.symlink_metadata(&name)?; - if metadata.is_dir() { - pending.push(directory.open_dir_nofollow(&name)?); + // Hold only the current ancestry chain. Opening every sibling before + // visiting any of them exhausted the process descriptor limit on wide + // verified generations and made the whole sealed census fail. + let mut pending = vec![(directory.read_dir(".")?, directory)]; + while let Some((entries, parent)) = pending.last_mut() { + let Some(entry) = entries.next() else { + pending.pop(); + continue; + }; + let entry = entry?; + let name = entry.file_name(); + let metadata = parent.symlink_metadata(&name)?; + if metadata.is_dir() { + let child = parent.open_dir_nofollow(&name)?; + pending.push((child.read_dir(".")?, child)); + continue; + } + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { continue; } - #[cfg(unix)] - { - use cap_std::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; - } - } - #[cfg(not(unix))] - let _ = &counted; - total = total.saturating_add(metadata.len()); } + #[cfg(not(unix))] + let _ = &counted; + total = total.saturating_add(metadata.len()); } Ok(total) } @@ -785,6 +791,65 @@ mod tests { Dir::open_ambient_dir(path, ambient_authority()).expect("open test directory") } + #[cfg(unix)] + #[test] + fn wide_verified_generation_census_stays_within_descriptor_budget() { + const CHILD: &str = "TRACEDECAY_WIDE_CENSUS_RLIMIT_CHILD"; + if std::env::var_os(CHILD).is_none() { + // Resource limits are process-wide; isolate the reduced budget + // from concurrently executing tests in the parent libtest runner. + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "capability_dir::tests::wide_verified_generation_census_stays_within_descriptor_budget", + "--nocapture", + ]) + .env(CHILD, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "census under reduced descriptor budget failed: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child must execute the actual behavioral regression: {}", + String::from_utf8_lossy(&output.stdout) + ); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("sealed.json"), b"owned").unwrap(); + for index in 0..128 { + let column = generation.join(format!("column-{index}")); + std::fs::create_dir(&column).unwrap(); + std::fs::write(column.join("value"), b"x").unwrap(); + } + + // The old breadth-first handle stack needed >=128 live handles + // here; the corrected walk needs only the current three levels. + let mut limit: libc::rlimit = unsafe { std::mem::zeroed() }; + assert_eq!(unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, 0); + limit.rlim_cur = limit.rlim_cur.min(64); + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0); + + let totals = census_receipted_directory_bytes( + &root, + false, + |_| ("sealed.json", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + ) + .unwrap(); + assert_eq!(totals.get(&true), Some(&(1, 133))); + assert_eq!(std::fs::read(generation.join("column-127/value")).unwrap(), b"x"); + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 02a570bcfea1a8b8d2db9a856fcaefa5e574f587 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:15:53 -0800 Subject: [PATCH 41/70] style(graph): satisfy Bazel rustfmt for sealed census regression Refs #4616 --- crates/tracedecay-private-fs/src/capability_dir.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 17501a7b29b..5374ca610d8 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -835,7 +835,10 @@ mod tests { // The old breadth-first handle stack needed >=128 live handles // here; the corrected walk needs only the current three levels. let mut limit: libc::rlimit = unsafe { std::mem::zeroed() }; - assert_eq!(unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, 0); + assert_eq!( + unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) }, + 0 + ); limit.rlim_cur = limit.rlim_cur.min(64); assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0); @@ -847,7 +850,10 @@ mod tests { ) .unwrap(); assert_eq!(totals.get(&true), Some(&(1, 133))); - assert_eq!(std::fs::read(generation.join("column-127/value")).unwrap(), b"x"); + assert_eq!( + std::fs::read(generation.join("column-127/value")).unwrap(), + b"x" + ); } #[test] From 66d70eeefdf2d9766fa24a8189f4055dfe81eb1e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:15:56 -0800 Subject: [PATCH 42/70] fix(graph): remove unowned sealed cleanup after checked retirement Drop obsolete pathname-cleanup helpers superseded by the receipt-verified retirement path. They are unused and fail the Bazel Clippy dead-code gate. Refs #4616 --- .../tracedecay-graph-db/src/sealed_store.rs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ba4cf6be8e8..257e117bdbc 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -904,18 +904,6 @@ pub(crate) fn sweep_abandoned_row_spills(database_path: &Path) { } } -fn remove_sealed_directory_checked(directory: &Path) -> Result<(), GraphDbError> { - let root = directory - .parent() - .ok_or_else(|| GraphDbError::unavailable("sealed directory has no parent"))?; - let name = directory - .file_name() - .ok_or_else(|| GraphDbError::unavailable("sealed directory has no entry name"))?; - tracedecay_private_fs::capability_dir::remove_child_directory_nofollow(root, name) - .map(|_| ()) - .map_err(|error| sealed_store_io_failure("sealed directory removal failed", error)) -} - /// Accepted replay retirement is not permission to recursively delete a /// substituted or unrecognizable artifact. Require the exact durable identity /// under a held child capability; preserve an unverified directory for retry. @@ -985,17 +973,6 @@ fn discard_failed_sealed_adoption( } } -fn remove_sealed_directory(directory: &Path) { - if let Err(error) = remove_sealed_directory_checked(directory) { - tracing::warn!( - event = "sealed_artifact_cleanup_deferred", - path = %directory.display(), - %error, - "sealed artifact cleanup did not complete" - ); - } -} - /// Open options for a sealed artifact: read-only, so a reopen for proof, /// adoption, or serving never re-serializes the immutable container on close /// and never moves the identity its marker binds. A sealed container is only From 807a738e5340bee642b25a5cf82000076ea46628 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:55:05 -0800 Subject: [PATCH 43/70] fix(graph): preserve sealed artifacts after uncertain reader close Refs #4616 Never unlink a verified sealed generation when its native reader failed physical close. Preserve the first non-idempotent close failure across layered readers, while allowing cleanly closed derived readers to retire. Add real sealed-build regressions for a poisoned native close and a previously closed reader. --- .../tracedecay-graph-db/src/sealed_store.rs | 80 +++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 257e117bdbc..2c35bef1130 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -408,13 +408,19 @@ impl SealedGenerationStore { } pub(crate) fn close(&self) -> Result<(), GraphDbError> { - let mut result = Ok(()); + // Closing the same derived reader twice is harmless. A durability + // failure is not: preserve the first real close failure even when a + // later layered-base reader was already closed. + let mut failure = None; for database in self.engines() { - if let Err(error) = database.close() { - result = Err(error); + match database.close() { + Ok(()) | Err(GraphDbError::Closed) => {} + Err(error) => { + failure.get_or_insert(error); + } } } - result + failure.map_or(Ok(()), Err) } pub(crate) fn hibernate_if_lazy(&self) -> Result<(), GraphDbError> { @@ -507,7 +513,9 @@ impl SealedGenerationStore { /// unlink. A failed unlink remains a typed error for the durable cleanup /// owner to retry; a sealed-only live head never enters this path. fn discard(&self) -> Result<(), GraphDbError> { - let _ = self.close(); + // An unconfirmed physical close cannot authorize unlinking the only + // durable sealed copy. Leave its receipt for a later retirement retry. + self.close()?; retire_sealed_directory_checked( &self.directory, &self.locator, @@ -3881,6 +3889,68 @@ mod build_tests { assert!(directory.join(SEALED_STORE_DATABASE_FILE).is_file()); } + #[cfg(unix)] + #[test] + fn retirement_keeps_sealed_bytes_when_reader_close_is_uncertain() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let directory = store.directory.clone(); + let container = directory.join(SEALED_STORE_DATABASE_FILE); + let receipt = directory.join(SEALED_STORE_RECEIPT_FILE); + assert!(container.is_file() && receipt.is_file()); + + // Poison the real native reader's engine lock. Its physical close + // now returns DurabilityUncertain; the old discard ignored that error + // and removed the verified directory anyway on Unix. + let reader = Arc::clone(&store.database); + let poisoned = std::thread::spawn(move || { + let _guard = reader.inner.database.write().unwrap(); + panic!("exercise uncertain sealed reader close"); + }); + assert!(poisoned.join().is_err()); + let refused = store.discard(); + assert!( + matches!(refused, Err(GraphDbError::DurabilityUncertain { .. })), + "retirement must surface the failed native close: {refused:?}" + ); + assert!(container.is_file(), "uncertain close must preserve sealed bytes"); + assert!(receipt.is_file(), "retry must retain its ownership receipt"); + } + + #[test] + fn retirement_accepts_an_already_closed_derived_reader() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let directory = store.directory.clone(); + store.database.close().unwrap(); + store.discard().unwrap(); + assert!(!directory.exists(), "a cleanly closed reader is reclaimable"); + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From 44f9220878afb656995607ace98bc04969582bcc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:57:00 -0800 Subject: [PATCH 44/70] fix(graph): retain poisoned close state across retirement retries Refs #4616 A failed native close consumes the engine handle but leaves durability poisoned; a subsequent close may return Closed. Preserve the poison signal so a later retirement tick cannot delete that unproven artifact. Exercise the exact prior-uncertain-close state with a native sealed store. --- .../tracedecay-graph-db/src/sealed_store.rs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 2c35bef1130..ea5b933bb02 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -419,6 +419,14 @@ impl SealedGenerationStore { failure.get_or_insert(error); } } + // An earlier failed engine.close() may have consumed its handle + // and set Closed on the next attempt. Closed is idempotent only + // when the underlying reader is not poisoned. + if database.inner.poisoned.load(AtomicOrdering::Acquire) { + failure.get_or_insert_with(|| GraphDbError::DurabilityUncertain { + message: "sealed reader physical close remains uncertain".to_owned(), + }); + } } failure.map_or(Ok(()), Err) } @@ -3951,6 +3959,41 @@ mod build_tests { assert!(!directory.exists(), "a cleanly closed reader is reclaimable"); } + #[test] + fn retirement_does_not_erase_a_prior_uncertain_close() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let directory = store.directory.clone(); + let container = directory.join(SEALED_STORE_DATABASE_FILE); + let receipt = directory.join(SEALED_STORE_RECEIPT_FILE); + + // Reproduce the post-failure state of native engine.close(): its + // handle is gone, Closed will be returned on retry, but durability + // remains uncertain. The first patch must not mistake that for a + // previously successful close and destroy its sealed receipt. + store.database.inner.poisoned.store(true, Ordering::Release); + let first = store.database.close(); + assert!(matches!(first, Err(GraphDbError::DurabilityUncertain { .. }))); + let retry = store.discard(); + assert!( + matches!(retry, Err(GraphDbError::DurabilityUncertain { .. })), + "a prior uncertain close must not become an unlink: {retry:?}" + ); + assert!(container.is_file() && receipt.is_file()); + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From b6f3ed5e6882b6d69e4965787c562135172ac789 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:56:48 -0800 Subject: [PATCH 45/70] fix(graph): retain failed-close readers across retirement retries Refs #4616 Retirement must retain its registered native reader when close or verified unlink fails. Otherwise the next tick takes the disk-only path and erases a sealed artifact whose physical close remains uncertain. Keep the map entry until verified removal, and remove it only if the same reader is still installed. Exercise two real retirement retries with a poisoned reader and successful normal retirement. Correct the three formatter-reported hunks without weakening CI. --- .../tracedecay-graph-db/src/sealed_store.rs | 140 ++++++++++++++++-- 1 file changed, 128 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index ea5b933bb02..80a320d16f3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -1893,18 +1893,33 @@ impl GraphDb { &self, locator: &GenerationLocator, ) -> Result<(), GraphDbError> { - let removed = self + let registered = self .inner .sealed_generations - .write() + .read() .map_err(|_| GraphDbError::unavailable("sealed generation registry lock poisoned"))? - .remove(locator); - if let Some(store) = removed { - return store.discard(); + .get(locator) + .cloned(); + if let Some(store) = registered { + // Keep the registered reader until its physical close AND sealed + // directory removal both succeed. A failed close may consume the + // native handle; dropping the registry entry would let the next + // tick take the disk-only path and unlink uncertain bytes. + store.discard()?; + let mut sealed = self.inner.sealed_generations.write().map_err(|_| { + GraphDbError::unavailable("sealed generation registry lock poisoned") + })?; + if sealed + .get(locator) + .is_some_and(|current| Arc::ptr_eq(current, &store)) + { + sealed.remove(locator); + } + return Ok(()); } - // A removed in-memory reader is not necessary for the artifact to - // exist on disk. Unlink by exact locator even after a prior failed - // retirement; a durable tombstone can then retry without the reader. + // The artifact may exist on disk without a resident reader, for + // example after reopening a store before generation activation. + // Only receipt-verified, exact-locator disk cleanup is authorized. let Some(reopen) = self.inner.reopen.as_ref() else { return Ok(()); }; @@ -3933,7 +3948,10 @@ mod build_tests { matches!(refused, Err(GraphDbError::DurabilityUncertain { .. })), "retirement must surface the failed native close: {refused:?}" ); - assert!(container.is_file(), "uncertain close must preserve sealed bytes"); + assert!( + container.is_file(), + "uncertain close must preserve sealed bytes" + ); assert!(receipt.is_file(), "retry must retain its ownership receipt"); } @@ -3956,7 +3974,10 @@ mod build_tests { let directory = store.directory.clone(); store.database.close().unwrap(); store.discard().unwrap(); - assert!(!directory.exists(), "a cleanly closed reader is reclaimable"); + assert!( + !directory.exists(), + "a cleanly closed reader is reclaimable" + ); } #[test] @@ -3981,11 +4002,14 @@ mod build_tests { // Reproduce the post-failure state of native engine.close(): its // handle is gone, Closed will be returned on retry, but durability - // remains uncertain. The first patch must not mistake that for a + // remains uncertain. Retirement must never mistake that state for a // previously successful close and destroy its sealed receipt. store.database.inner.poisoned.store(true, Ordering::Release); let first = store.database.close(); - assert!(matches!(first, Err(GraphDbError::DurabilityUncertain { .. }))); + assert!(matches!( + first, + Err(GraphDbError::DurabilityUncertain { .. }) + )); let retry = store.discard(); assert!( matches!(retry, Err(GraphDbError::DurabilityUncertain { .. })), @@ -3994,6 +4018,98 @@ mod build_tests { assert!(container.is_file() && receipt.is_file()); } + #[test] + fn retirement_retries_must_keep_a_reader_with_uncertain_close_registered() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let parent = GraphDb::open(GraphDbOpenOptions { + location: GraphDbLocation::Memory, + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::Memory, + cancellation: Arc::new(NeverCancelled), + }) + .unwrap(); + parent + .inner + .sealed_generations + .write() + .unwrap() + .insert(locator.clone(), Arc::clone(&store)); + + let container = store.directory.join(SEALED_STORE_DATABASE_FILE); + let receipt = store.directory.join(SEALED_STORE_RECEIPT_FILE); + let held_reader = Arc::clone(store.database()); + store.database.inner.poisoned.store(true, Ordering::Release); + for attempt in 1..=2 { + let retirement = parent.retire_sealed_generation_store(&locator); + assert!( + matches!(&retirement, Err(GraphDbError::DurabilityUncertain { .. })), + "retirement attempt {attempt} must refuse uncertain close: {retirement:?}" + ); + let registered = parent + .sealed_generation_reader(&locator) + .expect("failed close must preserve the registered reader"); + assert!(Arc::ptr_eq(®istered, &store)); + assert!(container.is_file() && receipt.is_file()); + } + assert!(Arc::ptr_eq(&held_reader, store.database())); + } + + #[test] + fn successful_retirement_uninstalls_the_reader_after_removing_artifact() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let parent = GraphDb::open(GraphDbOpenOptions { + location: GraphDbLocation::Memory, + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::Memory, + cancellation: Arc::new(NeverCancelled), + }) + .unwrap(); + parent + .inner + .sealed_generations + .write() + .unwrap() + .insert(locator.clone(), Arc::clone(&store)); + + parent.retire_sealed_generation_store(&locator).unwrap(); + assert!(parent.sealed_generation_reader(&locator).is_none()); + assert!(!store.directory.exists()); + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From 0b7c0558f3d1b2e985217eaddb16520638f60cd0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:57:54 -0800 Subject: [PATCH 46/70] test(graph): prove failed retirement cannot delete sealed bytes Refs #4616 Use the real persistent parent store in both retirement regressions. The prior implementation then takes the disk-only path on the second retry and erases the verified sealed artifact despite uncertain close; assert two typed refusals and byte-exact artifact preservation. --- crates/tracedecay-graph-db/src/sealed_store.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 80a320d16f3..db76bd82740 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -4039,9 +4039,9 @@ mod build_tests { identity.generation.clone(), ); let parent = GraphDb::open(GraphDbOpenOptions { - location: GraphDbLocation::Memory, + location: GraphDbLocation::Persistent(database_path.clone()), expected_format: GraphFormatVersion::current(), - durability: GraphDurability::Memory, + durability: GraphDurability::WalSync, cancellation: Arc::new(NeverCancelled), }) .unwrap(); @@ -4092,9 +4092,9 @@ mod build_tests { identity.generation.clone(), ); let parent = GraphDb::open(GraphDbOpenOptions { - location: GraphDbLocation::Memory, + location: GraphDbLocation::Persistent(database_path.clone()), expected_format: GraphFormatVersion::current(), - durability: GraphDurability::Memory, + durability: GraphDurability::WalSync, cancellation: Arc::new(NeverCancelled), }) .unwrap(); From e50180bbcf221a5738d72b0cf886532fdf899498 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:21:56 -0800 Subject: [PATCH 47/70] fix(graph): preserve poisoned close with an absent engine Refs #4616 A failed native close can consume the handle and leave durability poison. Even with no engine resident, a later close must never report success or idempotent Closed. Align the copied-receipt recovery fixture with the producer-bound staging authority, and start the live-reader timing bound after the release worker has actually started. --- crates/tracedecay-graph-db/src/runtime.rs | 14 +++++++++++--- .../verified_generation_contract/sealed_store.rs | 15 ++++++++++----- .../staging_footprint.rs | 6 ++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-graph-db/src/runtime.rs b/crates/tracedecay-graph-db/src/runtime.rs index 7c40e5b3573..a75438acdc4 100644 --- a/crates/tracedecay-graph-db/src/runtime.rs +++ b/crates/tracedecay-graph-db/src/runtime.rs @@ -1028,10 +1028,18 @@ impl GraphDb { } }; if !engine_is_open { - if self.inner.closed.swap(true, Ordering::AcqRel) { - return Err(GraphDbError::Closed); + let previously_closed = self.inner.closed.swap(true, Ordering::AcqRel); + // A failed physical close can already have consumed the engine. + // Its absent handle must not turn an uncertain result into a + // successful close (or an idempotent Closed on the next retry). + if self.inner.poisoned.load(Ordering::Acquire) { + return Err(durability_uncertain()); } - return Ok(()); + return if previously_closed { + Err(GraphDbError::Closed) + } else { + Ok(()) + }; } let _snapshot_gate = self.wait_snapshot_gate_write(); let mut guard = diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 61cbf81297e..ec402598a0f 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1229,9 +1229,9 @@ fn restart_recovery_adopts_or_discards_the_on_disk_artifact() { assert_snapshot_reads(&snapshot, &identity, "durable"); } -/// A completed interrupted seal carries an exact generation receipt and is -/// reclaimed when reopening the store. Unreceipted, mismatched, oversized and -/// linked lookalikes must survive instead of being deleted by filename. +/// A copied installed receipt is not a staging-owner receipt. Recovery must +/// preserve that lookalike alongside unreceipted, mismatched, oversized and +/// linked entries, rather than deleting them based on their names. #[test] fn reopening_the_store_reclaims_only_receipted_staging() { let temp = TempDir::new().unwrap(); @@ -1331,8 +1331,13 @@ fn reopening_the_store_reclaims_only_receipted_staging() { .unwrap(); assert!( - !verified.exists(), - "a verified abandoned seal must be reclaimed" + verified.is_dir(), + "a copied final receipt must not authorize abandoned-stage cleanup" + ); + assert_eq!( + std::fs::read(verified.join("sealed.json")).unwrap(), + receipt, + "recovery must preserve the copied final receipt byte-for-byte" ); assert_eq!( std::fs::read(unowned.join("generation.grafeo")).unwrap(), diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs index 01c55d10f34..fa83a05c605 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/staging_footprint.rs @@ -909,8 +909,10 @@ fn release_sweep_retries_idle_sealed_reader_hibernation() { let root = temp.path().to_path_buf(); let projection = replay.publication.key.projection.clone(); let (result_tx, result_rx) = mpsc::channel(); + let (started_tx, started_rx) = mpsc::channel(); let sweep = thread::spawn(move || { let (control, probe) = control_and_probe(); + started_tx.send(()).unwrap(); let result = registry.release_sealed_generation_staging_rows( registration(binding, &root), &mut authority, @@ -920,6 +922,10 @@ fn release_sweep_retries_idle_sealed_reader_hibernation() { result_tx.send(()).unwrap(); (authority, result) }); + // Start the bound only after the sweep thread has actually been + // scheduled; otherwise the test measures worker startup under CI load, + // not whether a live sealed reader blocks staging cleanup. + started_rx.recv().unwrap(); let busy_completion = result_rx.recv_timeout(Duration::from_millis(100)); let engines_during_read = database.sealed_generation_engine_census(); release.wait(); From 9dfe06dbd6a0e7d385129bc100b932c3215d9270 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:28:10 -0800 Subject: [PATCH 48/70] fix(graph): fence lazy reopen against physical close Refs #4616 A lazy opener checked availability before waiting for the database lock. A concurrent close could mark the handle Closed during that wait and the opener would still publish a native engine into the closed handle. Publish a no-engine close while holding the database read lock and recheck availability under the lazy opener's write lock. A native persistent-store regression races both never-opened and hibernated handles at the existing lifecycle seam, checks no engine resurrects, and verifies a new mount still reads the committed generation. --- crates/tracedecay-graph-db/src/runtime.rs | 16 ++++- .../tracedecay-graph-db/src/runtime/tests.rs | 70 +++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/runtime.rs b/crates/tracedecay-graph-db/src/runtime.rs index a75438acdc4..43ff17f5f56 100644 --- a/crates/tracedecay-graph-db/src/runtime.rs +++ b/crates/tracedecay-graph-db/src/runtime.rs @@ -1015,8 +1015,8 @@ impl GraphDb { /// management, still checkpoints in full before this returns. #[tracing::instrument(name = "graph_db.runtime.close", level = "trace", skip_all)] pub(crate) fn close(&self) -> Result<(), GraphDbError> { - let engine_is_open = match self.inner.database.read() { - Ok(database) => database.is_some(), + let engine = match self.inner.database.read() { + Ok(database) => database, Err(_) => { self.inner.closed.store(true, Ordering::Release); self.inner.poisoned.store(true, Ordering::Release); @@ -1027,7 +1027,9 @@ impl GraphDb { }); } }; - if !engine_is_open { + if engine.is_none() { + // Keep the absence observation locked until close is published. + // A lazy opener must not install an engine in that interval. let previously_closed = self.inner.closed.swap(true, Ordering::AcqRel); // A failed physical close can already have consumed the engine. // Its absent handle must not turn an uncertain result into a @@ -1041,6 +1043,7 @@ impl GraphDb { Ok(()) }; } + drop(engine); let _snapshot_gate = self.wait_snapshot_gate_write(); let mut guard = match crate::observe::wait_lock(crate::observe::LOCK_WAIT_DATABASE_WRITE, || { @@ -1721,11 +1724,16 @@ impl GraphDb { return Ok(()); } self.ensure_available()?; + #[cfg(test)] + test_seams::fire(test_seams::Seam::LazyOpenAdmission); let mut database = crate::observe::wait_lock(crate::observe::LOCK_WAIT_DATABASE_WRITE, || { self.inner.database.write() }) .map_err(|_| GraphDbError::unavailable("graph database write lock is poisoned"))?; + // Close or a failed hibernation may have won while this opener + // waited. Revalidate before publishing another native engine. + self.ensure_available()?; if database.is_some() { return Ok(()); } @@ -1973,6 +1981,8 @@ pub(crate) mod test_seams { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum Seam { + /// After lazy-open admission, before acquiring the database write lock. + LazyOpenAdmission, /// Immediately before grafeo opens the container at `site`. EngineOpen(GraphEngineOpenSite), /// After the engine has closed and synced the container, immediately diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index e682aad20fa..418ab47216a 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -6,6 +6,7 @@ use std::time::Duration; use grafeo_common::types::Value; +use super::test_seams::{self, Seam}; use super::{GraphDb, sync_wal}; use crate::location::PersistentGraphStoreState; use crate::recovery::set_projection_quarantine; @@ -29,6 +30,75 @@ fn memory_db() -> GraphDbLeaseV1 { .unwrap() } +#[test] +fn lazy_open_rechecks_close_before_materializing_an_engine() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("graph.grafeo"); + let options = || GraphDbOpenOptions { + location: GraphDbLocation::Persistent(path.clone()), + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::WalSync, + cancellation: Arc::new(NeverCancelled), + }; + let seed = GraphDb::open(options()).unwrap(); + seed.apply_unverified(scalar_batch("retained")).unwrap(); + seed.close().unwrap(); + drop(seed); + + // Both a never-materialized mount and a hibernated engine can be absent + // when close races a lazy reader's admission. + for hibernated in [false, true] { + let database = GraphDb::open_lazy_with_store_state(options(), PersistentGraphStoreState::Existing) + .unwrap(); + if hibernated { + database.ensure_opened().unwrap(); + database.hibernate_if_lazy().unwrap(); + } + assert!(database.inner.database.read().unwrap().is_none()); + + let closing = Arc::clone(&database); + let admissions = Arc::new(AtomicUsize::new(0)); + let observed = Arc::clone(&admissions); + let hook = test_seams::install(move |seam| { + if seam == Seam::LazyOpenAdmission { + observed.fetch_add(1, Ordering::Relaxed); + // The opener has already observed Ready. Complete the real + // close before it takes its database write lock. + closing.close().unwrap(); + } + }); + let result = database.ensure_opened(); + drop(hook); + + assert_eq!(admissions.load(Ordering::Relaxed), 1); + assert_eq!(result, Err(GraphDbError::Closed)); + assert_eq!(database.runtime_state(), GraphDbRuntimeState::Closed); + assert!( + database.inner.database.read().unwrap().is_none(), + "a completed close must not be followed by a resurrected engine" + ); + } + + // Refusing the stale opener must not corrupt the durable store or block + // a new, independently authorized mount from reading the committed row. + let reopened = GraphDb::open(options()).unwrap(); + let snapshot = reopened.snapshot().unwrap(); + let entity = snapshot + .entity( + &GraphNamespace::new("project").unwrap(), + &GraphEntityId::new("a").unwrap(), + Arc::new(NeverCancelled), + ) + .unwrap() + .unwrap(); + assert_eq!( + entity.properties.get(&GraphPropertyName::new("name").unwrap()), + Some(&GraphProperty::String("retained".to_owned())) + ); + drop(snapshot); + reopened.close().unwrap(); +} + #[test] fn runtime_state_distinguishes_ready_closed_and_durability_uncertain() { let ready = memory_db(); From 4af675591e5bfa25b8ab12230b5e5b34eee90a57 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:29:19 -0800 Subject: [PATCH 49/70] style(graph): format lazy-close regression setup Refs #4616 Use repository-standard multiline call wrapping in the new native test without changing its assertions or the close/reopen behavior. --- crates/tracedecay-graph-db/src/runtime/tests.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index 418ab47216a..8edccc786b9 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -48,8 +48,9 @@ fn lazy_open_rechecks_close_before_materializing_an_engine() { // Both a never-materialized mount and a hibernated engine can be absent // when close races a lazy reader's admission. for hibernated in [false, true] { - let database = GraphDb::open_lazy_with_store_state(options(), PersistentGraphStoreState::Existing) - .unwrap(); + let database = + GraphDb::open_lazy_with_store_state(options(), PersistentGraphStoreState::Existing) + .unwrap(); if hibernated { database.ensure_opened().unwrap(); database.hibernate_if_lazy().unwrap(); From a59921d4bcd4ef40c7a41ea5d3b51e72b20bdbc9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:33:51 -0800 Subject: [PATCH 50/70] fix(graph): verify failed spill publication cleanup Refs #4616 A rejected rename dropped TempDir with pathname-based recursive cleanup still enabled. Another real directory installed at the pending name could be deleted despite having no row-spill ownership receipt. Disable automatic cleanup before publication, and on failure reclaim only a receipt-verified child under its held directory capability. A native regression substitutes a foreign real directory before a refused publication and preserves both it and the occupied destination. --- .../src/generation/spill.rs | 94 ++++++++++++++++++- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 8add23cd36d..88fee1ee212 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -155,13 +155,62 @@ struct SpillDirectory(PathBuf); impl SpillDirectory { fn create(path: PathBuf) -> Result { - let mut staging = Self::prepare(&path, row_spill_owner())?; - tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), &path) - .map_err(|error| spill_io("directory install", error))?; - staging.disable_cleanup(true); + let owner = row_spill_owner(); + let mut staging = Self::prepare(&path, owner)?; + Self::install(&mut staging, &path, owner)?; Ok(Self(path)) } + fn install( + staging: &mut tempfile::TempDir, + path: &Path, + owner: u128, + ) -> Result<(), GraphDbError> { + // TempDir's default Drop recursively deletes by pathname. A failed + // no-replace publication must not delete a foreign directory moved + // into that name while we attempted installation. Disable it before + // the rename, including the successful-rename/reused-name window. + staging.disable_cleanup(true); + let name = path + .file_name() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no entry name"))?; + let receipt = row_spill_owner_receipt(name, owner) + .ok_or_else(|| GraphDbError::invalid("graph row spill directory name is invalid"))?; + let pending = staging_name(owner, &receipt); + if staging.path().file_name() != Some(OsStr::new(&pending)) { + return Err(GraphDbError::invalid( + "graph row spill staging name does not bind its owner receipt", + )); + } + if let Err(error) = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path) + { + let root = staging.path().parent().ok_or_else(|| { + GraphDbError::invalid("graph row spill directory has no parent") + })?; + // On failure, only the exact owner receipt read from the opened + // child authorizes recursive removal. A substituted directory, + // even an ordinary real directory, must survive unchanged. + if let Err(cleanup_error) = + tracedecay_private_fs::capability_dir::remove_receipted_child_directory_nofollow( + root, + OsStr::new(&pending), + OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + MAX_ROW_SPILL_OWNER_RECEIPT_BYTES, + |bytes| bytes == receipt, + ) + { + tracing::warn!( + event = "graph_row_spill_prepare_cleanup_refused", + path = %staging.path().display(), + error = %cleanup_error, + "unverified pending row spill preserved for inspection" + ); + } + return Err(spill_io("directory install", error)); + } + Ok(()) + } + fn prepare(path: &Path, owner: u128) -> Result { let name = path .file_name() @@ -1181,6 +1230,43 @@ mod owner_receipt_tests { row_spill_owner, staging_name, }; + #[cfg(unix)] + #[test] + fn failed_pending_publication_preserves_a_substituted_real_directory() { + let root = tempfile::tempdir().unwrap(); + let owner = row_spill_owner(); + let destination = root.path().join(format!("{ROW_SPILL_PREFIX}{owner}-0")); + let mut staged = SpillDirectory::prepare(&destination, owner).unwrap(); + let pending_path = staged.path().to_path_buf(); + let moved = root.path().join("original-pending-spill"); + std::fs::rename(&pending_path, &moved).unwrap(); + std::fs::create_dir(&pending_path).unwrap(); + std::fs::write(pending_path.join("sentinel"), b"foreign pending contents").unwrap(); + std::fs::create_dir(&destination).unwrap(); + std::fs::write(destination.join("sentinel"), b"occupied destination").unwrap(); + + let refused = SpillDirectory::install(&mut staged, &destination, owner); + assert!( + refused.is_err(), + "occupied publication destination must be refused" + ); + drop(staged); + assert_eq!( + std::fs::read(pending_path.join("sentinel")).unwrap(), + b"foreign pending contents", + "failed install must not recursively delete a substituted real directory" + ); + assert_eq!( + std::fs::read(destination.join("sentinel")).unwrap(), + b"occupied destination" + ); + let original_receipt = std::fs::read(moved.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(); + assert!( + original_receipt.starts_with(RECEIPT_PREFIX.as_bytes()), + "original owned staging bytes must remain separate from the foreign entry" + ); + } + #[cfg(unix)] #[test] fn receipt_write_failure_leaves_no_spill_directory() { From 9f04bedee24a8a46ee277a6eacf422264fb3afd3 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:34:27 -0800 Subject: [PATCH 51/70] style(graph): keep spill install result explicit Refs #4616 Name the no-replace outcome before matching its error, keeping the receipt-verified cleanup logic and regression unchanged. --- crates/tracedecay-graph-db/src/generation/spill.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 88fee1ee212..7d9d7937eb8 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -182,8 +182,8 @@ impl SpillDirectory { "graph row spill staging name does not bind its owner receipt", )); } - if let Err(error) = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path) - { + let installed = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path); + if let Err(error) = installed { let root = staging.path().parent().ok_or_else(|| { GraphDbError::invalid("graph row spill directory has no parent") })?; From f1f880fb899921d20a251a662ff51b2553a02379 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:44:13 -0800 Subject: [PATCH 52/70] fix(graph): keep pending receipt failure cleanup owner-bound Refs #4616 Disable TempDir pathname cleanup before creating a pending receipt. Use a held, no-follow directory for exclusive marker publication and nonrecursive cleanup after a failed write. Preserve substituted directories and verify the refusal with an on-disk ownership regression. --- .../src/generation/spill.rs | 78 ++++++++++++++++--- .../src/capability_dir.rs | 54 +++++++++++++ 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 7d9d7937eb8..4a533d5e3b3 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -223,24 +223,31 @@ impl SpillDirectory { // Receipt failures must leave the requested name available to retry. // Keep the partial marker private until the complete spill can be installed. let temporary = staging_name(owner, &receipt); - let staging = tempfile::Builder::new() + let mut staging = tempfile::Builder::new() .prefix(&temporary) .rand_bytes(0) .tempdir_in(parent) .map_err(|error| spill_io("directory create", error))?; - let marker = staging.path().join(ROW_SPILL_OWNER_RECEIPT_FILE); - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&marker) - .map_err(|error| spill_io("owner marker create", error))?; - file.write_all(&receipt) - .and_then(|()| file.sync_all()) - .map_err(|error| spill_io("owner marker persist", error))?; - drop(file); + Self::persist_pending_receipt(&mut staging, &receipt)?; Ok(staging) } + fn persist_pending_receipt( + staging: &mut tempfile::TempDir, + receipt: &[u8], + ) -> Result<(), GraphDbError> { + // TempDir's Drop is pathname-based even when receipt creation fails. + // Once our name has been replaced, that Drop could erase foreign + // bytes. Disarm it before the first fallible marker operation. + staging.disable_cleanup(true); + tracedecay_private_fs::capability_dir::write_exclusive_pending_receipt_nofollow( + staging.path(), + OsStr::new(ROW_SPILL_OWNER_RECEIPT_FILE), + receipt, + ) + .map_err(|error| spill_io("owner marker persist", error)) + } + fn path(&self) -> &Path { &self.0 } @@ -1227,7 +1234,7 @@ impl GraphGenerationManifestIdentity { mod owner_receipt_tests { use super::{ RECEIPT_PREFIX, ROW_SPILL_OWNER_RECEIPT_FILE, ROW_SPILL_PREFIX, SpillDirectory, - row_spill_owner, staging_name, + row_spill_owner, row_spill_owner_receipt, staging_name, }; #[cfg(unix)] @@ -1267,6 +1274,53 @@ mod owner_receipt_tests { ); } + #[cfg(unix)] + #[test] + fn failed_pending_receipt_open_preserves_substituted_directory() { + let root = tempfile::tempdir().unwrap(); + let owner = row_spill_owner(); + let destination = root.path().join(format!("{ROW_SPILL_PREFIX}{owner}-9")); + let receipt = row_spill_owner_receipt(destination.file_name().unwrap(), owner).unwrap(); + let name = staging_name(owner, &receipt); + let mut staging = tempfile::Builder::new() + .prefix(&name) + .rand_bytes(0) + .tempdir_in(root.path()) + .unwrap(); + let pending_path = staging.path().to_path_buf(); + let moved = root.path().join("original-pending"); + std::fs::rename(&pending_path, &moved).unwrap(); + std::fs::create_dir(&pending_path).unwrap(); + std::fs::write( + pending_path.join(ROW_SPILL_OWNER_RECEIPT_FILE), + b"foreign receipt", + ) + .unwrap(); + std::fs::write(pending_path.join("sentinel"), b"foreign contents").unwrap(); + + let refused = SpillDirectory::persist_pending_receipt(&mut staging, &receipt); + assert!( + refused + .unwrap_err() + .to_string() + .contains("owner marker persist") + ); + drop(staging); + assert_eq!( + std::fs::read(pending_path.join("sentinel")).unwrap(), + b"foreign contents", + "failed receipt creation must not drop a substituted real directory" + ); + assert_eq!( + std::fs::read(pending_path.join(ROW_SPILL_OWNER_RECEIPT_FILE)).unwrap(), + b"foreign receipt" + ); + assert!( + moved.is_dir(), + "the original staging directory cannot authorize deleting its replacement" + ); + } + #[cfg(unix)] #[test] fn receipt_write_failure_leaves_no_spill_directory() { diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 5374ca610d8..2ac29928977 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -228,6 +228,60 @@ fn single_component(name: &OsStr) -> io::Result<()> { Ok(()) } +/// Persist a new producer-owned staging receipt through a held directory. +/// +/// An exclusive marker creation is the only authority for removing an +/// incomplete marker. On write/sync failure, remove only that marker and +/// attempt a nonrecursive removal of the same held directory. A substituted +/// directory containing other bytes must survive. The caller must disable +/// pathname-based temporary-directory cleanup before invoking this operation. +pub fn write_exclusive_pending_receipt_nofollow( + directory_path: &Path, + filename: &OsStr, + contents: &[u8], +) -> io::Result<()> { + single_component(filename)?; + let directory = open_root_nofollow(directory_path)?; + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + let mut marker = directory.open_with(filename, &options)?; + #[cfg(unix)] + let marker_identity = { + use cap_std::fs::MetadataExt; + marker.metadata().ok().map(|metadata| (metadata.dev(), metadata.ino())) + }; + let written = marker.write_all(contents).and_then(|()| marker.sync_all()); + drop(marker); + if let Err(error) = written { + // Keep the original write failure. A different marker entry is not + // ours to delete, and a nonempty substituted directory is not ours + // to remove. Neither cleanup operation follows an ambient pathname. + #[cfg(unix)] + let same_marker = { + use cap_std::fs::MetadataExt; + marker_identity.is_some_and(|identity| { + directory + .symlink_metadata(filename) + .ok() + .is_some_and(|metadata| { + metadata.is_file() && (metadata.dev(), metadata.ino()) == identity + }) + }) + }; + #[cfg(not(unix))] + let same_marker = false; + if same_marker { + if let Err(cleanup) = directory.remove_file(filename) { + tracing::warn!(%cleanup, "pending owner marker could not be removed"); + } else if let Err(cleanup) = directory.remove_open_dir() { + tracing::warn!(%cleanup, "pending owner directory was not empty"); + } + } + return Err(error); + } + Ok(()) +} + /// Publish a rebuildable marker through one held directory. Replacing the /// pathname after acquisition cannot redirect its bytes into a foreign tree. pub fn atomic_replace_child_file_nofollow( From 72d885b06661ac558423b357bf69707769a79ae7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:58:29 -0800 Subject: [PATCH 53/70] fix(graph): attribute sealed hardlinks to serving heads Refs #4616 The sealed census charged a shared inode to the first directory visited. A superseded generation could therefore claim live-head physical bytes. Transfer hard-link charges to the stronger verified category regardless of read_dir order, retaining bounded metadata-only accounting. Prove the adverse visitation order using real native hard links and the production census. --- .../src/capability_dir.rs | 98 +++++++++++++++++-- 1 file changed, 92 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 2ac29928977..a8768369cfe 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -404,7 +404,10 @@ pub fn census_receipted_directory_bytes( Err(error) => return Err(error), }; let mut totals = std::collections::BTreeMap::::new(); - let mut counted = std::collections::HashSet::<(u64, u64)>::new(); + // Hard links belong to the highest-priority verified category, not the + // first entry returned by read_dir. Retain only multiply-linked identities + // so the census remains bounded independently of file count and size. + let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); for entry in root.read_dir(".")? { let entry = entry?; let name = entry.file_name(); @@ -421,7 +424,7 @@ pub fn census_receipted_directory_bytes( let bytes = if category == unrecognized { 0 } else { - directory_bytes_from_held_dir(child, &mut counted)? + directory_bytes_from_held_dir(child, category, &mut totals, &mut counted)? }; (category, bytes) } else { @@ -436,9 +439,11 @@ pub fn census_receipted_directory_bytes( /// Walk no-follow entries relative to the same held child, even after its /// old pathname has been renamed. The stack scales with directory depth. -fn directory_bytes_from_held_dir( +fn directory_bytes_from_held_dir( directory: Dir, - counted: &mut std::collections::HashSet<(u64, u64)>, + category: K, + totals: &mut std::collections::BTreeMap, + counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, ) -> io::Result { let mut total = 0_u64; // Hold only the current ancestry chain. Opening every sibling before @@ -461,8 +466,31 @@ fn directory_bytes_from_held_dir( #[cfg(unix)] { use cap_std::fs::MetadataExt; - if metadata.nlink() > 1 && !counted.insert((metadata.dev(), metadata.ino())) { - continue; + if metadata.nlink() > 1 { + let key = (metadata.dev(), metadata.ino()); + if let Some((previous_kind, charged_bytes)) = counted.get_mut(&key) { + // A lower-ordered category has stronger ownership. A + // superseded generation encountered first cannot claim + // hard-linked physical bytes from a later live head. + if category < *previous_kind { + let previous = totals.get_mut(&*previous_kind).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes lack their prior category", + ) + })?; + previous.1 = previous.1.checked_sub(*charged_bytes).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes exceed their prior category", + ) + })?; + total = total.saturating_add(*charged_bytes); + *previous_kind = category; + } + continue; + } + counted.insert(key, (category, metadata.len())); } } #[cfg(not(unix))] @@ -910,6 +938,64 @@ mod tests { ); } + /// Hard-linked base bytes count once and belong to the live head even + /// when a superseded generation was physically visited first. + #[cfg(unix)] + #[test] + fn shared_sealed_bytes_follow_head_priority_not_directory_visitation() { + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] + enum Kind { + Head, + Superseded, + Unrecognized, + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let superseded = root.join("old"); + let head = root.join("head"); + std::fs::create_dir_all(&superseded).unwrap(); + std::fs::create_dir(&head).unwrap(); + std::fs::write(superseded.join("owner"), b"old").unwrap(); + std::fs::write(head.join("owner"), b"head").unwrap(); + std::fs::write(superseded.join("shared"), b"shared-bytes").unwrap(); + std::fs::hard_link(superseded.join("shared"), head.join("shared")).unwrap(); + + // Exercise the actual walker in the adverse order without relying on + // the operating system's unspecified read_dir ordering. + let mut counted = std::collections::HashMap::new(); + let mut charged = std::collections::BTreeMap::::new(); + for (kind, path) in [(Kind::Superseded, &superseded), (Kind::Head, &head)] { + let bytes = + directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) + .unwrap(); + let entry = charged.entry(kind).or_default(); + entry.0 += 1; + entry.1 += bytes; + } + assert_eq!(charged.get(&Kind::Superseded), Some(&(1, 3))); + assert_eq!(charged.get(&Kind::Head), Some(&(1, 16))); + assert_eq!(counted.len(), 1, "one shared inode was visited twice"); + + // The production census also classifies receipt-bound children and + // must report one physical copy without charging retired bytes twice. + let reported = census_receipted_directory_bytes( + &root, + Kind::Unrecognized, + |_| ("owner", 64), + |_, receipt| match receipt { + Some(bytes) if bytes == b"head" => Kind::Head, + Some(bytes) if bytes == b"old" => Kind::Superseded, + _ => Kind::Unrecognized, + }, + ) + .unwrap(); + assert_eq!(reported.get(&Kind::Head), Some(&(1, 16))); + assert_eq!(reported.get(&Kind::Superseded), Some(&(1, 3))); + assert_eq!(reported.get(&Kind::Unrecognized), None); + assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 8b1b06206ae8ee9542bad85c860b055e171a9463 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:00:12 -0800 Subject: [PATCH 54/70] style(graph): normalize sealed hardlink regression wrapping Refs #4616 Keep the native test's method chain inside the repository rustfmt width. No production behavior or test assertion changes. --- crates/tracedecay-private-fs/src/capability_dir.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index a8768369cfe..0d1c8e55d15 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -966,9 +966,8 @@ mod tests { let mut counted = std::collections::HashMap::new(); let mut charged = std::collections::BTreeMap::::new(); for (kind, path) in [(Kind::Superseded, &superseded), (Kind::Head, &head)] { - let bytes = - directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) - .unwrap(); + let bytes = directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) + .unwrap(); let entry = charged.entry(kind).or_default(); entry.0 += 1; entry.1 += bytes; From 94b5d9fcc68f0242617e5e3a92acc92e1f840405 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:01:03 -0800 Subject: [PATCH 55/70] docs(graph): clarify shared-inode census accounting bounds Refs #4616 State that only multi-linked inode identities consume deduplication metadata, and explain the priority ordering used by the census. No behavior or test assertion changes. --- crates/tracedecay-private-fs/src/capability_dir.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 0d1c8e55d15..705c53ebbf3 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -392,6 +392,8 @@ pub fn install_child_directory_noreplace_nofollow( /// ambient paths. Unknown entries are counted but never traversed; only /// multiply linked inodes consume deduplication memory. Receipts are bounded /// and read from the same child whose bytes are subsequently charged. +/// Category ordering is authority ordering: lower keys receive shared +/// physical bytes before higher keys, independent of directory iteration. pub fn census_receipted_directory_bytes( root_path: &Path, unrecognized: K, @@ -405,8 +407,8 @@ pub fn census_receipted_directory_bytes( }; let mut totals = std::collections::BTreeMap::::new(); // Hard links belong to the highest-priority verified category, not the - // first entry returned by read_dir. Retain only multiply-linked identities - // so the census remains bounded independently of file count and size. + // first entry returned by read_dir. Only multiply-linked identities need + // tracking; memory depends on shared inodes, not file byte size. let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); for entry in root.read_dir(".")? { let entry = entry?; From b23f36fb15ef33d9c3e4f355dd5d808d0427e6d8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:43:28 -0800 Subject: [PATCH 56/70] style(graph): align native tests with pinned rustfmt Refs #4616 Apply the three exact formatting hunks reported by Bazel repository gates at PR head 94b5d9fc; no behavior, tests, or CI configuration changed. --- crates/tracedecay-graph-db/src/generation/spill.rs | 7 ++++--- crates/tracedecay-graph-db/src/runtime/tests.rs | 4 +++- crates/tracedecay-private-fs/src/capability_dir.rs | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 4a533d5e3b3..2efee498bcd 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -184,9 +184,10 @@ impl SpillDirectory { } let installed = tracedecay_private_fs::framed_log::rename_noreplace(staging.path(), path); if let Err(error) = installed { - let root = staging.path().parent().ok_or_else(|| { - GraphDbError::invalid("graph row spill directory has no parent") - })?; + let root = staging + .path() + .parent() + .ok_or_else(|| GraphDbError::invalid("graph row spill directory has no parent"))?; // On failure, only the exact owner receipt read from the opened // child authorizes recursive removal. A substituted directory, // even an ordinary real directory, must survive unchanged. diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index 8edccc786b9..dd15c12f819 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -93,7 +93,9 @@ fn lazy_open_rechecks_close_before_materializing_an_engine() { .unwrap() .unwrap(); assert_eq!( - entity.properties.get(&GraphPropertyName::new("name").unwrap()), + entity + .properties + .get(&GraphPropertyName::new("name").unwrap()), Some(&GraphProperty::String("retained".to_owned())) ); drop(snapshot); diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 705c53ebbf3..3f5dd0a7c3c 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -248,7 +248,10 @@ pub fn write_exclusive_pending_receipt_nofollow( #[cfg(unix)] let marker_identity = { use cap_std::fs::MetadataExt; - marker.metadata().ok().map(|metadata| (metadata.dev(), metadata.ino())) + marker + .metadata() + .ok() + .map(|metadata| (metadata.dev(), metadata.ino())) }; let written = marker.write_all(contents).and_then(|()| marker.sync_all()); drop(marker); From bca1e23adbef3b0e32f86dc345454bbdfc017c48 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:45:57 -0800 Subject: [PATCH 57/70] fix(graph): bound sealed census hard-link bookkeeping Refs #4616 Bound shared-inode metadata and reject a census as partial before its allocation scales indefinitely with millions of hard-linked files. Use fallible reservation and a filesystem regression for the refusal and the normal complete census. No CI or release settings changed. --- .../src/capability_dir.rs | 101 +++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 3f5dd0a7c3c..b77cd86f1ce 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -397,11 +397,32 @@ pub fn install_child_directory_noreplace_nofollow( /// and read from the same child whose bytes are subsequently charged. /// Category ordering is authority ordering: lower keys receive shared /// physical bytes before higher keys, independent of directory iteration. +/// Bound deduplication metadata independently of the size of the store: the +/// Doctor caller reports partial coverage rather than trusting a census +/// whose shared-inode inventory would grow without limit. +const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; + pub fn census_receipted_directory_bytes( + root_path: &Path, + unrecognized: K, + receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, +) -> io::Result> { + census_receipted_directory_bytes_bounded( + root_path, + unrecognized, + receipt_for, + classify, + MAX_SEALED_CENSUS_SHARED_INODES, + ) +} + +fn census_receipted_directory_bytes_bounded( root_path: &Path, unrecognized: K, mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + max_shared_inodes: usize, ) -> io::Result> { let root = match open_root_nofollow(root_path) { Ok(root) => root, @@ -429,7 +450,13 @@ pub fn census_receipted_directory_bytes( let bytes = if category == unrecognized { 0 } else { - directory_bytes_from_held_dir(child, category, &mut totals, &mut counted)? + directory_bytes_from_held_dir( + child, + category, + &mut totals, + &mut counted, + max_shared_inodes, + )? }; (category, bytes) } else { @@ -449,6 +476,7 @@ fn directory_bytes_from_held_dir( category: K, totals: &mut std::collections::BTreeMap, counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, + max_shared_inodes: usize, ) -> io::Result { let mut total = 0_u64; // Hold only the current ancestry chain. Opening every sibling before @@ -495,6 +523,18 @@ fn directory_bytes_from_held_dir( } continue; } + if counted.len() >= max_shared_inodes { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census shared-inode budget exhausted; coverage is partial", + )); + } + counted.try_reserve(1).map_err(|_| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census shared-inode allocation failed; coverage is partial", + ) + })?; counted.insert(key, (category, metadata.len())); } } @@ -971,8 +1011,14 @@ mod tests { let mut counted = std::collections::HashMap::new(); let mut charged = std::collections::BTreeMap::::new(); for (kind, path) in [(Kind::Superseded, &superseded), (Kind::Head, &head)] { - let bytes = directory_bytes_from_held_dir(open(path), kind, &mut charged, &mut counted) - .unwrap(); + let bytes = directory_bytes_from_held_dir( + open(path), + kind, + &mut charged, + &mut counted, + MAX_SEALED_CENSUS_SHARED_INODES, + ) + .unwrap(); let entry = charged.entry(kind).or_default(); entry.0 += 1; entry.1 += bytes; @@ -1000,6 +1046,55 @@ mod tests { assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); } + /// An adversarial store with many distinct hard-linked inodes must + /// decline its complete census before unbounded heap growth. A caller + /// can then truthfully mark coverage partial rather than report zeros as + /// a complete inventory. The ordinary production entrypoint still counts + /// all the same real files when within its budget. + #[cfg(unix)] + #[test] + fn sealed_census_refuses_excess_shared_inodes_without_losing_files() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("owner"), b"owned").unwrap(); + for index in 0..3 { + let source = generation.join(format!("payload-{index}")); + std::fs::write(&source, b"x").unwrap(); + std::fs::hard_link(&source, temp.path().join(format!("alias-{index}"))).unwrap(); + } + + let refused = census_receipted_directory_bytes_bounded( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + 2, + ) + .unwrap_err(); + assert_eq!(refused.kind(), io::ErrorKind::OutOfMemory, "{refused}"); + + let complete = census_receipted_directory_bytes( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + ) + .unwrap(); + assert_eq!(complete.get(&true), Some(&(1, 8))); + for index in 0..3 { + assert_eq!( + std::fs::read(generation.join(format!("payload-{index}"))).unwrap(), + b"x" + ); + assert_eq!( + std::fs::read(temp.path().join(format!("alias-{index}"))).unwrap(), + b"x" + ); + } + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 3c4fab3b7bbfcf9773ae94670a1aaa3d3fad81e4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:46:33 -0800 Subject: [PATCH 58/70] fix(graph): keep shared census budget portable Refs #4616 The bounded Unix inode census leaves its budget unused on other targets. Retain the existing non-Unix no-op binding to keep strict lint clean. --- crates/tracedecay-private-fs/src/capability_dir.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index b77cd86f1ce..426fba5da39 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -539,7 +539,7 @@ fn directory_bytes_from_held_dir( } } #[cfg(not(unix))] - let _ = &counted; + let _ = (&counted, max_shared_inodes); total = total.saturating_add(metadata.len()); } Ok(total) From d7bbee1d551773e1b1bf77ed56ee273ada627637 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:47:53 -0800 Subject: [PATCH 59/70] docs(graph): retain public sealed census API documentation Refs #4616 Attach the invariant documentation to the exported census API, not to the shared-inode memory ceiling. --- crates/tracedecay-private-fs/src/capability_dir.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 426fba5da39..5dc80d2e995 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -391,17 +391,17 @@ pub fn install_child_directory_noreplace_nofollow( rename_noreplace(&root, staging_name, &root, generation_name) } +/// Bound deduplication metadata independently of the size of the store. +/// The Doctor caller reports partial coverage instead of trusting a census +/// whose shared-inode inventory would otherwise grow without limit. +const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; + /// Census proof-bound children through held root/child capabilities, not /// ambient paths. Unknown entries are counted but never traversed; only /// multiply linked inodes consume deduplication memory. Receipts are bounded /// and read from the same child whose bytes are subsequently charged. /// Category ordering is authority ordering: lower keys receive shared /// physical bytes before higher keys, independent of directory iteration. -/// Bound deduplication metadata independently of the size of the store: the -/// Doctor caller reports partial coverage rather than trusting a census -/// whose shared-inode inventory would grow without limit. -const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; - pub fn census_receipted_directory_bytes( root_path: &Path, unrecognized: K, From 6027cbacfd7e53332053beeef4e6b172fc035b62 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:49:05 -0800 Subject: [PATCH 60/70] fix(graph): refuse missing registered sealed retirement Refs #4616 When a registered sealed store's root or generation directory was moved before receipt-verified deletion, no-follow cleanup returned Ok(false). That was mistaken for successful retirement, uninstalling its reader and marking generation state collected while durable bytes survived elsewhere. Only unregistered disk-only retries may treat an absent path as idempotent. A real sealed store regression moves the root and child independently, requires typed refusal and retained reader, then restores and reclaims both. --- .../tracedecay-graph-db/src/sealed_store.rs | 97 ++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index db76bd82740..831c31b8c56 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -965,9 +965,19 @@ fn retire_sealed_directory_checked( && expected_digest.is_none_or(|digest| receipt.recovered_digest == digest) }, ); - result - .map(|_| ()) - .map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error)) + let removed = + result.map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error))?; + // A resident reader carries exact proof that this artifact was installed. + // If its root or child disappeared before we acquired a deletion capability, + // physical retirement was not confirmed: the directory may have been moved + // aside and still contain the sole durable generation. Keep the reader + // registered so maintenance can retry rather than mark it collected. + if !removed && expected_digest.is_some() { + return Err(GraphDbError::unavailable( + "registered sealed generation retirement found no owned artifact", + )); + } + Ok(()) } /// Staging and invalid-artifact cleanup are best effort. Retiring an accepted @@ -4110,6 +4120,87 @@ mod build_tests { assert!(!store.directory.exists()); } + /// A missing pathname is not evidence that an installed generation's + /// durable bytes were removed. It may have been relocated between + /// registration and the retirement attempt. + #[test] + fn registered_retirement_refuses_disappeared_root_or_child_until_restored() { + let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); + for move_root in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let manifest = manifest(2, 1); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(check).unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Manifest(&manifest), + &identity, + &expected, + &database_path, + check, + ) + .unwrap(); + let locator = crate::lease::GenerationLocator::new( + identity.projection.clone(), + identity.generation.clone(), + ); + let parent = GraphDb::open(GraphDbOpenOptions { + location: GraphDbLocation::Persistent(database_path.clone()), + expected_format: GraphFormatVersion::current(), + durability: GraphDurability::WalSync, + cancellation: Arc::new(NeverCancelled), + }) + .unwrap(); + parent + .inner + .sealed_generations + .write() + .unwrap() + .insert(locator.clone(), Arc::clone(&store)); + + let root = sealed_store_root(&database_path); + let (original, relocated) = if move_root { + (root, temp.path().join("relocated-sealed-root")) + } else { + (store.directory.clone(), root.join("relocated-sealed-child")) + }; + std::fs::rename(&original, &relocated).unwrap(); + let surviving_receipt = if move_root { + relocated + .join(store.directory.file_name().unwrap()) + .join(SEALED_STORE_RECEIPT_FILE) + } else { + relocated.join(SEALED_STORE_RECEIPT_FILE) + }; + let receipt = std::fs::read(&surviving_receipt).unwrap(); + + let refused = parent.retire_sealed_generation_store(&locator); + assert!( + matches!(&refused, Err(GraphDbError::Unavailable { .. })), + "missing owned artifact must remain pending (root={move_root}): {refused:?}" + ); + assert_eq!( + std::fs::read(&surviving_receipt).unwrap(), + receipt, + "retirement must preserve the relocated sealed receipt" + ); + let retained = parent + .sealed_generation_reader(&locator) + .expect("failed retirement must retain its registered reader"); + assert!(Arc::ptr_eq(&retained, &store)); + drop(retained); + + // Retry against the restored owner must really delete the bytes + // and remove the reader. A permanent refusal cannot pass. + std::fs::rename(&relocated, &original).unwrap(); + parent.retire_sealed_generation_store(&locator).unwrap(); + assert!(parent.sealed_generation_reader(&locator).is_none()); + assert!(!store.directory.exists()); + // Disk-only duplicate notifications remain idempotent. + retire_sealed_directory_checked(&store.directory, &locator, None).unwrap(); + } + } + #[cfg(unix)] #[test] fn sealed_build_refuses_a_foreign_root_symlink() { From 9c3dfc482ae3143d6d29739632684706477f8f85 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:50:14 -0800 Subject: [PATCH 61/70] style(graph): align retirement error mapping with rustfmt Refs #4616 Keep the new typed missing-artifact retirement branch unchanged. --- crates/tracedecay-graph-db/src/sealed_store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 831c31b8c56..a36636f4119 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -965,8 +965,8 @@ fn retire_sealed_directory_checked( && expected_digest.is_none_or(|digest| receipt.recovered_digest == digest) }, ); - let removed = - result.map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error))?; + let removed = result + .map_err(|error| sealed_store_io_failure("sealed generation retirement refused", error))?; // A resident reader carries exact proof that this artifact was installed. // If its root or child disappeared before we acquired a deletion capability, // physical retirement was not confirmed: the directory may have been moved From b9798d7169bcc15f9ad19731d483be863c760621 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:50:59 -0800 Subject: [PATCH 62/70] test(graph): exercise relocated sealed paths on Unix Refs #4616 The native directory-rename regression uses Unix unlink/rename semantics. Keep cross-platform product behavior unchanged. --- crates/tracedecay-graph-db/src/sealed_store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index a36636f4119..7e2558c9e5e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -4123,6 +4123,7 @@ mod build_tests { /// A missing pathname is not evidence that an installed generation's /// durable bytes were removed. It may have been relocated between /// registration and the retirement attempt. + #[cfg(unix)] #[test] fn registered_retirement_refuses_disappeared_root_or_child_until_restored() { let check: &dyn Fn() -> Result<(), GraphDbError> = &|| Ok(()); From 1f3e3364cb0885c055cf26e9dccd0cc4b0029e88 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:47:19 -0800 Subject: [PATCH 63/70] fix(graph): import sealed retirement verifier in native test Refs #4616 The moved-sealed-root regression calls the production receipt-verified retirement helper from its nested test module without importing it. Both Linux Bazel graph-db unit compilation and strict Clippy fail with E0425 before the regression executes. Import the actual helper and keep the native relocation, refusal, restoration, and idempotency assertions. --- crates/tracedecay-graph-db/src/sealed_store.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7e2558c9e5e..c54bb1b1a83 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3508,9 +3508,9 @@ mod build_tests { MAX_SEALED_RECEIPT_BYTES, SEALED_STAGING_OWNER_FILE, SEALED_STORE_DATABASE_FILE, SEALED_STORE_RECEIPT_FILE, SealedRowSource, SealedStagingOwnerV1, build_or_open_sealed_store, build_sealed_container, census_sealed_store, - initial_sealed_staging_owner, persist_sealed_store_check, sealed_artifact_database_options, - sealed_copy_proof, sealed_generation_directory, sealed_store_root, - sweep_abandoned_sealed_staging, + initial_sealed_staging_owner, persist_sealed_store_check, retire_sealed_directory_checked, + sealed_artifact_database_options, sealed_copy_proof, sealed_generation_directory, + sealed_store_root, sweep_abandoned_sealed_staging, }; use crate::location::PersistentGraphStoreState; use crate::{ From 736779337edbefd5a640c37c1dff432e5079c2ec Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:05:39 -0800 Subject: [PATCH 64/70] test(graph): align graph-db unit tests with sealed retirement contracts Refs #4616 Unit tests never executed while compilation failed, so three assertions still described superseded contracts. A registered sealed generation whose artifact is absent is now an unconfirmed retirement (typed refusal), with only a disk-only retry idempotent. A close retry after a poisoned close stays DurabilityUncertain, not Closed, and the engine is still verified released. A prepared pending row spill survives drop by design, so the live-producer test installs it before checking name reuse. Co-Authored-By: Claude Sonnet 5.5 --- crates/tracedecay-graph-db/src/generation/spill.rs | 12 +++++++++--- crates/tracedecay-graph-db/src/runtime/tests.rs | 11 +++++++++-- crates/tracedecay-graph-db/src/sealed_store.rs | 9 ++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-graph-db/src/generation/spill.rs b/crates/tracedecay-graph-db/src/generation/spill.rs index 2efee498bcd..15f501d30c7 100644 --- a/crates/tracedecay-graph-db/src/generation/spill.rs +++ b/crates/tracedecay-graph-db/src/generation/spill.rs @@ -1474,7 +1474,7 @@ mod owner_receipt_tests { let stale = SpillDirectory::prepare(&previous_path, previous) .unwrap() .keep(); - let live = SpillDirectory::prepare(&final_path, current).unwrap(); + let mut live = SpillDirectory::prepare(&final_path, current).unwrap(); let peer_path = root.join(format!(".rows-{current}-1")); let _peer = SpillDirectory::create(peer_path.clone()).unwrap(); @@ -1482,8 +1482,14 @@ mod owner_receipt_tests { assert!(!stale.exists()); assert!(live.path().is_dir()); assert!(peer_path.is_dir()); - drop(live); - let _published = SpillDirectory::create(final_path.clone()).unwrap(); + // A prepared pending spill deliberately survives a drop; the live + // producer completes it by installing under its canonical name. + SpillDirectory::install(&mut live, &final_path, current).unwrap(); + let published = SpillDirectory(final_path.clone()); + assert!(final_path.is_dir()); + drop(published); + assert!(!final_path.exists(), "the owner releases its own spill"); + let _reused = SpillDirectory::create(final_path.clone()).unwrap(); assert!(final_path.is_dir()); } diff --git a/crates/tracedecay-graph-db/src/runtime/tests.rs b/crates/tracedecay-graph-db/src/runtime/tests.rs index dd15c12f819..879e362892b 100644 --- a/crates/tracedecay-graph-db/src/runtime/tests.rs +++ b/crates/tracedecay-graph-db/src/runtime/tests.rs @@ -136,10 +136,17 @@ fn owner_close_releases_the_physical_database_after_durability_uncertainty() { .to_owned(), }) ); + assert!( + !owner.engine_is_open().unwrap(), + "the poisoned close must release the physical database" + ); assert_eq!( owner.close(), - Err(GraphDbError::Closed), - "the poisoned close must release the physical database before a second close" + Err(GraphDbError::DurabilityUncertain { + message: "the handle was poisoned after an observed post-commit persistence failure" + .to_owned(), + }), + "a retry after a poisoned close must stay uncertain, never idempotent Closed" ); } diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index c54bb1b1a83..7b2cf45ee89 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3386,7 +3386,14 @@ mod receipt_budget_tests { !directory.exists(), "verified retired generation is physically removed" ); - retire_sealed_directory_checked(&directory, &locator, Some("digest")).unwrap(); + // A registered reader proves the artifact was installed, so its + // absence is unconfirmed retirement. Only a disk-only retry with no + // resident reader may treat an already-absent directory as done. + assert!( + retire_sealed_directory_checked(&directory, &locator, Some("digest")).is_err(), + "a registered generation must not report retirement of an absent artifact" + ); + retire_sealed_directory_checked(&directory, &locator, None).unwrap(); } #[cfg(unix)] From 6939336af6c195e05642b6d94e20cc15f57abec7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:18:13 -0800 Subject: [PATCH 65/70] fix(graph): release fully observed hard-link census identities Refs #4616 A sealed store with many disjoint hard-link pairs exhausted the fixed shared-inode budget although only one dedup identity was unresolved at a time. Track the remaining filesystem-reported links and discard an identity after the final alias. Preserve ownership-priority byte transfer and the fail-closed bound for links outside the scanned store. Native filesystem regression walks four disjoint hard-link pairs with a one-entry unresolved budget; the old implementation fails on pair two. --- .../src/capability_dir.rs | 65 +++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 5dc80d2e995..db2191690c6 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -432,8 +432,9 @@ fn census_receipted_directory_bytes_bounded( let mut totals = std::collections::BTreeMap::::new(); // Hard links belong to the highest-priority verified category, not the // first entry returned by read_dir. Only multiply-linked identities need - // tracking; memory depends on shared inodes, not file byte size. - let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); + // tracking; memory depends on unresolved shared inodes, not the total + // number of linked files visited or their byte size. + let mut counted = std::collections::HashMap::<(u64, u64), (K, u64, u64)>::new(); for entry in root.read_dir(".")? { let entry = entry?; let name = entry.file_name(); @@ -475,7 +476,7 @@ fn directory_bytes_from_held_dir( directory: Dir, category: K, totals: &mut std::collections::BTreeMap, - counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, + counted: &mut std::collections::HashMap<(u64, u64), (K, u64, u64)>, max_shared_inodes: usize, ) -> io::Result { let mut total = 0_u64; @@ -501,7 +502,9 @@ fn directory_bytes_from_held_dir( use cap_std::fs::MetadataExt; if metadata.nlink() > 1 { let key = (metadata.dev(), metadata.ino()); - if let Some((previous_kind, charged_bytes)) = counted.get_mut(&key) { + if let Some((previous_kind, charged_bytes, remaining_links)) = + counted.get_mut(&key) + { // A lower-ordered category has stronger ownership. A // superseded generation encountered first cannot claim // hard-linked physical bytes from a later live head. @@ -521,6 +524,15 @@ fn directory_bytes_from_held_dir( total = total.saturating_add(*charged_bytes); *previous_kind = category; } + // An immutable hard-link group no longer needs its key + // after every filesystem-reported link has been seen. + // Retaining completed groups makes a store with many + // disjoint pairs exhaust the budget despite bounded + // concurrent deduplication demand. + *remaining_links -= 1; + if *remaining_links == 0 { + counted.remove(&key); + } continue; } if counted.len() >= max_shared_inodes { @@ -535,7 +547,7 @@ fn directory_bytes_from_held_dir( "sealed census shared-inode allocation failed; coverage is partial", ) })?; - counted.insert(key, (category, metadata.len())); + counted.insert(key, (category, metadata.len(), metadata.nlink() - 1)); } } #[cfg(not(unix))] @@ -1025,7 +1037,10 @@ mod tests { } assert_eq!(charged.get(&Kind::Superseded), Some(&(1, 3))); assert_eq!(charged.get(&Kind::Head), Some(&(1, 16))); - assert_eq!(counted.len(), 1, "one shared inode was visited twice"); + assert!( + counted.is_empty(), + "a fully observed hard-link group must free its deduplication slot" + ); // The production census also classifies receipt-bound children and // must report one physical copy without charging retired bytes twice. @@ -1046,6 +1061,44 @@ mod tests { assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); } + /// Completed hard-link groups need no further identity retention. A + /// bounded census must still cover many disjoint alias pairs whose total + /// exceeds its simultaneous unresolved-link budget. + #[cfg(unix)] + #[test] + fn complete_link_pairs_reuse_census_deduplication_budget() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let generation = root.join("generation"); + std::fs::create_dir_all(&generation).unwrap(); + std::fs::write(generation.join("owner"), b"owned").unwrap(); + for index in 0..4 { + let column = generation.join(format!("column-{index}")); + std::fs::create_dir(&column).unwrap(); + let original = column.join("original"); + std::fs::write(&original, b"rows").unwrap(); + std::fs::hard_link(&original, column.join("alias")).unwrap(); + } + + // Each column is fully traversed before the next one is opened. + // Previously the four completed identities accumulated and this + // real filesystem walk failed at the second column's first link. + let census = census_receipted_directory_bytes_bounded( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + 1, + ) + .unwrap(); + assert_eq!(census.get(&true), Some(&(1, 21))); + for index in 0..4 { + let column = generation.join(format!("column-{index}")); + assert_eq!(std::fs::read(column.join("original")).unwrap(), b"rows"); + assert_eq!(std::fs::read(column.join("alias")).unwrap(), b"rows"); + } + } + /// An adversarial store with many distinct hard-linked inodes must /// decline its complete census before unbounded heap growth. A caller /// can then truthfully mark coverage partial rather than report zeros as From b1a9acd1067832281e32162418ee6c82c6ae1552 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:07:01 -0800 Subject: [PATCH 66/70] test(graph): fix canonical corruption paths and formatting Compare corruption census paths with the physical canonical path selected by the registry, including macOS temporary roots reached through /var aliases. Apply the pinned formatter to the new hard-link census branch. Bazel: graph-db 198 passed with one pre-existing ignored; private-fs 49 passed; repository rustfmt and strict Clippy on both unit targets passed. Concurrent-publication census correctness remains a separate unresolved finding. --- crates/tracedecay-graph-db/src/registry.rs | 4 +++- crates/tracedecay-private-fs/src/capability_dir.rs | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-graph-db/src/registry.rs b/crates/tracedecay-graph-db/src/registry.rs index 2cb9827ec44..808696092f3 100644 --- a/crates/tracedecay-graph-db/src/registry.rs +++ b/crates/tracedecay-graph-db/src/registry.rs @@ -2806,7 +2806,9 @@ mod tests { registry .retain_verification_fault_for_lease(&lease, &fault) .unwrap(); - let path = corrupt_registration.canonical_path().to_path_buf(); + // TempDir may live under a symlinked temporary root. The registry + // reports the physical canonical path selected at admission. + let path = std::fs::canonicalize(corrupt_registration.canonical_path()).unwrap(); let bytes = std::fs::read(&path).unwrap(); for _ in 0..2 { diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index db2191690c6..255132c2644 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -502,8 +502,7 @@ fn directory_bytes_from_held_dir( use cap_std::fs::MetadataExt; if metadata.nlink() > 1 { let key = (metadata.dev(), metadata.ino()); - if let Some((previous_kind, charged_bytes, remaining_links)) = - counted.get_mut(&key) + if let Some((previous_kind, charged_bytes, remaining_links)) = counted.get_mut(&key) { // A lower-ordered category has stronger ownership. A // superseded generation encountered first cannot claim From 580c546ff371374ab6812d8ae6aed23d735ce9ae Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:24:30 -0800 Subject: [PATCH 67/70] fix(fs): retain census identities across sealed publication A filesystem link count does not close an inode group. Layer publication can add aliases after an observed group completes, including files originally visited with one link. Retain bounded inode evidence for the whole census and preserve priority transfer; refuse incomplete coverage when that evidence fills. Two native regressions publish real layered artifacts between visits of one census. Both reproduced a 42,246-byte excess charge before the fix and now match independent unique-byte and category accounting. Keep the disjoint-pair fixture with exact accounting at sufficient capacity and typed refusal below it. Validation: graph-db unit suite 200 passed, one pre-existing ignored; private-fs unit suite 49 passed; strict Bazel Clippy and pinned rustfmt checks pass for both. The 65,536-entry bound remains unmeasured; this does not establish retirement or CPU incident acceptance. --- .../tracedecay-graph-db/src/sealed_store.rs | 173 +++++++++++++++ .../src/capability_dir.rs | 198 +++++++++++------- 2 files changed, 290 insertions(+), 81 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 7b2cf45ee89..914d1dd955e 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3617,6 +3617,179 @@ mod build_tests { .unwrap() } + /// Publish a real layered artifact between visits of the same census. + /// Both a formerly single link and an exhausted two-link group must + /// remain one physical charge owned by the newly visited head. + #[cfg(unix)] + #[test] + fn sealed_census_counts_concurrently_published_single_base_link_once() { + census_with_concurrent_layer_publication(0); + } + + #[cfg(unix)] + #[test] + fn sealed_census_counts_concurrently_published_completed_link_group_once() { + census_with_concurrent_layer_publication(1); + } + + #[cfg(unix)] + fn census_with_concurrent_layer_publication(initial_layers: u64) { + use std::os::unix::fs::MetadataExt; + + use tracedecay_private_fs::capability_dir::test_support::census_in_entry_order; + + let temp = tempfile::tempdir().unwrap(); + let database_path = temp.path().join("source.grafeo"); + let database = open_source(&database_path); + let manifest = manifest(3, 2); + let identity = manifest.identity(); + let expected = manifest.expected_recovered_digest(&|| Ok(())).unwrap(); + let mut spill = database + .generation_row_spill(manifest.projection.clone()) + .unwrap(); + spill + .push_batch( + manifest.entities.clone(), + manifest.relations.clone(), + &|| Ok(()), + ) + .unwrap(); + std::fs::write(spill.attachment_path(), b"producer inputs").unwrap(); + let spilled = spill.finish(identity.clone(), &|| Ok(())).unwrap(); + let (cold, _) = build_or_open_sealed_store( + SealedRowSource::Spilled(&spilled), + &identity, + &expected, + &database_path, + &|| Ok(()), + ) + .unwrap(); + drop(spilled); + let base = cold.sealed_base(&|| Ok(())).unwrap().unwrap(); + let publish = |generation: &str| { + let mut identity = identity.clone(); + identity.generation = GraphGenerationId::new(generation).unwrap(); + let layered = database + .layered_row_spill(identity.projection.clone(), base.clone()) + .unwrap() + .finish(identity.clone(), &|| Ok(())) + .unwrap(); + let (store, _) = build_or_open_sealed_store( + SealedRowSource::Layered(&layered), + &identity, + layered.expected_recovered_digest(), + &database_path, + &|| Ok(()), + ) + .unwrap(); + drop(layered); + store + }; + let mut stores = vec![cold]; + if initial_layers == 1 { + stores.push(publish("generation:child1")); + } + let root = sealed_store_root(&database_path); + let original = stores[0].directory.join(SEALED_STORE_DATABASE_FILE); + let original_bytes = std::fs::read(&original).unwrap(); + assert_eq!( + std::fs::metadata(&original).unwrap().nlink(), + 1 + initial_layers + ); + let initial_names: Vec<_> = stores + .iter() + .map(|store| store.directory.file_name().unwrap().to_owned()) + .collect(); + let mut initial_names = initial_names.into_iter(); + let receipts: Vec<_> = stores + .iter() + .map(|store| { + let path = store.directory.join(SEALED_STORE_RECEIPT_FILE); + let bytes = std::fs::read(&path).unwrap(); + (path, bytes) + }) + .collect(); + let mut published = false; + // The entry source determines only scheduling: all receipt reads, + // capability opens and accounting use the production census core. + let entries = std::iter::from_fn(|| { + if let Some(name) = initial_names.next() { + return Some(Ok(name)); + } + if published { + return None; + } + let child = publish("generation:child2"); + let name = child.directory.file_name().unwrap().to_owned(); + stores.push(child); + published = true; + Some(Ok(name)) + }); + let totals = census_in_entry_order( + &root, + entries, + 2_u8, + |_| (SEALED_STORE_RECEIPT_FILE, MAX_SEALED_RECEIPT_BYTES), + |_, receipt| { + let receipt: super::SealedStoreReceiptV1 = + serde_json::from_slice(receipt.unwrap()).unwrap(); + if receipt.generation == "generation:child2" { + 0 + } else { + 1 + } + }, + ) + .unwrap(); + assert!(published); + assert_eq!( + std::fs::metadata(&original).unwrap().nlink(), + 2 + initial_layers + ); + assert_eq!(std::fs::read(&original).unwrap(), original_bytes); + for (path, bytes) in receipts { + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + // Independent native identity census of the completed real artifacts. + // Compare each category as well as the unique physical total. + let mut physical = BTreeMap::<_, (u8, u64)>::new(); + for (index, store) in stores.iter().enumerate() { + let kind = if index + 1 == stores.len() { 0 } else { 1 }; + for entry in std::fs::read_dir(&store.directory).unwrap() { + let metadata = entry.unwrap().metadata().unwrap(); + assert!(metadata.is_file()); + physical + .entry((metadata.dev(), metadata.ino())) + .and_modify(|(owner, _)| *owner = (*owner).min(kind)) + .or_insert((kind, metadata.len())); + } + } + for kind in [0, 1] { + let bytes: u64 = physical + .values() + .filter(|(owner, _)| *owner == kind) + .map(|(_, bytes)| bytes) + .sum(); + assert_eq!( + totals[&kind].1, bytes, + "initial layers {initial_layers}, category {kind}" + ); + } + assert_eq!(totals[&0].0, 1); + assert_eq!(totals[&1].0, 1 + initial_layers); + let census = census_sealed_store( + &database_path, + &BTreeSet::from(["generation:child2".to_owned()]), + ) + .unwrap(); + assert_eq!(census.head_bytes, totals[&0].1); + assert_eq!(census.superseded_bytes, totals[&1].1); + for store in stores { + store.close().unwrap(); + } + database.close().unwrap(); + } + #[cfg(unix)] #[test] fn sealed_census_only_attributes_verified_abandoned_staging() { diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index 255132c2644..f4c179e072b 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -393,13 +393,15 @@ pub fn install_child_directory_noreplace_nofollow( /// Bound deduplication metadata independently of the size of the store. /// The Doctor caller reports partial coverage instead of trusting a census -/// whose shared-inode inventory would otherwise grow without limit. -const MAX_SEALED_CENSUS_SHARED_INODES: usize = 65_536; +/// whose inode inventory would otherwise grow without limit. +const MAX_SEALED_CENSUS_INODES: usize = 65_536; /// Census proof-bound children through held root/child capabilities, not -/// ambient paths. Unknown entries are counted but never traversed; only -/// multiply linked inodes consume deduplication memory. Receipts are bounded -/// and read from the same child whose bytes are subsequently charged. +/// ambient paths. Unknown entries are counted but never traversed. On Unix, +/// every visited inode consumes bounded deduplication memory until the walk +/// ends: publication can add links even to a formerly single-linked file. +/// Exhausting that evidence budget returns `OutOfMemory`, never complete +/// totals. Receipts are bounded and read from the child being charged. /// Category ordering is authority ordering: lower keys receive shared /// physical bytes before higher keys, independent of directory iteration. pub fn census_receipted_directory_bytes( @@ -413,32 +415,50 @@ pub fn census_receipted_directory_bytes( unrecognized, receipt_for, classify, - MAX_SEALED_CENSUS_SHARED_INODES, + MAX_SEALED_CENSUS_INODES, ) } fn census_receipted_directory_bytes_bounded( root_path: &Path, unrecognized: K, - mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), - mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, - max_shared_inodes: usize, + receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + max_inodes: usize, ) -> io::Result> { let root = match open_root_nofollow(root_path) { Ok(root) => root, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Default::default()), Err(error) => return Err(error), }; + census_receipted_children( + &root, + root.read_dir(".")? + .map(|entry| entry.map(|entry| entry.file_name())), + unrecognized, + receipt_for, + classify, + max_inodes, + ) +} + +fn census_receipted_children( + root: &Dir, + entries: impl Iterator>, + unrecognized: K, + mut receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + mut classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + max_inodes: usize, +) -> io::Result> { let mut totals = std::collections::BTreeMap::::new(); // Hard links belong to the highest-priority verified category, not the - // first entry returned by read_dir. Only multiply-linked identities need - // tracking; memory depends on unresolved shared inodes, not the total - // number of linked files visited or their byte size. - let mut counted = std::collections::HashMap::<(u64, u64), (K, u64, u64)>::new(); - for entry in root.read_dir(".")? { - let entry = entry?; - let name = entry.file_name(); - let child = if entry.file_type()?.is_dir() { + // first entry returned by read_dir. A link count is not a closed group: + // concurrent publication can add an alias after its last observed link. + let mut counted = std::collections::HashMap::<(u64, u64), (K, u64)>::new(); + for name in entries { + let name = name?; + single_component(&name)?; + let child = if root.symlink_metadata(&name)?.is_dir() { root.open_dir_nofollow(&name).ok() } else { None @@ -456,7 +476,7 @@ fn census_receipted_directory_bytes_bounded( category, &mut totals, &mut counted, - max_shared_inodes, + max_inodes, )? }; (category, bytes) @@ -470,14 +490,40 @@ fn census_receipted_directory_bytes_bounded( Ok(totals) } +/// Deterministic native scheduling for consumer publication regressions. +/// Receipt validation, held capabilities and inode accounting are shared with +/// the public census; only the filesystem's unspecified entry order is supplied. +#[cfg(feature = "test-helpers")] +pub mod test_support { + use super::{OsStr, OsString, Path, census_receipted_children, io, open_root_nofollow}; + + pub fn census_in_entry_order( + root_path: &Path, + entries: impl Iterator>, + unrecognized: K, + receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), + classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, + ) -> io::Result> { + let root = open_root_nofollow(root_path)?; + census_receipted_children( + &root, + entries, + unrecognized, + receipt_for, + classify, + super::MAX_SEALED_CENSUS_INODES, + ) + } +} + /// Walk no-follow entries relative to the same held child, even after its /// old pathname has been renamed. The stack scales with directory depth. fn directory_bytes_from_held_dir( directory: Dir, category: K, totals: &mut std::collections::BTreeMap, - counted: &mut std::collections::HashMap<(u64, u64), (K, u64, u64)>, - max_shared_inodes: usize, + counted: &mut std::collections::HashMap<(u64, u64), (K, u64)>, + max_inodes: usize, ) -> io::Result { let mut total = 0_u64; // Hold only the current ancestry chain. Opening every sibling before @@ -500,57 +546,45 @@ fn directory_bytes_from_held_dir( #[cfg(unix)] { use cap_std::fs::MetadataExt; - if metadata.nlink() > 1 { - let key = (metadata.dev(), metadata.ino()); - if let Some((previous_kind, charged_bytes, remaining_links)) = counted.get_mut(&key) - { - // A lower-ordered category has stronger ownership. A - // superseded generation encountered first cannot claim - // hard-linked physical bytes from a later live head. - if category < *previous_kind { - let previous = totals.get_mut(&*previous_kind).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "hard-linked sealed bytes lack their prior category", - ) - })?; - previous.1 = previous.1.checked_sub(*charged_bytes).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "hard-linked sealed bytes exceed their prior category", - ) - })?; - total = total.saturating_add(*charged_bytes); - *previous_kind = category; - } - // An immutable hard-link group no longer needs its key - // after every filesystem-reported link has been seen. - // Retaining completed groups makes a store with many - // disjoint pairs exhaust the budget despite bounded - // concurrent deduplication demand. - *remaining_links -= 1; - if *remaining_links == 0 { - counted.remove(&key); - } - continue; + let key = (metadata.dev(), metadata.ino()); + if let Some((previous_kind, charged_bytes)) = counted.get_mut(&key) { + // A lower-ordered category has stronger ownership. A + // superseded generation encountered first cannot claim + // hard-linked physical bytes from a later live head. + if category < *previous_kind { + let previous = totals.get_mut(&*previous_kind).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes lack their prior category", + ) + })?; + previous.1 = previous.1.checked_sub(*charged_bytes).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "hard-linked sealed bytes exceed their prior category", + ) + })?; + total = total.saturating_add(*charged_bytes); + *previous_kind = category; } - if counted.len() >= max_shared_inodes { - return Err(io::Error::new( - io::ErrorKind::OutOfMemory, - "sealed census shared-inode budget exhausted; coverage is partial", - )); - } - counted.try_reserve(1).map_err(|_| { - io::Error::new( - io::ErrorKind::OutOfMemory, - "sealed census shared-inode allocation failed; coverage is partial", - ) - })?; - counted.insert(key, (category, metadata.len(), metadata.nlink() - 1)); + continue; + } + if counted.len() >= max_inodes { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census inode budget exhausted; coverage is partial", + )); } + counted.try_reserve(1).map_err(|_| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "sealed census inode allocation failed; coverage is partial", + ) + })?; + counted.insert(key, (category, metadata.len())); } #[cfg(not(unix))] - let _ = (&counted, max_shared_inodes); + let _ = (&counted, max_inodes); total = total.saturating_add(metadata.len()); } Ok(total) @@ -1027,7 +1061,7 @@ mod tests { kind, &mut charged, &mut counted, - MAX_SEALED_CENSUS_SHARED_INODES, + MAX_SEALED_CENSUS_INODES, ) .unwrap(); let entry = charged.entry(kind).or_default(); @@ -1036,10 +1070,6 @@ mod tests { } assert_eq!(charged.get(&Kind::Superseded), Some(&(1, 3))); assert_eq!(charged.get(&Kind::Head), Some(&(1, 16))); - assert!( - counted.is_empty(), - "a fully observed hard-link group must free its deduplication slot" - ); // The production census also classifies receipt-bound children and // must report one physical copy without charging retired bytes twice. @@ -1060,12 +1090,12 @@ mod tests { assert_eq!(std::fs::read(head.join("shared")).unwrap(), b"shared-bytes"); } - /// Completed hard-link groups need no further identity retention. A - /// bounded census must still cover many disjoint alias pairs whose total - /// exceeds its simultaneous unresolved-link budget. + /// Disjoint pairs still require evidence for the full walk: a publisher + /// can add another alias after any pair completes. A small budget must + /// refuse; a sufficient budget must count every pair exactly once. #[cfg(unix)] #[test] - fn complete_link_pairs_reuse_census_deduplication_budget() { + fn complete_link_pairs_require_retained_census_evidence() { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("sealed"); let generation = root.join("generation"); @@ -1079,15 +1109,21 @@ mod tests { std::fs::hard_link(&original, column.join("alias")).unwrap(); } - // Each column is fully traversed before the next one is opened. - // Previously the four completed identities accumulated and this - // real filesystem walk failed at the second column's first link. + let refused = census_receipted_directory_bytes_bounded( + &root, + false, + |_| ("owner", 64), + |_, receipt| receipt == Some(b"owned".as_slice()), + 2, + ) + .unwrap_err(); + assert_eq!(refused.kind(), io::ErrorKind::OutOfMemory, "{refused}"); let census = census_receipted_directory_bytes_bounded( &root, false, |_| ("owner", 64), |_, receipt| receipt == Some(b"owned".as_slice()), - 1, + 5, ) .unwrap(); assert_eq!(census.get(&true), Some(&(1, 21))); @@ -1122,7 +1158,7 @@ mod tests { false, |_| ("owner", 64), |_, receipt| receipt == Some(b"owned".as_slice()), - 2, + 3, ) .unwrap_err(); assert_eq!(refused.kind(), io::ErrorKind::OutOfMemory, "{refused}"); From cc772b303d9616031a6bba2392fd2d3cf7225dde Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:41:43 -0800 Subject: [PATCH 68/70] test(graph): keep census scheduling private and preserve layered proof Refs #4616 Keep deterministic concurrent hard-link publication coverage in the filesystem census's private unit tests instead of exporting a feature-gated production API. Retain real layered-generation coverage through the public census, and move Unix-only metadata imports to module scope. --- .../tracedecay-graph-db/src/sealed_store.rs | 83 ++++---------- .../src/capability_dir.rs | 105 +++++++++++++----- 2 files changed, 101 insertions(+), 87 deletions(-) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 914d1dd955e..69f773ae1a3 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3506,6 +3506,8 @@ mod receipt_budget_tests { #[cfg(test)] mod build_tests { use std::collections::{BTreeMap, BTreeSet}; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3617,27 +3619,23 @@ mod build_tests { .unwrap() } - /// Publish a real layered artifact between visits of the same census. - /// Both a formerly single link and an exhausted two-link group must - /// remain one physical charge owned by the newly visited head. + /// Real layered generations share one physical base. The production + /// census must attribute those hard links to the live head, regardless + /// of how many older layered generations already reference the base. #[cfg(unix)] #[test] - fn sealed_census_counts_concurrently_published_single_base_link_once() { - census_with_concurrent_layer_publication(0); + fn sealed_census_counts_published_single_base_link_once() { + census_with_layered_publication(0); } #[cfg(unix)] #[test] - fn sealed_census_counts_concurrently_published_completed_link_group_once() { - census_with_concurrent_layer_publication(1); + fn sealed_census_counts_published_multiple_base_links_once() { + census_with_layered_publication(1); } #[cfg(unix)] - fn census_with_concurrent_layer_publication(initial_layers: u64) { - use std::os::unix::fs::MetadataExt; - - use tracedecay_private_fs::capability_dir::test_support::census_in_entry_order; - + fn census_with_layered_publication(initial_layers: u64) { let temp = tempfile::tempdir().unwrap(); let database_path = temp.path().join("source.grafeo"); let database = open_source(&database_path); @@ -3689,18 +3687,12 @@ mod build_tests { if initial_layers == 1 { stores.push(publish("generation:child1")); } - let root = sealed_store_root(&database_path); let original = stores[0].directory.join(SEALED_STORE_DATABASE_FILE); let original_bytes = std::fs::read(&original).unwrap(); assert_eq!( std::fs::metadata(&original).unwrap().nlink(), 1 + initial_layers ); - let initial_names: Vec<_> = stores - .iter() - .map(|store| store.directory.file_name().unwrap().to_owned()) - .collect(); - let mut initial_names = initial_names.into_iter(); let receipts: Vec<_> = stores .iter() .map(|store| { @@ -3709,39 +3701,7 @@ mod build_tests { (path, bytes) }) .collect(); - let mut published = false; - // The entry source determines only scheduling: all receipt reads, - // capability opens and accounting use the production census core. - let entries = std::iter::from_fn(|| { - if let Some(name) = initial_names.next() { - return Some(Ok(name)); - } - if published { - return None; - } - let child = publish("generation:child2"); - let name = child.directory.file_name().unwrap().to_owned(); - stores.push(child); - published = true; - Some(Ok(name)) - }); - let totals = census_in_entry_order( - &root, - entries, - 2_u8, - |_| (SEALED_STORE_RECEIPT_FILE, MAX_SEALED_RECEIPT_BYTES), - |_, receipt| { - let receipt: super::SealedStoreReceiptV1 = - serde_json::from_slice(receipt.unwrap()).unwrap(); - if receipt.generation == "generation:child2" { - 0 - } else { - 1 - } - }, - ) - .unwrap(); - assert!(published); + stores.push(publish("generation:child2")); assert_eq!( std::fs::metadata(&original).unwrap().nlink(), 2 + initial_layers @@ -3764,26 +3724,27 @@ mod build_tests { .or_insert((kind, metadata.len())); } } + let census = census_sealed_store( + &database_path, + &BTreeSet::from(["generation:child2".to_owned()]), + ) + .unwrap(); for kind in [0, 1] { let bytes: u64 = physical .values() .filter(|(owner, _)| *owner == kind) .map(|(_, bytes)| bytes) .sum(); + let observed = if kind == 0 { + census.head_bytes + } else { + census.superseded_bytes + }; assert_eq!( - totals[&kind].1, bytes, + observed, bytes, "initial layers {initial_layers}, category {kind}" ); } - assert_eq!(totals[&0].0, 1); - assert_eq!(totals[&1].0, 1 + initial_layers); - let census = census_sealed_store( - &database_path, - &BTreeSet::from(["generation:child2".to_owned()]), - ) - .unwrap(); - assert_eq!(census.head_bytes, totals[&0].1); - assert_eq!(census.superseded_bytes, totals[&1].1); for store in stores { store.close().unwrap(); } diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index f4c179e072b..c81e789311a 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -490,32 +490,6 @@ fn census_receipted_children( Ok(totals) } -/// Deterministic native scheduling for consumer publication regressions. -/// Receipt validation, held capabilities and inode accounting are shared with -/// the public census; only the filesystem's unspecified entry order is supplied. -#[cfg(feature = "test-helpers")] -pub mod test_support { - use super::{OsStr, OsString, Path, census_receipted_children, io, open_root_nofollow}; - - pub fn census_in_entry_order( - root_path: &Path, - entries: impl Iterator>, - unrecognized: K, - receipt_for: impl FnMut(&OsStr) -> (&'static str, usize), - classify: impl FnMut(&OsStr, Option<&[u8]>) -> K, - ) -> io::Result> { - let root = open_root_nofollow(root_path)?; - census_receipted_children( - &root, - entries, - unrecognized, - receipt_for, - classify, - super::MAX_SEALED_CENSUS_INODES, - ) - } -} - /// Walk no-follow entries relative to the same held child, even after its /// old pathname has been renamed. The stack scales with directory depth. fn directory_bytes_from_held_dir( @@ -801,6 +775,8 @@ pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(test)] mod replacement_tests { use std::io::Read; + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; use super::*; @@ -1183,6 +1159,83 @@ mod tests { } } + /// A link group is not closed by its observed nlink count: a layered + /// publisher can add another alias after the census has visited every + /// old link. The head must inherit the one physical charge in both cases. + #[cfg(unix)] + #[test] + fn census_retains_inode_identity_across_later_link_publication() { + for initial_peer in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("sealed"); + let old = root.join("old"); + std::fs::create_dir_all(&old).unwrap(); + std::fs::write(old.join("owner"), b"old").unwrap(); + let base = old.join("shared"); + std::fs::write(&base, b"rows").unwrap(); + let mut initial = vec![OsString::from("old")]; + if initial_peer { + let peer = root.join("old-peer"); + std::fs::create_dir(&peer).unwrap(); + std::fs::write(peer.join("owner"), b"old").unwrap(); + std::fs::hard_link(&base, peer.join("shared")).unwrap(); + initial.push(OsString::from("old-peer")); + } + assert_eq!( + std::fs::metadata(&base).unwrap().nlink(), + if initial_peer { 2 } else { 1 }, + ); + let root_dir = open_root_nofollow(&root).unwrap(); + let mut names = initial.into_iter(); + let mut published = false; + let entries = std::iter::from_fn(|| { + if let Some(name) = names.next() { + return Some(Ok(name)); + } + if published { + return None; + } + let head = root.join("head"); + std::fs::create_dir(&head).unwrap(); + std::fs::write(head.join("owner"), b"head").unwrap(); + std::fs::hard_link(&base, head.join("shared")).unwrap(); + published = true; + Some(Ok(OsString::from("head"))) + }); + let census = census_receipted_children( + &root_dir, + entries, + 2_u8, + |_| ("owner", 16), + |_, receipt| { + if receipt == Some(b"head".as_slice()) { + 0 + } else if receipt == Some(b"old".as_slice()) { + 1 + } else { + 2 + } + }, + MAX_SEALED_CENSUS_INODES, + ) + .unwrap(); + assert!(published, "the head must have been published mid-census"); + assert_eq!( + std::fs::metadata(&base).unwrap().nlink(), + if initial_peer { 3 } else { 2 }, + ); + assert_eq!(census.get(&0), Some(&(1, 8)), "live head owns shared bytes"); + assert_eq!( + census.get(&1), + Some(&(if initial_peer { 2 } else { 1 }, if initial_peer { 6 } else { 3 })), + "old generations retain only their own receipts" + ); + assert!(!census.contains_key(&2)); + assert_eq!(std::fs::read(root.join("head/shared")).unwrap(), b"rows"); + assert_eq!(std::fs::read(&base).unwrap(), b"rows"); + } + } + #[test] fn staged_generation_install_preserves_an_occupied_empty_directory() { let temp = tempfile::tempdir().unwrap(); From 80977b3e6d6264e5a2dd6a48bebde2762176dcc2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:43:16 -0800 Subject: [PATCH 69/70] test(fs): import Unix inode metadata in census test module Refs #4616 Keep the private concurrent-publication regression compilable by placing the filesystem metadata trait in the module that owns that test, rather than the unrelated marker-replacement tests. --- crates/tracedecay-private-fs/src/capability_dir.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-private-fs/src/capability_dir.rs b/crates/tracedecay-private-fs/src/capability_dir.rs index c81e789311a..734a209b8ff 100644 --- a/crates/tracedecay-private-fs/src/capability_dir.rs +++ b/crates/tracedecay-private-fs/src/capability_dir.rs @@ -775,8 +775,6 @@ pub fn remove_open_dir_all_nofollow(directory: Dir) -> io::Result<()> { #[cfg(test)] mod replacement_tests { use std::io::Read; - #[cfg(unix)] - use std::os::unix::fs::MetadataExt; use super::*; @@ -931,6 +929,9 @@ mod replacement_tests { any(target_os = "macos", all(target_os = "linux", target_env = "gnu")) ))] mod tests { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + use cap_std::ambient_authority; use super::*; From 6ac18619cace62957cabbb11cee95b9cdf1e41b9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:45:07 -0800 Subject: [PATCH 70/70] test(graph): assert layered census generation counts Refs #4616 The real layered-generation regression must require a single live head, all retired siblings and no unrecognized artifacts, not only byte totals. --- crates/tracedecay-graph-db/src/sealed_store.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/tracedecay-graph-db/src/sealed_store.rs b/crates/tracedecay-graph-db/src/sealed_store.rs index 69f773ae1a3..683a3e14f59 100644 --- a/crates/tracedecay-graph-db/src/sealed_store.rs +++ b/crates/tracedecay-graph-db/src/sealed_store.rs @@ -3729,6 +3729,9 @@ mod build_tests { &BTreeSet::from(["generation:child2".to_owned()]), ) .unwrap(); + assert_eq!(census.head_count, 1); + assert_eq!(census.superseded_count, 1 + initial_layers); + assert_eq!(census.unrecognized_count, 0); for kind in [0, 1] { let bytes: u64 = physical .values()