From 7a1dff3cadd832f88b22ff4171009d4c57e3fb47 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 10 Oct 2026 17:08:38 -0700 Subject: [PATCH 01/31] refactor!: cut over to Node Bundle v1 --- .../fleet/authority/publication_tests.rs | 2 +- crates/cellule-ltx/src/cell_layout.rs | 6 +- .../docs/write-performance-design.md | 2 +- .../cellule-runtime/src/node/bundle/codec.rs | 94 +------ .../src/node/bundle/index/codec.rs | 24 +- .../src/node/bundle/index/dense/mod.rs | 1 - .../src/node/bundle/index/dense/tests.rs | 5 +- .../src/node/bundle/index/history.rs | 12 +- .../src/node/bundle/index/io/mod.rs | 24 +- .../src/node/bundle/index/mod.rs | 32 +-- .../src/node/bundle/index/tests/admission.rs | 2 - .../src/node/bundle/index/tests/format.rs | 91 +++++++ .../src/node/bundle/index/tests/legacy.rs | 172 ------------ .../src/node/bundle/index/tests/mod.rs | 3 +- crates/cellule-runtime/src/node/bundle/mod.rs | 3 +- .../cellule-runtime/src/node/bundle/store.rs | 37 --- .../src/node/bundle/tests/faults.rs | 10 +- .../src/node/bundle/tests/index/checkpoint.rs | 2 +- .../node/bundle/tests/index/compatibility.rs | 248 ------------------ .../node/bundle/tests/index/copy_on_write.rs | 2 +- .../src/node/bundle/tests/index/encoding.rs | 6 +- .../src/node/bundle/tests/index/format.rs | 133 ++++++++++ .../src/node/bundle/tests/index/integrity.rs | 113 ++++++++ .../src/node/bundle/tests/index/inventory.rs | 2 +- .../src/node/bundle/tests/index/mod.rs | 3 +- .../src/node/bundle/tests/index/pipeline.rs | 2 +- .../node/bundle/tests/index/preparation.rs | 6 +- .../src/node/bundle/tests/pipeline.rs | 3 +- .../src/node/bundle/tests/ranges.rs | 2 +- docs/bundle-coverage-implementation.md | 42 +-- docs/managed-bundle-pipeline.md | 5 +- docs/single-pass-bundle-encoding.md | 5 +- docs/typed-bundle-staging.md | 6 +- 33 files changed, 444 insertions(+), 656 deletions(-) create mode 100644 crates/cellule-runtime/src/node/bundle/index/tests/format.rs delete mode 100644 crates/cellule-runtime/src/node/bundle/index/tests/legacy.rs delete mode 100644 crates/cellule-runtime/src/node/bundle/tests/index/compatibility.rs create mode 100644 crates/cellule-runtime/src/node/bundle/tests/index/format.rs create mode 100644 crates/cellule-runtime/src/node/bundle/tests/index/integrity.rs diff --git a/crates/cellule-axum/examples/fleet/authority/publication_tests.rs b/crates/cellule-axum/examples/fleet/authority/publication_tests.rs index 3162d5b1..01f8148d 100644 --- a/crates/cellule-axum/examples/fleet/authority/publication_tests.rs +++ b/crates/cellule-axum/examples/fleet/authority/publication_tests.rs @@ -40,7 +40,7 @@ impl ObjectStore for PausedBundleStore { payload: object_store::PutPayload, options: object_store::PutOptions, ) -> object_store::Result { - if path.as_ref().ends_with(".cnb") && self.armed.swap(false, Ordering::AcqRel) { + if path.as_ref().ends_with(".nodebundle") && self.armed.swap(false, Ordering::AcqRel) { self.entered.notify_one(); self.resume.notified().await; } diff --git a/crates/cellule-ltx/src/cell_layout.rs b/crates/cellule-ltx/src/cell_layout.rs index e59076ee..3b268c65 100644 --- a/crates/cellule-ltx/src/cell_layout.rs +++ b/crates/cellule-ltx/src/cell_layout.rs @@ -325,7 +325,7 @@ impl CellStorageLayout { )) } - /// Exact immutable node-wide coverage catalog and native range object. + /// Exact immutable Node Bundle v1 coverage catalog and native range object. #[must_use] pub fn node_coverage_bundle_path( &self, @@ -334,7 +334,7 @@ impl CellStorageLayout { digest: &[u8; 32], ) -> Path { Path::from(format!( - "{}/cells/v1/node-logs/{}/{epoch}/coverage/v1/{}.cnb", + "{}/cells/v1/node-logs/{}/{epoch}/coverage/node-bundle/v1/{}.nodebundle", self.root, encode_hex(session), encode_hex(digest) @@ -396,7 +396,7 @@ mod tests { layout .node_coverage_bundle_path(&[0xdd; 16], 7, &[0xef; 32]) .as_ref(), - "tenant-root/cells/v1/node-logs/dddddddddddddddddddddddddddddddd/7/coverage/v1/efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef.cnb" + "tenant-root/cells/v1/node-logs/dddddddddddddddddddddddddddddddd/7/coverage/node-bundle/v1/efefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefef.nodebundle" ); assert_eq!( layout diff --git a/crates/cellule-runtime/docs/write-performance-design.md b/crates/cellule-runtime/docs/write-performance-design.md index ee90e844..cfec8cef 100644 --- a/crates/cellule-runtime/docs/write-performance-design.md +++ b/crates/cellule-runtime/docs/write-performance-design.md @@ -291,7 +291,7 @@ The encoder comparison preserves exact bytes in the external old-encoder oracle, but establishes no repeatable application improvement and fails performance qualification. -Changed CNB3 shards now retain a one-use canonical encoding plan. The encoder +Changed Node Bundle v1 shards now retain a one-use canonical encoding plan. The encoder records each detached history field, assigns native and history offsets, then fills those same-width fields without serializing all sibling controls again. Complete local self-verification, fresh origin verification and the canonical diff --git a/crates/cellule-runtime/src/node/bundle/codec.rs b/crates/cellule-runtime/src/node/bundle/codec.rs index a358322e..83737241 100644 --- a/crates/cellule-runtime/src/node/bundle/codec.rs +++ b/crates/cellule-runtime/src/node/bundle/codec.rs @@ -1,8 +1,8 @@ -//! Bounded binary catalog and verbatim native frame extents. +//! Bounded Node Bundle v1 catalog rows; native frames live in indexed extents. use super::*; use crate::codec::{BoundedDecoder, BoundedEncoder, read_fixed}; -const MAGIC: &[u8] = b"CNB1"; +const MAGIC: &[u8] = b"cellule.catalog-rows.v1"; fn position_write(e: &mut BoundedEncoder, p: cellule_ltx::Position) -> Result<()> { e.write_u64(p.txid)?; @@ -26,7 +26,7 @@ struct Metadata { extents: Vec<([u8; 32], std::ops::Range)>, } -fn metadata(catalog: &Catalog, frames: usize) -> Result { +fn metadata(catalog: &Catalog) -> Result { catalog.validate()?; let mut e = BoundedEncoder::new(MAX_BUNDLE_BYTES as u32)?; let mut extents = Vec::new(); @@ -40,8 +40,8 @@ fn metadata(catalog: &Catalog, frames: usize) -> Result { e.write_u64(catalog.selected_through)?; e.write_count(catalog.bindings.len())?; for binding in &catalog.bindings { - if binding.locators.len() > MAX_INLINE_LOCATORS { - return Err(Error::Capacity("inline bundle locator count")); + if binding.locators.len() > 1 { + return Err(Error::Capacity("bundle row history descriptor count")); } e.write_bytes(binding.application.as_bytes())?; e.write_u64(binding.first_commit)?; @@ -73,73 +73,26 @@ fn metadata(catalog: &Catalog, frames: usize) -> Result { extents.push((*pin.as_bytes(), start..e.encoded_len())); } } - e.write_count(frames)?; Ok(Metadata { encoder: e, extents, }) } -#[cfg(test)] -pub(super) fn encode( - catalog: &mut Catalog, - frames: &[cellule_ltx::VerifiedNodeFrame], -) -> Result { - if frames.len() > MAX_FRAMES { - return Err(Error::Capacity("bundle frame count")); - } - let mut offset = metadata(catalog, frames.len())?.encoder.finish().len() as u64; - for frame in frames { - offset = offset - .checked_add(4) - .ok_or(Error::Capacity("bundle extent"))?; - let digest = Digest::from_bytes(frame.digest()); - let matches = catalog - .bindings - .iter_mut() - .flat_map(|binding| &mut binding.locators) - .filter(|locator| locator.object.is_none() && locator.frame_digest == digest) - .map(|locator| { - locator.offset = offset; - locator.bytes = frame.encoded().len() as u64; - }) - .count(); - if matches != 1 { - return Err(Error::Node("bundle frame locator is not unique")); - } - offset = offset - .checked_add(frame.encoded().len() as u64) - .ok_or(Error::Capacity("bundle bytes"))?; - } - let mut e = metadata(catalog, frames.len())?.encoder; - for frame in frames { - e.write_bytes(frame.encoded())?; - } - Ok(Bytes::from(e.finish())) -} - #[cfg(test)] pub(super) fn encode_leaf(catalog: &Catalog) -> Result { Ok(Bytes::from(encode_leaf_plan(catalog)?.body)) } pub(super) fn encode_leaf_plan(catalog: &Catalog) -> Result { - let metadata = metadata(catalog, 0)?; + let metadata = metadata(catalog)?; Ok(LeafEncoding { body: metadata.encoder.finish(), extents: metadata.extents, }) } -pub(super) fn decode(body: &Bytes) -> Result { - decode_inner(body, true) -} - pub(super) fn decode_leaf(body: &Bytes) -> Result { - decode_inner(body, false) -} - -fn decode_inner(body: &Bytes, complete_object: bool) -> Result { let mut d = BoundedDecoder::new(body, MAX_BUNDLE_BYTES as u32)?; if d.read_bytes()? != MAGIC { return Err(Error::Node("unsupported node bundle format")); @@ -183,7 +136,7 @@ fn decode_inner(body: &Bytes, complete_object: bool) -> Result { let selected_commit = d.read_u64()?; let selected_position = position_read(&mut d)?; let count = d.read_count()?; - if count > MAX_INLINE_LOCATORS { + if count > 1 { return Err(Error::Capacity("bundle locator count")); } let mut locators = Vec::with_capacity(count); @@ -217,22 +170,6 @@ fn decode_inner(body: &Bytes, complete_object: bool) -> Result { locators, }); } - let count = d.read_count()?; - if count > MAX_FRAMES || (!complete_object && count != 0) { - return Err(Error::Capacity("bundle frame count")); - } - let mut local = Vec::with_capacity(count); - for _ in 0..count { - let frame = d.read_bytes()?; - // This offset comes from the borrowed input, rather than trusting an - // encoded locator or scanning other node objects for a matching row. - let offset = frame.as_ptr() as usize - body.as_ptr() as usize; - local.push(( - offset as u64, - frame.len() as u64, - Digest::from_bytes(*blake3::hash(frame).as_bytes()), - )); - } d.finish()?; let catalog = Catalog { session, @@ -243,22 +180,5 @@ fn decode_inner(body: &Bytes, complete_object: bool) -> Result { index: None, }; catalog.validate()?; - let locators: Vec<_> = catalog - .bindings - .iter() - .flat_map(|binding| &binding.locators) - .filter(|locator| locator.object.is_none()) - .map(|locator| (locator.offset, locator.bytes, locator.frame_digest)) - .collect(); - if complete_object - && (locators.len() != local.len() - || local - .iter() - .any(|extent| locators.iter().filter(|locator| *locator == extent).count() != 1)) - { - return Err(Error::Node( - "bundle local extents differ from complete manifest", - )); - } Ok(catalog) } diff --git a/crates/cellule-runtime/src/node/bundle/index/codec.rs b/crates/cellule-runtime/src/node/bundle/index/codec.rs index cde83291..f5adbc6a 100644 --- a/crates/cellule-runtime/src/node/bundle/index/codec.rs +++ b/crates/cellule-runtime/src/node/bundle/index/codec.rs @@ -1,7 +1,9 @@ -//! Canonical fixed-size authenticated root header; leaf codecs remain CNB1. +//! Canonical fixed-size authenticated Node Bundle v1 header. use super::*; use crate::codec::{BoundedDecoder, BoundedEncoder, read_fixed}; +const PREFIX_BYTES: usize = MAGIC.len() + 4; + pub(in crate::node::bundle) fn write_extent( e: &mut BoundedEncoder, extent: &Locator, @@ -34,7 +36,7 @@ pub(super) fn read_extent(d: &mut BoundedDecoder<'_>) -> Result { pub(super) fn encode(root: &Root) -> Result { validate(root)?; - let mut e = BoundedEncoder::new((HEADER_BYTES - 12) as u32)?; + let mut e = BoundedEncoder::new((HEADER_BYTES - PREFIX_BYTES) as u32)?; e.write_bytes(root.session.as_bytes())?; e.write_u64(root.epoch)?; e.write_bool(root.predecessor.is_some())?; @@ -56,7 +58,7 @@ pub(super) fn encode(root: &Root) -> Result { } let payload = e.finish(); let mut header = Vec::with_capacity(HEADER_BYTES); - header.extend_from_slice(if root.detached { DENSE_MAGIC } else { MAGIC }); + header.extend_from_slice(MAGIC); header.extend_from_slice(&(payload.len() as u32).to_be_bytes()); header.extend_from_slice(&payload); header.resize(HEADER_BYTES, 0); @@ -64,24 +66,25 @@ pub(super) fn encode(root: &Root) -> Result { } pub(super) fn decode(header: &[u8]) -> Result { - if header.len() != HEADER_BYTES - || !matches!(header.get(..8), Some(magic) if magic == MAGIC || magic == DENSE_MAGIC) - { + if header.len() != HEADER_BYTES || !header.starts_with(MAGIC) { return Err(Error::Node("unsupported bundle index header")); } let length = u32::from_be_bytes( - header[8..12] + header[MAGIC.len()..PREFIX_BYTES] .try_into() .map_err(|_| Error::Node("truncated bundle index header"))?, ) as usize; - let end = 12_usize + let end = PREFIX_BYTES .checked_add(length) .filter(|end| *end <= HEADER_BYTES) .ok_or(Error::Node("bundle index header length"))?; if header[end..].iter().any(|byte| *byte != 0) { return Err(Error::Node("noncanonical bundle index padding")); } - let mut d = BoundedDecoder::new(&header[12..end], (HEADER_BYTES - 12) as u32)?; + let mut d = BoundedDecoder::new( + &header[PREFIX_BYTES..end], + (HEADER_BYTES - PREFIX_BYTES) as u32, + )?; let session = SessionId::from_bytes(read_fixed(&mut d, "bundle index session length")?); let epoch = d.read_u64()?; let predecessor = if d.read_bool()? { @@ -115,7 +118,6 @@ pub(super) fn decode(header: &[u8]) -> Result { } d.finish()?; let root = Root { - detached: header.get(..8) == Some(DENSE_MAGIC.as_slice()), session, epoch, predecessor, @@ -162,7 +164,7 @@ fn validate(root: &Root) -> Result<()> { } local_end += frame.bytes; } - if local_end > root.object_bytes || (!root.detached && local_end != root.object_bytes) { + if local_end > root.object_bytes { return Err(Error::Node("bundle index object length differs")); } Ok(()) diff --git a/crates/cellule-runtime/src/node/bundle/index/dense/mod.rs b/crates/cellule-runtime/src/node/bundle/index/dense/mod.rs index ac55e067..e2869814 100644 --- a/crates/cellule-runtime/src/node/bundle/index/dense/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/dense/mod.rs @@ -162,7 +162,6 @@ pub(super) fn encode( } } let root = Root { - detached: true, session: catalog.session, epoch: catalog.epoch, predecessor: catalog.predecessor, diff --git a/crates/cellule-runtime/src/node/bundle/index/dense/tests.rs b/crates/cellule-runtime/src/node/bundle/index/dense/tests.rs index b215cffb..060a0c11 100644 --- a/crates/cellule-runtime/src/node/bundle/index/dense/tests.rs +++ b/crates/cellule-runtime/src/node/bundle/index/dense/tests.rs @@ -1,4 +1,4 @@ -// Reference CNB3 encoder from aaed329, retained only for byte compatibility. +// Independent reference layout encoder, using the current format identifiers. // Keep its two-pass shard construction independent of the production plan. use super::*; @@ -154,7 +154,6 @@ pub(in crate::node::bundle) fn encode_original( } } let root = Root { - detached: true, session: catalog.session, epoch: catalog.epoch, predecessor: catalog.predecessor, @@ -197,7 +196,7 @@ fn encode_leaf_original( } let inline = crate::node::bundle::codec::encode_leaf(&compact)?; let mut e = BoundedEncoder::new(crate::node::bundle::MAX_BUNDLE_BYTES as u32)?; - e.write_bytes(b"CBL3")?; + e.write_bytes(history::LEAF_MAGIC)?; e.write_bytes(&inline)?; e.write_count( compact diff --git a/crates/cellule-runtime/src/node/bundle/index/history.rs b/crates/cellule-runtime/src/node/bundle/index/history.rs index 8477fe88..720ed17c 100644 --- a/crates/cellule-runtime/src/node/bundle/index/history.rs +++ b/crates/cellule-runtime/src/node/bundle/index/history.rs @@ -3,8 +3,8 @@ use super::*; use crate::codec::{BoundedDecoder, BoundedEncoder, read_fixed}; pub(super) const MAX_HISTORY_BYTES: u64 = 32 << 10; -const LEAF_MAGIC: &[u8] = b"CBL3"; -const HISTORY_MAGIC: &[u8] = b"CLH3"; +pub(super) const LEAF_MAGIC: &[u8] = b"cellule.catalog-shard.v1"; +const HISTORY_MAGIC: &[u8] = b"cellule.cell-history.v1"; #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct History { @@ -68,9 +68,8 @@ pub(super) fn decode( let mut native_bytes = 0_u64; for _ in 0..count { let locator = super::codec::read_extent(&mut d)?; - // Native locators from the complete legacy format may precede the - // fixed index header, so the history preserves that older byte offset. - if locator.bytes == 0 + if locator.offset < HEADER_BYTES as u64 + || locator.bytes == 0 || locator .offset .checked_add(locator.bytes) @@ -180,9 +179,6 @@ pub(super) fn encode_leaf( } pub(super) fn decode_leaf(body: &Bytes) -> Result<(Catalog, BTreeMap<[u8; 32], History>)> { - if body.get(..8) != Some(b"\0\0\0\x04CBL3".as_slice()) { - return Ok((super::super::codec::decode_leaf(body)?, BTreeMap::new())); - } let mut d = BoundedDecoder::new(body, MAX_BUNDLE_BYTES as u32)?; if d.read_bytes()? != LEAF_MAGIC { return Err(Error::Node("unsupported detached bundle leaf")); diff --git a/crates/cellule-runtime/src/node/bundle/index/io/mod.rs b/crates/cellule-runtime/src/node/bundle/index/io/mod.rs index 5e1947da..0306f5f1 100644 --- a/crates/cellule-runtime/src/node/bundle/index/io/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/io/mod.rs @@ -12,7 +12,7 @@ async fn load_root( session: SessionId, head: NodeBundleHead, origin: Option<&super::super::origin::ProposalMetadata<'_>>, -) -> Result> { +) -> Result { let header = super::super::origin::read_metadata_range( layout, session, @@ -22,9 +22,6 @@ async fn load_root( origin, ) .await?; - if !matches!(header.get(..8), Some(magic) if magic == MAGIC || magic == DENSE_MAGIC) { - return Ok(None); - } if *blake3::hash(&header).as_bytes() != *head.digest.as_bytes() { return Err(Error::Node("bundle index header digest differs")); } @@ -40,7 +37,7 @@ async fn load_root( shard.extent.object = Some(head.digest); } } - Ok(Some(root)) + Ok(root) } async fn load_rows( @@ -145,9 +142,7 @@ async fn load_inner( origin: Option<&super::super::origin::ProposalMetadata<'_>>, mut preparation: Option<&mut BundlePreparation>, ) -> Result { - let Some(root) = load_root(layout, session, head, origin).await? else { - return super::super::store::load_legacy_catalog(layout, session, head).await; - }; + let root = load_root(layout, session, head, origin).await?; if let Some(preparation) = preparation.as_deref_mut() { preparation.bind(layout, session, head.epoch); } @@ -414,10 +409,7 @@ pub(in crate::node::bundle) async fn ensure_drained( session: SessionId, head: NodeBundleHead, ) -> Result<()> { - let Some(root) = load_root(layout, session, head, None).await? else { - let catalog = super::super::store::load_legacy_catalog(layout, session, head).await?; - return check_closed(&catalog.bindings); - }; + let root = load_root(layout, session, head, None).await?; let mut pins = std::collections::HashSet::new(); for (id, shard) in root.shards.iter().enumerate() { let Some(shard) = shard else { continue }; @@ -453,13 +445,7 @@ pub(in crate::node::bundle) async fn binding_inventory( session: SessionId, head: NodeBundleHead, ) -> Result> { - let Some(root) = load_root(layout, session, head, None).await? else { - return Ok( - super::super::store::load_legacy_catalog(layout, session, head) - .await? - .bindings, - ); - }; + let root = load_root(layout, session, head, None).await?; let mut bindings = Vec::new(); let mut pins = std::collections::HashSet::new(); for (id, shard) in root.shards.iter().enumerate() { diff --git a/crates/cellule-runtime/src/node/bundle/index/mod.rs b/crates/cellule-runtime/src/node/bundle/index/mod.rs index 9ecbc230..0536eeca 100644 --- a/crates/cellule-runtime/src/node/bundle/index/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/mod.rs @@ -22,8 +22,6 @@ pub(super) use dense::encode_original; pub(super) use history::validate_deferred; #[cfg(test)] pub(super) use io::load; -#[cfg(test)] -pub(super) use tests::encode_inline; #[cfg(test)] pub(super) fn history_extent(catalog: &Catalog, pin: Digest) -> Option { @@ -38,8 +36,7 @@ pub(super) use io::{binding_inventory, ensure_drained, load_cells}; pub(super) const HEADER_BYTES: usize = 32 << 10; const SHARDS: usize = 256; -const MAGIC: &[u8; 8] = b"\0\0\0\x04CNB2"; -const DENSE_MAGIC: &[u8; 8] = b"\0\0\0\x04CNB3"; +const MAGIC: &[u8] = b"cellule.node-bundle.v1\0"; pub(super) type CellKey = ([u8; 16], [u8; 32]); #[derive(Clone, Debug, PartialEq, Eq)] @@ -50,7 +47,6 @@ struct Shard { #[derive(Clone, Debug, PartialEq, Eq)] struct Root { - detached: bool, session: SessionId, epoch: u64, predecessor: Option, @@ -127,11 +123,6 @@ fn extent_bytes(body: &Bytes, locator: &Locator) -> Result { Ok(Bytes::copy_from_slice(bytes)) } -#[cfg(test)] -fn decode_leaf(body: Bytes, shard: &Shard, id: u8, root: &Root) -> Result { - decode_leaf_with_histories(body, shard, id, root).map(|(catalog, _)| catalog) -} - fn decode_leaf_with_histories( body: Bytes, shard: &Shard, @@ -144,9 +135,6 @@ fn decode_leaf_with_histories( return Err(Error::Node("bundle catalog shard digest differs")); } let (catalog, histories) = history::decode_leaf(&body)?; - if !root.detached && body.get(..8) == Some(b"\0\0\0\x04CBL3".as_slice()) { - return Err(Error::Node("inline bundle index contains detached history")); - } if catalog.session != root.session || catalog.epoch != root.epoch || catalog.selected_through > root.selected_through @@ -163,16 +151,12 @@ fn decode_leaf_with_histories( } pub(super) fn body_digest(body: &Bytes) -> Result { - if matches!(body.get(..8), Some(magic) if magic == MAGIC || magic == DENSE_MAGIC) { - let header = body - .get(..HEADER_BYTES) - .ok_or(Error::Node("truncated bundle index header"))?; - let root = codec::decode(header)?; - if root.object_bytes != body.len() as u64 { - return Err(Error::Node("bundle proposal object length differs")); - } - Ok(Digest::from_bytes(*blake3::hash(header).as_bytes())) - } else { - Ok(Digest::from_bytes(*blake3::hash(body).as_bytes())) + let header = body + .get(..HEADER_BYTES) + .ok_or(Error::Node("truncated bundle index header"))?; + let root = codec::decode(header)?; + if root.object_bytes != body.len() as u64 { + return Err(Error::Node("bundle proposal object length differs")); } + Ok(Digest::from_bytes(*blake3::hash(header).as_bytes())) } diff --git a/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs b/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs index 22e2fa15..153b9055 100644 --- a/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs +++ b/crates/cellule-runtime/src/node/bundle/index/tests/admission.rs @@ -64,7 +64,6 @@ async fn aggregate_history_budget_is_checked_before_the_first_history_read() { bindings: 129, }); let root = Root { - detached: true, session, epoch: 2, predecessor: None, @@ -125,7 +124,6 @@ async fn selected_shard_budget_is_checked_before_loading_any_leaf() { }); } let root = Root { - detached: false, session, epoch: 2, predecessor: None, diff --git a/crates/cellule-runtime/src/node/bundle/index/tests/format.rs b/crates/cellule-runtime/src/node/bundle/index/tests/format.rs new file mode 100644 index 00000000..b60a35ab --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/index/tests/format.rs @@ -0,0 +1,91 @@ +use super::*; +use crate::codec::BoundedEncoder; + +#[test] +fn old_inner_identifiers_are_rejected_even_with_matching_history_digests() { + let binding = history_binding([4; 32], 1); + let session = SessionId::from_bytes([1; 16]); + let mut encoder = BoundedEncoder::new(history::MAX_HISTORY_BYTES as u32).unwrap(); + encoder.write_bytes(b"CLH3").unwrap(); + let body = Bytes::from(encoder.finish()); + let descriptor = history::History { + extent: Locator { + object: Some(Digest::from_bytes([7; 32])), + offset: HEADER_BYTES as u64, + bytes: body.len() as u64, + frame_digest: Digest::from_bytes(*blake3::hash(&body).as_bytes()), + }, + count: 1, + native_bytes: 128, + loaded: None, + }; + assert!(matches!( + history::decode(&body, session, 2, &binding, &descriptor), + Err(Error::Fenced) + )); + for magic in [b"CBL3".as_slice(), b"CNB1".as_slice()] { + let mut encoder = BoundedEncoder::new(MAX_BUNDLE_BYTES as u32).unwrap(); + encoder.write_bytes(magic).unwrap(); + let body = Bytes::from(encoder.finish()); + assert!(matches!( + history::decode_leaf(&body), + Err(Error::Node("unsupported detached bundle leaf")) + )); + assert!(matches!( + crate::node::bundle::codec::decode_leaf(&body), + Err(Error::Node("unsupported node bundle format")) + )); + } +} + +#[test] +fn inner_objects_use_the_named_v1_identifiers() { + let session = SessionId::from_bytes([1; 16]); + let binding = history_binding([4; 32], 1); + let body = history::encode(session, 2, &binding).unwrap(); + let mut decoder = + crate::codec::BoundedDecoder::new(&body, history::MAX_HISTORY_BYTES as u32).unwrap(); + assert_eq!(decoder.read_bytes().unwrap(), b"cellule.cell-history.v1"); + let catalog = Catalog { + session, + epoch: 2, + predecessor: None, + selected_through: 0, + bindings: Vec::new(), + index: None, + }; + let body = history::encode_leaf(&catalog, &BTreeMap::new()).unwrap(); + let mut decoder = crate::codec::BoundedDecoder::new(&body, MAX_BUNDLE_BYTES as u32).unwrap(); + assert_eq!(decoder.read_bytes().unwrap(), b"cellule.catalog-shard.v1"); + let mut rows = + crate::codec::BoundedDecoder::new(decoder.read_bytes().unwrap(), MAX_BUNDLE_BYTES as u32) + .unwrap(); + assert_eq!(rows.read_bytes().unwrap(), b"cellule.catalog-rows.v1"); +} + +#[test] +fn root_header_length_and_padding_are_checked_after_the_named_identifier() { + let mut catalog = Catalog { + session: SessionId::from_bytes([1; 16]), + epoch: 2, + predecessor: None, + selected_through: 0, + bindings: Vec::new(), + index: None, + }; + let (body, digest) = encode(&mut catalog, &[]).unwrap(); + assert_eq!(body_digest(&body).unwrap(), digest); + let mut header = body.to_vec(); + header[MAGIC.len()..MAGIC.len() + 4].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(matches!( + codec::decode(&header), + Err(Error::Node("bundle index header length")) + )); + let mut header = body.to_vec(); + *header.last_mut().unwrap() = 1; + assert!(matches!( + codec::decode(&header), + Err(Error::Node("noncanonical bundle index padding")) + )); + assert!(codec::decode(&body[..HEADER_BYTES - 1]).is_err()); +} diff --git a/crates/cellule-runtime/src/node/bundle/index/tests/legacy.rs b/crates/cellule-runtime/src/node/bundle/index/tests/legacy.rs deleted file mode 100644 index 6896d3c9..00000000 --- a/crates/cellule-runtime/src/node/bundle/index/tests/legacy.rs +++ /dev/null @@ -1,172 +0,0 @@ -use super::*; - -pub(in crate::node::bundle) fn encode_inline( - catalog: &mut Catalog, - frames: &[cellule_ltx::VerifiedNodeFrame], -) -> Result<(Bytes, Digest)> { - catalog.validate()?; - if frames.len() > MAX_FRAMES { - return Err(Error::Capacity("bundle frame count")); - } - let groups = grouped(catalog); - let mut shards = match &catalog.index { - Some(index) => { - if index.root.session != catalog.session || index.root.epoch != catalog.epoch { - return Err(Error::Fenced); - } - // A partial catalog may modify only a shard it authenticated first. - if groups.keys().any(|id| !index.loaded.contains_key(id)) { - return Err(Error::Node("bundle modifies an unloaded catalog shard")); - } - index.root.shards.clone() - } - None => vec![None; SHARDS], - }; - let changed: BTreeSet = match &catalog.index { - Some(index) => index - .loaded - .iter() - .filter_map(|(id, original)| { - (groups.get(id).map(Vec::as_slice).unwrap_or(&[]) != original.as_slice()) - .then_some(*id) - }) - .collect(), - None => groups.keys().copied().collect(), - }; - // Native extents follow the changed shards; changing numeric offsets never - // changes the leaf encoding length. Compute sizes before filling offsets. - let mut offset = HEADER_BYTES as u64; - for id in &changed { - let rows = groups.get(id).cloned().unwrap_or_default(); - if rows.is_empty() { - shards[usize::from(*id)] = None; - } else { - let bytes = crate::node::bundle::codec::encode_leaf(&leaf(catalog, rows))?.len() as u64; - shards[usize::from(*id)] = Some(Shard { - extent: Locator { - object: None, - offset, - bytes, - frame_digest: Digest::from_bytes([0; 32]), - }, - bindings: groups[id].len(), - }); - offset = offset - .checked_add(bytes) - .ok_or(Error::Capacity("bundle shard bytes"))?; - } - } - let mut native = Vec::with_capacity(frames.len()); - for frame in frames { - let digest = Digest::from_bytes(frame.digest()); - let mut matches = 0; - for binding in &mut catalog.bindings { - let id = binding_shard(binding); - for locator in &mut binding.locators { - if locator.object.is_none() && locator.frame_digest == digest { - if !changed.contains(&id) { - return Err(Error::Node("native frame belongs to an unchanged shard")); - } - locator.offset = offset; - locator.bytes = frame.encoded().len() as u64; - matches += 1; - } - } - } - if matches != 1 { - return Err(Error::Node("bundle frame locator is not unique")); - } - native.push(Locator { - object: None, - offset, - bytes: frame.encoded().len() as u64, - frame_digest: digest, - }); - offset = offset - .checked_add(frame.encoded().len() as u64) - .ok_or(Error::Capacity("bundle bytes"))?; - } - if offset > MAX_BUNDLE_BYTES { - return Err(Error::Capacity("bundle bytes")); - } - let groups = grouped(catalog); - let mut bodies = Vec::new(); - for id in &changed { - if let Some(reference) = &mut shards[usize::from(*id)] { - let body = crate::node::bundle::codec::encode_leaf(&leaf(catalog, groups[id].clone()))?; - if body.len() as u64 != reference.extent.bytes { - return Err(Error::Node("bundle shard size changed during encoding")); - } - reference.extent.frame_digest = Digest::from_bytes(*blake3::hash(&body).as_bytes()); - bodies.push(body); - } - } - let root = Root { - detached: false, - session: catalog.session, - epoch: catalog.epoch, - predecessor: catalog.predecessor, - selected_through: catalog.selected_through, - object_bytes: offset, - shards, - frames: native, - }; - let header = codec::encode(&root)?; - let digest = Digest::from_bytes(*blake3::hash(&header).as_bytes()); - let mut body = Vec::with_capacity(offset as usize); - body.extend_from_slice(&header); - for leaf in bodies { - body.extend_from_slice(&leaf); - } - for frame in frames { - body.extend_from_slice(frame.encoded()); - } - let body = Bytes::from(body); - verify_local(&body, &root)?; - Ok((body, digest)) -} - -/// Self-verification covers all bytes of a newly uploaded object. References -/// to old immutable shards remain authenticated by the predecessor selection. -fn verify_local(body: &Bytes, expected: &Root) -> Result<()> { - let root = codec::decode(&body[..HEADER_BYTES])?; - if &root != expected || body.len() as u64 != root.object_bytes { - return Err(Error::Node("bundle index self-verification differs")); - } - let mut offset = HEADER_BYTES as u64; - let mut locators = Vec::new(); - for (id, shard) in root.shards.iter().enumerate() { - let Some(shard) = shard else { continue }; - if shard.extent.object.is_some() { - continue; - } - if shard.extent.offset != offset { - return Err(Error::Node("bundle local shard extents are not canonical")); - } - let bytes = extent_bytes(body, &shard.extent)?; - let leaf = decode_leaf(bytes, shard, id as u8, &root)?; - locators.extend( - leaf.bindings - .into_iter() - .flat_map(|binding| binding.locators) - .filter(|locator| locator.object.is_none()), - ); - offset += shard.extent.bytes; - } - if locators.len() != root.frames.len() { - return Err(Error::Node("bundle local native extent count differs")); - } - for frame in &root.frames { - if frame.offset != offset - || locators.iter().filter(|locator| *locator == frame).count() != 1 - { - return Err(Error::Node("bundle local native extents differ")); - } - extent_bytes(body, frame)?; - offset += frame.bytes; - } - if offset != root.object_bytes { - return Err(Error::Node("bundle has unauthenticated trailing bytes")); - } - Ok(()) -} diff --git a/crates/cellule-runtime/src/node/bundle/index/tests/mod.rs b/crates/cellule-runtime/src/node/bundle/index/tests/mod.rs index e7e5e12d..fb62da3f 100644 --- a/crates/cellule-runtime/src/node/bundle/index/tests/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/tests/mod.rs @@ -66,6 +66,5 @@ fn history_binding(cell: [u8; 32], count: usize) -> Binding { } mod admission; +mod format; mod histories; -mod legacy; -pub(in crate::node::bundle) use legacy::encode_inline; diff --git a/crates/cellule-runtime/src/node/bundle/mod.rs b/crates/cellule-runtime/src/node/bundle/mod.rs index 39e8ce59..e08cee6b 100644 --- a/crates/cellule-runtime/src/node/bundle/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/mod.rs @@ -1,4 +1,4 @@ -//! Verified node-wide coverage and bounded per-Cell reconstruction locators. +//! Node Bundle v1: verified node-wide coverage and bounded Cell locators. //! //! Upload is a proposal. The canonical node-record CAS selects its complete //! binding catalog and native range together. Cell authority pins each binding @@ -70,7 +70,6 @@ pub(crate) const MAX_BUNDLE_BYTES: u64 = 4 << 20; const MAX_BINDINGS: usize = 4_096; pub(crate) const LIVE_PREFIX_INDEX_BYTES: usize = 448 << 10; const MAX_LOCATORS: usize = 256; -const MAX_INLINE_LOCATORS: usize = 32; const MAX_FRAMES: usize = 64; // Reconstruction retains at most one Cell suffix. Selection streams checked // historical frames and shares one fresh cohort body for its new extents. diff --git a/crates/cellule-runtime/src/node/bundle/store.rs b/crates/cellule-runtime/src/node/bundle/store.rs index de48884b..4c7451ac 100644 --- a/crates/cellule-runtime/src/node/bundle/store.rs +++ b/crates/cellule-runtime/src/node/bundle/store.rs @@ -196,43 +196,6 @@ pub(super) async fn load_catalog_cells( index::load_cells(layout, session, head, cells, None, None).await } -pub(super) async fn load_legacy_catalog( - layout: &cellule_ltx::CellStorageLayout, - session: SessionId, - head: NodeBundleHead, -) -> Result { - let (body, _) = layout - .store() - .get_with_etag_bounded( - &layout.node_coverage_bundle_path( - session.as_bytes(), - head.epoch, - head.digest.as_bytes(), - ), - MAX_BUNDLE_BYTES, - ) - .await?; - if *blake3::hash(&body).as_bytes() != *head.digest.as_bytes() { - return Err(Error::Node("bundle catalog digest differs")); - } - let mut catalog = codec::decode(&body)?; - if catalog.session != session - || catalog.epoch != head.epoch - || catalog.selected_through != head.selected_through - { - return Err(Error::Node("bundle catalog scope differs")); - } - for locator in catalog - .bindings - .iter_mut() - .flat_map(|binding| &mut binding.locators) - { - if locator.object.is_none() { - locator.object = Some(head.digest); - } - } - Ok(catalog) -} /// A Cell pin cannot exist outside the complete canonical inventory, including /// when callers use the lower-level CellAuthority transition directly. pub(crate) async fn ensure_enrollment( diff --git a/crates/cellule-runtime/src/node/bundle/tests/faults.rs b/crates/cellule-runtime/src/node/bundle/tests/faults.rs index 047d44d5..fadc2927 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/faults.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/faults.rs @@ -75,7 +75,7 @@ impl ObjectStore for ReplyFault { self.node_started.notify_one(); self.node_resume.notified().await; } - if path.as_ref().ends_with(".cnb") { + if path.as_ref().ends_with(".nodebundle") { self.coverage_puts.fetch_add(1, Ordering::SeqCst); } if node_update && mode == 4 { @@ -95,7 +95,7 @@ impl ObjectStore for ReplyFault { { return Err(denied()); } - let eligible = (mode == 1 && path.as_ref().ends_with(".cnb")) + let eligible = (mode == 1 && path.as_ref().ends_with(".nodebundle")) || (mode == 8 && path.as_ref().ends_with("/control.json") && matches!(opts.mode, object_store::PutMode::Update(_))) @@ -128,7 +128,9 @@ impl ObjectStore for ReplyFault { self.node_resume.notified().await; } let metadata = match &opts.range { - Some(object_store::GetRange::Bounded(range)) if path.as_ref().ends_with(".cnb") => { + Some(object_store::GetRange::Bounded(range)) + if path.as_ref().ends_with(".nodebundle") => + { (mode == 14 && range.start != 0) || (mode == 15 && self @@ -155,7 +157,7 @@ impl ObjectStore for ReplyFault { self.node_resume.notified().await; } if self.mode.load(Ordering::SeqCst) == 11 - && path.as_ref().ends_with(".cnb") + && path.as_ref().ends_with(".nodebundle") && opts.range.is_some() { self.range_started.fetch_add(1, Ordering::SeqCst); diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/checkpoint.rs b/crates/cellule-runtime/src/node/bundle/tests/index/checkpoint.rs index 0dcb2055..7f88c744 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/checkpoint.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/checkpoint.rs @@ -137,7 +137,7 @@ async fn checkpoint_uses_the_exact_materialized_proof_without_scanning_siblings_ .count .requests() .into_iter() - .filter(|read| read.location.ends_with(".cnb")) + .filter(|read| read.location.ends_with(".nodebundle")) .collect(); assert_eq!( reads.len(), diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/compatibility.rs b/crates/cellule-runtime/src/node/bundle/tests/index/compatibility.rs deleted file mode 100644 index d0f75432..00000000 --- a/crates/cellule-runtime/src/node/bundle/tests/index/compatibility.rs +++ /dev/null @@ -1,248 +0,0 @@ -use super::*; - -#[tokio::test] -async fn inline_indexed_suffix_migrates_to_detached_history_with_the_same_pin() { - let mut f = Fixture::new().await; - let mut cell = f.cell(4).await; - let (_, frames, assignment) = f.append(&mut cell, 2); - let proposal = f - .directory - .prepare_node_bundle(&f.node, &frames, &[assignment], NOW) - .await - .unwrap(); - f.node = f - .directory - .select_node_bundle(&f.node, &proposal, &f.lease, Limits::default(), NOW) - .await - .unwrap() - .0; - let original = f.node.advertisement().bundle_head().unwrap(); - let mut catalog = load_catalog(&f.layout, head_session(&f), original) - .await - .unwrap(); - catalog.index = None; - catalog.predecessor = Some(original.digest()); - let (body, digest) = catalog_index::encode_inline(&mut catalog, &[]).unwrap(); - assert_eq!(&body[..8], b"\0\0\0\x04CNB2"); - let legacy = PreparedNodeBundle(Arc::new(StagedNodeBundle { - original: Some(original), - head: NodeBundleHead { - epoch: EPOCH, - digest, - selected_through: original.selected_through(), - }, - body, - catalog, - assignments: Vec::new(), - })); - f.layout - .store() - .put_exact( - &f.layout - .node_coverage_bundle_path(&[1; 16], EPOCH, digest.as_bytes()), - legacy.body.clone(), - ) - .await - .unwrap(); - f.node = f - .directory - .select_catalog(&f.node, &legacy, NOW) - .await - .unwrap(); - let old_pin = cell.control.value().bundle_binding.unwrap(); - assert_eq!( - f.directory - .load_bundle_coverage(&cell.authority, &cell.control, Limits::default()) - .await - .unwrap() - .commit_sequence(), - 2 - ); - let (_, frames, assignment) = f.append(&mut cell, 3); - let proposal = f - .directory - .prepare_node_bundle(&f.node, &frames, &[assignment], NOW) - .await - .unwrap(); - assert_eq!(&proposal.body[..8], b"\0\0\0\x04CNB3"); - let (_, proofs) = f - .directory - .select_node_bundle(&f.node, &proposal, &f.lease, Limits::default(), NOW) - .await - .unwrap(); - assert_eq!(proofs[0].binding(), old_pin); - assert_eq!(proofs[0].locator_count(), 2); - assert_eq!(proofs[0].commit_sequence(), 3); -} - -#[tokio::test] -async fn legacy_selected_catalog_is_read_and_migrated_without_changing_the_cell_pin() { - let mut f = Fixture::new().await; - let mut cell = f.cell(4).await; - let original = f.node.advertisement().bundle_head().unwrap(); - let mut catalog = load_catalog(&f.layout, head_session(&f), original) - .await - .unwrap(); - catalog.index = None; - catalog.predecessor = Some(original.digest()); - let body = codec::encode(&mut catalog, &[]).unwrap(); - let legacy = PreparedNodeBundle(Arc::new(StagedNodeBundle { - original: Some(original), - head: NodeBundleHead { - epoch: EPOCH, - digest: Digest::from_bytes(*blake3::hash(&body).as_bytes()), - selected_through: 0, - }, - body, - catalog, - assignments: Vec::new(), - })); - f.layout - .store() - .put_exact( - &f.layout - .node_coverage_bundle_path(&[1; 16], EPOCH, legacy.head.digest.as_bytes()), - legacy.body.clone(), - ) - .await - .unwrap(); - f.node = f - .directory - .select_catalog(&f.node, &legacy, NOW) - .await - .unwrap(); - let old_pin = cell.control.value().bundle_binding.unwrap(); - let loaded = f - .directory - .load_bundle_coverage(&cell.authority, &cell.control, Limits::default()) - .await - .unwrap(); - assert_eq!(loaded.binding(), old_pin); - assert_eq!(loaded.commit_sequence(), 1); - let (_, frames, assigned) = f.append(&mut cell, 2); - let proposal = f - .directory - .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) - .await - .unwrap(); - assert_eq!(&proposal.body[..8], b"\0\0\0\x04CNB3"); - let (_, proofs) = f - .directory - .select_node_bundle(&f.node, &proposal, &f.lease, Limits::default(), NOW) - .await - .unwrap(); - assert_eq!(proofs[0].binding(), old_pin); - assert_eq!(proofs[0].commit_sequence(), 2); -} - -#[tokio::test] -async fn indexed_shard_corruption_and_missing_reused_shards_fail_closed() { - let mut f = Fixture::new().await; - let mut a = f.cell(4).await; - let mut b = f.cell(5).await; - assert_ne!( - catalog_index::shard( - a.authority.layout().application_id(), - a.control.value().cell.as_bytes() - ), - catalog_index::shard( - b.authority.layout().application_id(), - b.control.value().cell.as_bytes() - ) - ); - let (_, frames, assigned) = f.append(&mut a, 2); - let proposal = f - .directory - .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) - .await - .unwrap(); - let (selected, _) = f - .directory - .select_node_bundle(&f.node, &proposal, &f.lease, Limits::default(), NOW) - .await - .unwrap(); - f.node = selected; - let old_path = - f.layout - .node_coverage_bundle_path(&[1; 16], EPOCH, proposal.head.digest.as_bytes()); - let (_, frames, assigned) = f.append(&mut b, 2); - let sibling = f - .directory - .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) - .await - .unwrap(); - let (selected, _) = f - .directory - .select_node_bundle(&f.node, &sibling, &f.lease, Limits::default(), NOW) - .await - .unwrap(); - f.node = selected; - f.count.reset(); - assert_eq!( - f.directory - .load_bundle_coverage(&a.authority, &a.control, Limits::default()) - .await - .unwrap() - .commit_sequence(), - 2 - ); - let reads: Vec<_> = f - .count - .requests() - .into_iter() - .filter(|read| read.location.ends_with(".cnb")) - .collect(); - assert_eq!( - reads.len(), - 4, - "a reused shard/history does not require the old root header" - ); - assert_eq!( - reads - .iter() - .filter(|read| read.location == old_path.as_ref()) - .count(), - 3 - ); - // The head/header stays unchanged; the independently authenticated reused - // shard must still detect a corrupt provider payload in the old object. - let mut corrupted = proposal.body.to_vec(); - corrupted[catalog_index::HEADER_BYTES + 8] ^= 1; - f.layout - .store() - .put_overwrite(&old_path, Bytes::from(corrupted)) - .await - .unwrap(); - assert!( - f.directory - .load_bundle_coverage(&a.authority, &a.control, Limits::default()) - .await - .is_err() - ); - // Unrelated Cell data remains readable; complete maintenance inventory must - // refuse the missing/corrupt dependency rather than omitting this Cell. - assert_eq!( - f.directory - .load_bundle_coverage(&b.authority, &b.control, Limits::default()) - .await - .unwrap() - .commit_sequence(), - 2 - ); - assert!( - load_catalog( - &f.layout, - head_session(&f), - f.node.advertisement().bundle_head().unwrap() - ) - .await - .is_err() - ); - f.layout.store().delete(&old_path).await.unwrap(); - assert!( - f.directory - .load_bundle_coverage(&a.authority, &a.control, Limits::default()) - .await - .is_err() - ); -} diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/copy_on_write.rs b/crates/cellule-runtime/src/node/bundle/tests/index/copy_on_write.rs index 909af294..d30e7528 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/copy_on_write.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/copy_on_write.rs @@ -41,7 +41,7 @@ async fn updating_one_of_a_thousand_cells_does_not_rewrite_the_complete_inventor .count .requests() .into_iter() - .filter(|read| read.location.ends_with(".cnb")) + .filter(|read| read.location.ends_with(".nodebundle")) .collect(); assert!( catalog_reads diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs b/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs index 89bed590..7a45f480 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs @@ -29,7 +29,9 @@ async fn dense_multi_cell_encoding_preserves_the_original_native_order_and_bytes // original roots are deliberately unavailable: encoding may retain their // references, but selecting this cohort must verify only its own bases. inventory(&mut f, &cells[0], 1_937).await; - let original_sizes = [504_123, 511_931, 518_715, 525_755, 532_603, 539_643]; + // Node Bundle v1's named identifiers expand the 58 touched shards and 64 + // histories; the fixed header size and verbatim native bytes stay the same. + let encoded_sizes = [507_369, 515_177, 521_961, 529_001, 535_849, 542_889]; let mut latest = Vec::new(); for commit in 2..=7 { let now = f.heartbeat().await; @@ -45,7 +47,7 @@ async fn dense_multi_cell_encoding_preserves_the_original_native_order_and_bytes .prepare_node_bundle(&f.node, &frames, &assignments, now) .await .unwrap(); - assert_eq!(proposal.body.len(), original_sizes[(commit - 2) as usize]); + assert_eq!(proposal.body.len(), encoded_sizes[(commit - 2) as usize]); // Compare this exact capture with the pre-optimization encoder. Native // roots include generated identities, so a stored digest is not a // stable cross-run wire fixture. Both encoders must agree byte for byte diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/format.rs b/crates/cellule-runtime/src/node/bundle/tests/index/format.rs new file mode 100644 index 00000000..a13e9cd6 --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/index/format.rs @@ -0,0 +1,133 @@ +use super::*; + +#[tokio::test] +async fn node_bundle_v1_uses_the_named_format_and_exact_object_path() { + let mut f = Fixture::new().await; + let mut cell = f.cell(4).await; + let (_, frames, assigned) = f.append(&mut cell, 2); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + assert!(prepared.body.starts_with(b"cellule.node-bundle.v1\0")); + let path = f + .layout + .node_coverage_bundle_path(&[1; 16], EPOCH, prepared.head.digest.as_bytes()); + assert!(path.as_ref().contains("/coverage/node-bundle/v1/")); + assert!(path.as_ref().ends_with(".nodebundle")); + f.node = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap() + .0; + let recovered = f + .directory + .load_bundle_coverage(&cell.authority, &cell.control, Limits::default()) + .await + .unwrap(); + assert_eq!(recovered.commit_sequence(), 2); + assert_eq!( + recovered.position(), + frames.last().unwrap().segment().position() + ); +} + +#[tokio::test] +async fn point_recovery_does_not_search_the_previous_object_path() { + let mut f = Fixture::new().await; + let cell = f.cell(4).await; + let head = f.node.advertisement().bundle_head().unwrap(); + let path = f + .layout + .node_coverage_bundle_path(&[1; 16], EPOCH, head.digest.as_bytes()); + let previous = Path::from( + path.as_ref() + .replace("/coverage/node-bundle/v1/", "/coverage/v1/") + .replace(".nodebundle", ".cnb"), + ); + assert_ne!(path, previous); + let (body, _) = f + .layout + .store() + .get_with_etag_bounded(&path, MAX_BUNDLE_BYTES) + .await + .unwrap(); + // Even supported bytes at the previous path cannot satisfy the pinned head. + f.layout.store().put_exact(&previous, body).await.unwrap(); + f.layout.store().delete(&path).await.unwrap(); + f.count.reset(); + assert!( + f.directory + .load_bundle_coverage(&cell.authority, &cell.control, Limits::default()) + .await + .is_err() + ); + let reads = f.count.requests(); + assert!(reads.iter().any(|read| read.location == path.as_ref())); + assert!(reads.iter().all(|read| read.location != previous.as_ref())); +} + +#[tokio::test] +async fn previous_format_identifiers_fail_point_recovery_and_maintenance_without_fallback() { + for magic in [b"\0\0\0\x04CNB1", b"\0\0\0\x04CNB2", b"\0\0\0\x04CNB3"] { + let mut f = Fixture::new().await; + let cell = f.cell(4).await; + let original = f.node.advertisement().bundle_head().unwrap(); + let mut body = vec![0; catalog_index::HEADER_BYTES]; + body[..magic.len()].copy_from_slice(magic); + let head = NodeBundleHead { + epoch: EPOCH, + digest: Digest::from_bytes(*blake3::hash(&body).as_bytes()), + selected_through: original.selected_through(), + }; + assert!(catalog_index::body_digest(&Bytes::from(body.clone())).is_err()); + f.layout + .store() + .put_exact( + &f.layout + .node_coverage_bundle_path(&[1; 16], EPOCH, head.digest.as_bytes()), + Bytes::from(body), + ) + .await + .unwrap(); + // Simulate an authority-pinned old head. No new writer can select these + // bytes, but recovery must also refuse them without a compatibility GET. + let mut advertisement = f.node.advertisement().clone(); + advertisement.bundle = Some(head); + advertisement.generation += 1; + f.node = f + .directory + .update_advertisement(&f.node, advertisement, NOW) + .await + .unwrap(); + f.count.reset(); + assert!(matches!( + f.directory + .load_bundle_coverage(&cell.authority, &cell.control, Limits::default()) + .await, + Err(Error::Node("unsupported bundle index header")) + )); + assert_eq!( + f.count + .requests() + .iter() + .filter(|read| read.location.ends_with(".nodebundle")) + .count(), + 1 + ); + f.count.reset(); + assert!(matches!( + catalog_index::ensure_drained(&f.layout, head_session(&f), head).await, + Err(Error::Node("unsupported bundle index header")) + )); + assert_eq!(f.count.requests().len(), 1); + f.count.reset(); + assert!(matches!( + catalog_index::binding_inventory(&f.layout, head_session(&f), head).await, + Err(Error::Node("unsupported bundle index header")) + )); + assert_eq!(f.count.requests().len(), 1); + } +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/integrity.rs b/crates/cellule-runtime/src/node/bundle/tests/index/integrity.rs new file mode 100644 index 00000000..e458c74e --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/index/integrity.rs @@ -0,0 +1,113 @@ +use super::*; + +#[tokio::test] +async fn indexed_shard_corruption_and_missing_reused_shards_fail_closed() { + let mut f = Fixture::new().await; + let mut a = f.cell(4).await; + let mut b = f.cell(5).await; + assert_ne!( + catalog_index::shard( + a.authority.layout().application_id(), + a.control.value().cell.as_bytes() + ), + catalog_index::shard( + b.authority.layout().application_id(), + b.control.value().cell.as_bytes() + ) + ); + let (_, frames, assigned) = f.append(&mut a, 2); + let proposal = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (selected, _) = f + .directory + .select_node_bundle(&f.node, &proposal, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + f.node = selected; + let old_path = + f.layout + .node_coverage_bundle_path(&[1; 16], EPOCH, proposal.head.digest.as_bytes()); + let (_, frames, assigned) = f.append(&mut b, 2); + let sibling = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (selected, _) = f + .directory + .select_node_bundle(&f.node, &sibling, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + f.node = selected; + f.count.reset(); + assert_eq!( + f.directory + .load_bundle_coverage(&a.authority, &a.control, Limits::default()) + .await + .unwrap() + .commit_sequence(), + 2 + ); + let reads: Vec<_> = f + .count + .requests() + .into_iter() + .filter(|read| read.location.ends_with(".nodebundle")) + .collect(); + assert_eq!( + reads.len(), + 4, + "a reused shard/history does not require the old root header" + ); + assert_eq!( + reads + .iter() + .filter(|read| read.location == old_path.as_ref()) + .count(), + 3 + ); + // The head/header stays unchanged; the independently authenticated reused + // shard must still detect a corrupt provider payload in the old object. + let mut corrupted = proposal.body.to_vec(); + corrupted[catalog_index::HEADER_BYTES + 8] ^= 1; + f.layout + .store() + .put_overwrite(&old_path, Bytes::from(corrupted)) + .await + .unwrap(); + assert!( + f.directory + .load_bundle_coverage(&a.authority, &a.control, Limits::default()) + .await + .is_err() + ); + // Unrelated Cell data remains readable; complete maintenance inventory must + // refuse the missing/corrupt dependency rather than omitting this Cell. + assert_eq!( + f.directory + .load_bundle_coverage(&b.authority, &b.control, Limits::default()) + .await + .unwrap() + .commit_sequence(), + 2 + ); + assert!( + load_catalog( + &f.layout, + head_session(&f), + f.node.advertisement().bundle_head().unwrap() + ) + .await + .is_err() + ); + f.layout.store().delete(&old_path).await.unwrap(); + assert!( + f.directory + .load_bundle_coverage(&a.authority, &a.control, Limits::default()) + .await + .is_err() + ); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/inventory.rs b/crates/cellule-runtime/src/node/bundle/tests/index/inventory.rs index edb8deb6..2c41aac8 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/inventory.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/inventory.rs @@ -37,7 +37,7 @@ async fn streamed_maintenance_inventory_verifies_the_last_shard_before_accepting .count .requests() .into_iter() - .filter(|read| read.location.ends_with(".cnb")) + .filter(|read| read.location.ends_with(".nodebundle")) .collect(); assert!(reads.len() > 240, "closure checks every nonempty shard"); assert!( diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/index/mod.rs index 76c563ee..131ca307 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/mod.rs @@ -43,13 +43,14 @@ mod bootstrap; mod checkpoint; mod checkpoint_cohort; mod cohort; -mod compatibility; mod composed; mod copy_on_write; mod density; mod encoding; mod extension; +mod format; mod history_cohort; +mod integrity; mod inventory; mod metadata_cohort; mod metadata_windows; diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs index 417a7d22..bcd4bea6 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs @@ -325,7 +325,7 @@ impl ObjectStore for PausedBundleStore { payload: object_store::PutPayload, options: object_store::PutOptions, ) -> object_store::Result { - if path.as_ref().ends_with(".cnb") && self.armed.swap(false, Ordering::AcqRel) { + if path.as_ref().ends_with(".nodebundle") && self.armed.swap(false, Ordering::AcqRel) { self.entered.notify_one(); self.resume.notified().await; } diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/preparation.rs b/crates/cellule-runtime/src/node/bundle/tests/index/preparation.rs index 3602a0ea..63258530 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/preparation.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/preparation.rs @@ -55,7 +55,7 @@ async fn prepared_shards_remove_repeat_reads_without_changing_the_proposal() { .requests() .into_iter() .filter(|read| { - read.location.ends_with(".cnb") + read.location.ends_with(".nodebundle") && read.kind == cellule_store::test_support::ObjectReadKind::Range }) .count(); @@ -74,7 +74,7 @@ async fn prepared_shards_remove_repeat_reads_without_changing_the_proposal() { .requests() .into_iter() .filter(|read| { - read.location.ends_with(".cnb") + read.location.ends_with(".nodebundle") && read.kind == cellule_store::test_support::ObjectReadKind::Range }) .count(); @@ -98,7 +98,7 @@ async fn prepared_shards_remove_repeat_reads_without_changing_the_proposal() { f.count .requests() .iter() - .filter(|read| read.location.ends_with(".cnb") + .filter(|read| read.location.ends_with(".nodebundle") && read.kind == cellule_store::test_support::ObjectReadKind::Range) .count() >= 5, diff --git a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs index a9654b91..c1646a7f 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs @@ -39,7 +39,8 @@ impl ObjectStore for ManagedStore { payload: object_store::PutPayload, options: object_store::PutOptions, ) -> object_store::Result { - let number = if path.as_ref().ends_with(".cnb") && self.armed.load(Ordering::Acquire) { + let number = if path.as_ref().ends_with(".nodebundle") && self.armed.load(Ordering::Acquire) + { Some(self.puts.fetch_add(1, Ordering::AcqRel)) } else { None diff --git a/crates/cellule-runtime/src/node/bundle/tests/ranges.rs b/crates/cellule-runtime/src/node/bundle/tests/ranges.rs index b47276b7..1f38b6a8 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/ranges.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/ranges.rs @@ -216,7 +216,7 @@ async fn corrupt_origin_dependency_and_overlapping_ranges_fail_closed() { .layout .node_coverage_bundle_path(&[1; 16], EPOCH, prepared.head.digest.as_bytes()); let mut corrupt = prepared.body.to_vec(); - // Native bodies precede the detached history in CNB3. Corrupt the actual + // Native bodies precede the detached history in Node Bundle v1. Corrupt the actual // retained native extent, rather than an older superseded history page. let native_offset = prepared .catalog diff --git a/docs/bundle-coverage-implementation.md b/docs/bundle-coverage-implementation.md index dcc8c7f2..2fbf1be1 100644 --- a/docs/bundle-coverage-implementation.md +++ b/docs/bundle-coverage-implementation.md @@ -174,9 +174,9 @@ unknown fields are rejected. Old readers must not operate on those records. Use a fresh development prefix and initialize the lane before native issuance; there is no live-format migration or same-boot bundle-epoch rotation. -New `CNB3` objects contain a fixed authenticated root index, changed catalog -shards, native extents and detached histories under the existing -`cells/v1/node-logs///coverage/v1/.cnb` path. The head digest +Node Bundle v1 objects contain a fixed authenticated root index, changed catalog +shards, native extents and detached histories under the new +`cells/v1/node-logs///coverage/node-bundle/v1/.nodebundle` path. The head digest authenticates the 32 KiB header; shard digests authenticate each binding row and its exact history extent; history digests authenticate every native reference. New histories share the one immutable upload. Old native bodies are never copied @@ -202,12 +202,21 @@ before the first history read. This bounds protocol input, not total heap usage or host admission. Hydration alone does not rewrite a shard; modifications to an unloaded history reject. -`CNB1` complete catalogs and `CNB2` inline indexed shards remain readable, with -their original 32-inline-locator bound. Updating them selects `CNB3` without -changing Cell pins; unchanged inline shards can remain referenced. Older binaries -cannot read `CNB3`. Upgrade every recovery consumer before selecting it, using a -fresh development prefix. This does not migrate unbound live actor activations -or enable bundle responses. +Node Bundle v1 is a hard format cutover. Its root identifier is +`cellule.node-bundle.v1` followed by a NUL byte; catalog shards, catalog rows, +and Cell histories use `cellule.catalog-shard.v1`, `cellule.catalog-rows.v1`, +and `cellule.cell-history.v1` respectively. The 32-KiB header stores the root +identifier, a big-endian u32 payload length, the authenticated index payload, +and zero padding. Inner identifiers use the existing bounded length encoding. + +Writers and readers use only the new identifiers and +`coverage/node-bundle/v1/.nodebundle` objects. Previous bundle formats +and object paths are rejected; there are no aliases, fallback reads, or automatic +migration. Upgrade every publication and recovery consumer together. Existing +bundle-backed prefixes are unsupported; start the new binaries with a fresh +prefix. Before retiring an existing deployment, drain and materialize its +complete acknowledged history with the previous binary. This change does not +rewrite existing objects or discard recovery obligations. | Development bound | Value | | --- | ---: | @@ -218,7 +227,6 @@ or enable bundle responses. | Native frames per selection | 64 | | Detached exact frame references per Cell | 256 | | One encoded history | 32 KiB | -| Legacy inline locators per Cell | 32 | | Native suffix bytes verified per Cell | 4 MiB | | Distinct base-root origin dependencies verified per Cell | 65,536 | @@ -233,6 +241,9 @@ latest application measurement fails availability. See the ## Verification and remaining delivery +The milestone measurements and verification counts below predate the Node Bundle +v1 cutover. They describe the tested revisions, not the new format's performance. + The atomic-coverage snapshot passed all twelve contributor checks: **1,940 top-level workspace cases passed, 38 ignored; 60 local LTX cases passed**. The first full run timed out at an unchanged five-second follower-backlog drain. @@ -257,7 +268,7 @@ and Cell CAS. The 256-reference boundary also costs four in this fixture; an oversized changed image preserves the ordinary bundle fallback. The new one-Cell update among 1,000 bindings uses **35,416 metadata bytes**, versus -35,223 with inline `CNB2`. Recovery uses four coverage-object ranges: header, +35,223 with the previous inline format. Recovery uses four coverage-object ranges: header, shard, history and native frame. Prefix checkpoint uses three ranges with no native-body reads. The extra history read buys selective dense lookup; it is not a claimed read TPS improvement. One shared shard test corrupts a sibling history: @@ -265,8 +276,9 @@ the requested Cell still selects without fetching or rewriting it, while a complete reconstruction load rejects the missing/corrupt sibling. Additional codec tests cover original pin/session/epoch, exact count/native-byte claims, truncation, the 256-reference limit and aggregate admission before history I/O. -Both legacy complete catalogs and inline indexed suffixes migrate with the same -Cell pin. All raw logs remain outside Git. +That milestone also verified migration from previous complete catalogs and +inline indexed suffixes with the same Cell pin. The hard cutover removes that +support. All raw logs remain outside Git. The frozen detached-history and streaming-materialization source passed all twelve contributor checks: **1,908 workspace tests passed, 38 ignored; 60 local @@ -328,7 +340,7 @@ application TPS: The index regression reproduces a **783,146-byte** metadata rewrite for one update among 1,000 Cells on the old source. The same test measures **35,223 bytes** -with the original `CNB2` (95.5% less), retains one immutable PUT plus one node CAS, and recovers +with the original inline format (95.5% less), retains one immutable PUT plus one node CAS, and recovers through three coverage-object ranges: header, chosen shard and native frame. An exact materialized-prefix checkpoint among those 1,000 Cells falls from **253 coverage-object reads to two**, without reading old native frames. A @@ -345,7 +357,7 @@ requires at least 215 commands per Cell checkpoint if that four-PUT path and 64/64 cohorts hold, before retries or maintenance. Real larger checkpoints must be measured; that original 32-locator safety ceiling did not satisfy this constraint. -Additional tests cover legacy migration with the same Cell pin, a reused shard +At that milestone, additional tests covered migration with the same Cell pin, a reused shard without reading its old header, corrupt and missing reused shards, and refusal to omit a corrupt shard from complete inventory. Bootstrap enrollment accepts the runtime's command-zero root only before any native issuance; the first diff --git a/docs/managed-bundle-pipeline.md b/docs/managed-bundle-pipeline.md index 754909cd..7650d3be 100644 --- a/docs/managed-bundle-pipeline.md +++ b/docs/managed-bundle-pipeline.md @@ -8,10 +8,13 @@ completes 873.35–1,038.80 writes/s and 8,877.55–10,897.50 reads/s, with **The 2,000-Cell mixed-load goal remains unmet; these short pairs do not qualify sustainable capacity.** +These measurements predate the [Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds). +They are historical evidence, not measurements of the new format. + ## Publication and admission `NodeBundlePublicationAuthority::begin_round` acquires catalog ordering and -observes its original head. The round stages immutable CNB3 proposals, uploads +observes its original head. The round stages immutable Node Bundle v1 proposals, uploads them and selects each through complete fresh dependency verification and the existing authority CAS. Heartbeat state is released during staging and upload. Binding, checkpoint and close operations retain the same catalog ordering. diff --git a/docs/single-pass-bundle-encoding.md b/docs/single-pass-bundle-encoding.md index 55d8a4d0..18739980 100644 --- a/docs/single-pass-bundle-encoding.md +++ b/docs/single-pass-bundle-encoding.md @@ -4,6 +4,9 @@ Measured October 10, 2026. Canonical shard encoding uses less CPU in the isolate release replay. Two application pairs give conflicting throughput results. **The 2,000-Cell mixed-load goal remains unqualified.** +These measurements predate the [Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds). +They are historical evidence, not measurements of the new format. + ## Change and compatibility Baseline is main `aaed329e28236b6cb3314c40add0669809d180b1`; candidate is @@ -13,7 +16,7 @@ and fills its same-width history extent fields after assigning native offsets. It avoids serializing every sibling control record twice. Plans exist only during encoding and retain no authority or availability proof. -CNB3 bytes, 64-frame/4-MiB cohorts, admission budgets, complete local +At the tested revisions, bundle bytes, 64-frame/4-MiB cohorts, admission budgets, complete local self-verification, fresh origin verification and authoritative selection CAS are unchanged. The 64-Cell/2,000-binding fixture compares every encoded byte, header digest and resulting catalog against the original encoder over six captures, diff --git a/docs/typed-bundle-staging.md b/docs/typed-bundle-staging.md index 5bd8f732..191ecf9e 100644 --- a/docs/typed-bundle-staging.md +++ b/docs/typed-bundle-staging.md @@ -7,6 +7,8 @@ is pending. At `d42b978` the managed serving publisher remained serial. The runtime producer. **No new application throughput or capacity claim follows from this change.** +The verification evidence below predates the [Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds). + ## Protocol and ownership | Phase | Value | Permitted action | @@ -32,8 +34,8 @@ from the fresh origin view required by selection, which still reads the complete uploaded object and verifies every needed reconstruction dependency. Immutable PUT completion supplies no authority or response credit. A changed catalog or fenced writer rejects selection; heartbeat-only rebasing keeps its canonical -checks. CNB3 encoding and the existing frame, history and object bounds remain -unchanged. +checks. At the tested revisions, bundle encoding and the existing frame, history +and object bounds remained unchanged. ## Deterministic checks From f7ba1edc78f2acba0c9c59921c2c3557d615e119 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 11:03:05 -0700 Subject: [PATCH 02/31] fix(runtime): preserve foreground capacity during publication --- crates/cellule-axum/examples/fleet/mod.rs | 3 +- crates/cellule-axum/examples/sql.rs | 30 +- .../cellule-axum/examples/sql_metrics/mod.rs | 37 +++ .../examples/sql_metrics/publication.rs | 198 ++++++++++++ .../examples/sql_metrics/tests.rs | 51 +++ .../cellule-runtime/src/cell/actor/handle.rs | 26 +- .../src/cell/actor/materialization/mod.rs | 27 +- .../src/cell/actor/materialization/tests.rs | 52 +++ .../src/cell/actor/requests.rs | 5 +- .../cellule-runtime/src/cell/actor/runtime.rs | 7 + crates/cellule-runtime/src/fleet/resource.rs | 181 ++++++++++- crates/cellule-runtime/src/fleet/telemetry.rs | 300 ++++++++++++++++++ .../src/node/bundle/index/preparation/mod.rs | 6 +- .../src/node/bundle/selection.rs | 93 +++--- .../src/node/bundle/tests/pipeline.rs | 37 +++ .../cellule-runtime/src/node/directory/mod.rs | 13 + .../src/node/durability/mod.rs | 3 +- .../src/node/durability/publication/mod.rs | 45 ++- .../node/durability/publication/pipeline.rs | 73 +++-- .../src/node/durability/receipts.rs | 1 + .../src/node/log_shipper/mod.rs | 4 + crates/cellule-runtime/src/publication/mod.rs | 51 +-- docs/managed-bundle-pipeline.md | 47 ++- 23 files changed, 1182 insertions(+), 108 deletions(-) create mode 100644 crates/cellule-axum/examples/sql_metrics/publication.rs create mode 100644 crates/cellule-runtime/src/cell/actor/materialization/tests.rs diff --git a/crates/cellule-axum/examples/fleet/mod.rs b/crates/cellule-axum/examples/fleet/mod.rs index 12431b4f..208adb97 100644 --- a/crates/cellule-axum/examples/fleet/mod.rs +++ b/crates/cellule-axum/examples/fleet/mod.rs @@ -83,7 +83,8 @@ impl Config { metrics: Arc, ) -> Result { let tls = self.tls()?; - let directory = NodeDirectory::new(layout, tls.fleet(), code, code); + let directory = NodeDirectory::new(layout, tls.fleet(), code, code) + .with_telemetry(runtime.telemetry_handle()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .map_err(transport_error)?; diff --git a/crates/cellule-axum/examples/sql.rs b/crates/cellule-axum/examples/sql.rs index 5130b6c0..3d900db6 100644 --- a/crates/cellule-axum/examples/sql.rs +++ b/crates/cellule-axum/examples/sql.rs @@ -154,6 +154,7 @@ struct Order { struct ServiceState { app: ApplicationHandle, targets: Arc>, + metrics: Arc, } impl FromRef for ApplicationHandle { @@ -182,6 +183,18 @@ async fn create_order( app: Cellule, State(state): State, Json(input): Json, +) -> Result<(StatusCode, CellJson), HttpError> { + let result = create_order_inner(app, &state, input).await; + if let Err(error) = &result { + state.metrics.http_failure(error, true); + } + result +} + +async fn create_order_inner( + app: Cellule, + state: &ServiceState, + input: CreateOrder, ) -> Result<(StatusCode, CellJson), HttpError> { let target = state.target(input.id)?; let sql = app.sql::(target)?; @@ -234,7 +247,11 @@ async fn get_order( HttpPath(id): HttpPath, MinimumReceipt(minimum): MinimumReceipt, ) -> Result>, HttpError> { - read_order(&app, state.target(id)?, id, minimum).await + let result = async { read_order(&app, state.target(id)?, id, minimum).await }.await; + if let Err(error) = &result { + state.metrics.http_failure(error, false); + } + result } async fn read_order( @@ -531,6 +548,16 @@ async fn main() -> ExampleResult<()> { }, None => serde_json::Value::Null, }; + sample["retained_work"] = match runtime.retained_work() { + Ok(snapshot) => serde_json::json!({ + "total": snapshot.used.retained_bytes(), + "capacity": snapshot.limit.retained_bytes(), + "stages": cellule_runtime::fleet::resource::RetentionStage::ALL.iter() + .map(|stage| (stage.label().to_owned(), snapshot.retained_by_stage[*stage as usize])) + .collect::>(), + }), + Err(error) => serde_json::json!({"error": error.to_string()}), + }; sample["runtime"] = serde_json::json!({ "active_cells": stats.active_cells(), "retained_bytes": stats.retained_bytes(), @@ -549,6 +576,7 @@ async fn main() -> ExampleResult<()> { .with_state(ServiceState { app: typed, targets: Arc::new(targets.clone()), + metrics: query_metrics.clone(), }); // This local fixture binds only loopback. A product installs its own // authentication and authorization before exposing these handlers. diff --git a/crates/cellule-axum/examples/sql_metrics/mod.rs b/crates/cellule-axum/examples/sql_metrics/mod.rs index 9bde586d..7444150a 100644 --- a/crates/cellule-axum/examples/sql_metrics/mod.rs +++ b/crates/cellule-axum/examples/sql_metrics/mod.rs @@ -10,6 +10,7 @@ use std::{ }; mod capture; +mod publication; mod storage; mod submission; use capture::CaptureMetrics; @@ -42,6 +43,7 @@ pub(super) struct QueryMetrics { selected_roots: AtomicU64, materialized_commits: AtomicU64, shared: SharedMetrics, + publication: publication::Metrics, follower_frames: AtomicU64, follower_sync_calls: AtomicU64, follower_failures: AtomicU64, @@ -314,6 +316,35 @@ impl CellTelemetry for QueryMetrics { self.writes.objects.fetch_add(objects, Ordering::Relaxed); self.writes.bytes.fetch_add(bytes, Ordering::Relaxed); } + fn node_publication_started( + &self, + phase: cellule_runtime::fleet::telemetry::NodePublicationPhase, + ) { + self.publication.started(phase); + } + fn node_publication_phase( + &self, + phase: cellule_runtime::fleet::telemetry::NodePublicationPhase, + elapsed: Duration, + outcome: cellule_runtime::fleet::telemetry::PublicationPhaseOutcome, + ) { + self.publication.completed(phase, elapsed, outcome); + } + fn node_publication_cohort( + &self, + cohort: cellule_runtime::fleet::telemetry::NodePublicationCohort, + ) { + self.publication.cohort(cohort); + } + fn node_publication_fallback( + &self, + reason: cellule_runtime::fleet::telemetry::PublicationPipelineFallback, + ) { + self.publication.fallback(reason); + } + fn admission_refused(&self, reason: cellule_runtime::fleet::telemetry::AdmissionRefusal) { + self.publication.refused(reason); + } fn shared_publication(&self, timing: SharedPublicationTiming) { self.shared.cohorts.fetch_add(1, Ordering::Relaxed); self.shared @@ -374,6 +405,10 @@ impl CellTelemetry for QueryMetrics { } impl QueryMetrics { + pub(super) fn http_failure(&self, error: &cellule_axum::HttpError, write: bool) { + self.publication.http_failure(error, write); + } + pub(super) fn peer_phase(&self, phase: PeerPhase, elapsed: Duration) { self.peer[phase as usize].observe(elapsed); } @@ -425,6 +460,7 @@ impl QueryMetrics { ] { histograms.insert(name.into(), histogram.raw()); } + histograms.extend(self.publication.histograms()); histograms.extend(self.capture.window_snapshot()); histograms.extend(self.submission.window_snapshot()); value["histograms"] = histograms.into(); @@ -498,6 +534,7 @@ impl QueryMetrics { .collect(); serde_json::json!({ "schema_version": 3, + "node_publication": self.publication.snapshot(), "sample_session": self.sample_session.get_or_init(uuid::Uuid::now_v7).to_string(), "sample_elapsed_ns": nanos(self.sample_origin.get_or_init(std::time::Instant::now).elapsed()), "host_capacity": self.host_capacity, diff --git a/crates/cellule-axum/examples/sql_metrics/publication.rs b/crates/cellule-axum/examples/sql_metrics/publication.rs new file mode 100644 index 00000000..50da32cc --- /dev/null +++ b/crates/cellule-axum/examples/sql_metrics/publication.rs @@ -0,0 +1,198 @@ +//! Managed bundle phases and retained-work pressure, with finite labels. +use super::*; +use cellule_runtime::fleet::telemetry::{ + AdmissionRefusal, NodePublicationCohort, NodePublicationPhase, PublicationPhaseOutcome, + PublicationPipelineFallback, +}; + +#[derive(Default)] +struct Phase { + active: AtomicU64, + succeeded: AtomicU64, + failed: AtomicU64, + cancelled: AtomicU64, + elapsed: Histogram, +} + +#[derive(Default)] +pub(super) struct Metrics { + phases: [Phase; NodePublicationPhase::ALL.len()], + http_failures: [[AtomicU64; FAILURE_LABELS.len()]; 2], + refusals: [AtomicU64; AdmissionRefusal::ALL.len()], + fallbacks: [AtomicU64; 4], + cohorts: AtomicU64, + captures: AtomicU64, + frames: AtomicU64, + bytes: AtomicU64, + checkpoints: AtomicU64, + max_frames: AtomicU64, + max_bytes: AtomicU64, +} + +const FAILURE_LABELS: [&str; 13] = [ + "publication_backlog", + "pending_publication_bytes", + "node_retained_bytes", + "mailbox_requests", + "mailbox_bytes", + "other_capacity", + "fenced", + "draining", + "closed", + "deadline", + "pending_publication", + "other", + "outcome_unknown", +]; + +fn failure_class(error: &(dyn std::error::Error + 'static)) -> usize { + let mut source = Some(error); + for _ in 0..16 { + let Some(error) = source else { + break; + }; + if let Some(runtime) = error.downcast_ref::() { + use cellule_runtime::Error; + if let Error::Shared(original) = runtime { + return failure_class(original.as_ref()); + } + return match runtime { + Error::Capacity("publication backlog") => 0, + Error::Capacity("pending publication bytes") => 1, + Error::Capacity("node retained bytes") => 2, + Error::Capacity("Cell mailbox requests") => 3, + Error::Capacity("Cell mailbox bytes") => 4, + Error::Capacity(_) => 5, + Error::Fenced => 6, + Error::CellDraining => 7, + Error::RuntimeClosed => 8, + Error::Deadline => 9, + Error::PendingPublication => 10, + _ => 11, + }; + } + source = error.source(); + } + 11 +} + +impl Metrics { + pub(super) fn http_failure(&self, error: &cellule_axum::HttpError, write: bool) { + let reason = if error.code() == "outcome_unknown" { + 12 + } else { + failure_class(error) + }; + self.http_failures[usize::from(write)][reason].fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn started(&self, phase: NodePublicationPhase) { + self.phases[phase as usize] + .active + .fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn completed( + &self, + phase: NodePublicationPhase, + elapsed: Duration, + outcome: PublicationPhaseOutcome, + ) { + let phase = &self.phases[phase as usize]; + let counter = match outcome { + PublicationPhaseOutcome::Succeeded => &phase.succeeded, + PublicationPhaseOutcome::Failed => &phase.failed, + PublicationPhaseOutcome::Cancelled => &phase.cancelled, + }; + phase.elapsed.observe(elapsed); + counter.fetch_add(1, Ordering::Relaxed); + phase.active.fetch_sub(1, Ordering::Relaxed); + } + + pub(super) fn refused(&self, reason: AdmissionRefusal) { + self.refusals[reason as usize].fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn fallback(&self, reason: PublicationPipelineFallback) { + self.fallbacks[reason as usize].fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn cohort(&self, cohort: NodePublicationCohort) { + self.cohorts.fetch_add(1, Ordering::Relaxed); + self.captures + .fetch_add(cohort.captures as u64, Ordering::Relaxed); + self.frames + .fetch_add(cohort.frames as u64, Ordering::Relaxed); + self.bytes.fetch_add(cohort.bytes as u64, Ordering::Relaxed); + self.checkpoints + .fetch_add(cohort.checkpoints as u64, Ordering::Relaxed); + self.max_frames + .fetch_max(cohort.frames as u64, Ordering::Relaxed); + self.max_bytes + .fetch_max(cohort.bytes as u64, Ordering::Relaxed); + } + + pub(super) fn histograms(&self) -> serde_json::Map { + NodePublicationPhase::ALL + .iter() + .map(|phase| { + ( + format!("node_publication_{}", phase.label()), + self.phases[*phase as usize].elapsed.raw(), + ) + }) + .collect() + } + + pub(super) fn snapshot(&self) -> serde_json::Value { + let read = |value: &AtomicU64| value.load(Ordering::Relaxed); + let phases = NodePublicationPhase::ALL + .iter() + .map(|phase| { + let value = &self.phases[*phase as usize]; + ( + phase.label().to_owned(), + serde_json::json!({ + "active": read(&value.active), "succeeded": read(&value.succeeded), + "failed": read(&value.failed), "cancelled": read(&value.cancelled), + "elapsed": value.elapsed.snapshot(), + }), + ) + }) + .collect::>(); + let http_failures = ["reads", "writes"] + .iter() + .enumerate() + .map(|(kind, label)| { + ( + (*label).to_owned(), + FAILURE_LABELS + .iter() + .enumerate() + .map(|(reason, label)| { + ( + (*label).to_owned(), + read(&self.http_failures[kind][reason]).into(), + ) + }) + .collect::>() + .into(), + ) + }) + .collect::>(); + serde_json::json!({ + "phases": phases, + "http_failures": http_failures, + "admission_refusals": AdmissionRefusal::ALL.iter().map(|reason| ( + reason.label().to_owned(), read(&self.refusals[*reason as usize]).into() + )).collect::>(), + "pipeline_fallbacks": {"checkpoints": read(&self.fallbacks[0]), + "capacity": read(&self.fallbacks[1]), "no_successor": read(&self.fallbacks[2]), + "headroom": read(&self.fallbacks[3])}, + "cohorts": {"count": read(&self.cohorts), "captures": read(&self.captures), + "frames": read(&self.frames), "native_bytes": read(&self.bytes), + "checkpoints": read(&self.checkpoints), "max_frames": read(&self.max_frames), + "max_native_bytes": read(&self.max_bytes)}, + }) + } +} diff --git a/crates/cellule-axum/examples/sql_metrics/tests.rs b/crates/cellule-axum/examples/sql_metrics/tests.rs index 6156bebb..c0712f35 100644 --- a/crates/cellule-axum/examples/sql_metrics/tests.rs +++ b/crates/cellule-axum/examples/sql_metrics/tests.rs @@ -1,5 +1,56 @@ use super::*; +#[test] +fn node_publication_metrics_keep_phase_failures_and_http_sources_separate() { + use cellule_runtime::fleet::telemetry::{ + NodePublicationPhase as Phase, PublicationPhaseOutcome, + }; + let metrics = QueryMetrics::default(); + metrics.node_publication_started(Phase::Verification); + metrics.node_publication_phase( + Phase::Verification, + Duration::from_millis(3), + PublicationPhaseOutcome::Failed, + ); + metrics.node_publication_started(Phase::Checkpoint); + metrics.node_publication_phase( + Phase::Checkpoint, + Duration::from_millis(4), + PublicationPhaseOutcome::Cancelled, + ); + metrics.http_failure( + &cellule_axum::HttpError::from(cellule_runtime::Error::Shared(std::sync::Arc::new( + cellule_runtime::Error::Capacity("pending publication bytes"), + ))), + true, + ); + let snapshot = metrics.window_snapshot(); + assert_eq!( + snapshot["node_publication"]["phases"]["verification"]["failed"], + 1 + ); + assert_eq!( + snapshot["node_publication"]["phases"]["verification"]["active"], + 0 + ); + assert_eq!( + snapshot["node_publication"]["phases"]["checkpoint"]["cancelled"], + 1 + ); + assert_eq!( + snapshot["node_publication"]["http_failures"]["writes"]["pending_publication_bytes"], + 1 + ); + assert_eq!( + snapshot["node_publication"]["http_failures"]["reads"]["pending_publication_bytes"], + 0 + ); + assert_eq!( + snapshot["histograms"]["node_publication_verification"]["total_ns"], + 3_000_000 + ); +} + #[test] fn sparse_export_preserves_cumulative_histogram_indices_and_overflow() { let histogram = Histogram::default(); diff --git a/crates/cellule-runtime/src/cell/actor/handle.rs b/crates/cellule-runtime/src/cell/actor/handle.rs index cec21e31..ad6462c0 100644 --- a/crates/cellule-runtime/src/cell/actor/handle.rs +++ b/crates/cellule-runtime/src/cell/actor/handle.rs @@ -84,7 +84,9 @@ impl WorkAdmission { // own this exact cut without racing a second global admission. Keep // the input and actual reply charged until their command is dropped. let publication = if publication_bytes != 0 && publication_bytes <= unused { - Some(self._node_bytes.split_retained(publication_bytes)?) + let mut publication = self._node_bytes.split_retained(publication_bytes)?; + publication.set_stage(crate::fleet::resource::RetentionStage::Capture)?; + Some(publication) } else { None }; @@ -587,6 +589,21 @@ impl CellHandle { kind: AdmissionKind, operation_bytes: usize, max_result_bytes: usize, + ) -> crate::Result { + let result = self.reserve_work_kind_inner(kind, operation_bytes, max_result_bytes); + if let Err(error) = &result { + self.inner + .telemetry + .admission_refused(crate::fleet::telemetry::AdmissionRefusal::from_error(error)); + } + result + } + + fn reserve_work_kind_inner( + &self, + kind: AdmissionKind, + operation_bytes: usize, + max_result_bytes: usize, ) -> crate::Result { if self.inner.shutting_down.load(Ordering::Acquire) { return Err(Error::RuntimeClosed); @@ -622,7 +639,7 @@ impl CellHandle { let retained = snapshot.used.retained_bytes(); let limit = snapshot.limit.retained_bytes(); let disk = self.inner.replica_host.local_disk_budget(); - if retained >= limit.saturating_sub(limit / 4) + if retained >= crate::fleet::resource::retained_admission_threshold(limit) || disk.used() >= disk.capacity().saturating_sub(disk.capacity() / 4) { return Err(Error::Capacity("publication backlog")); @@ -643,7 +660,10 @@ impl CellHandle { _node_bytes: self .inner .resources - .try_reserve(ResourceCost::zero().with_retained_bytes(reservation_bytes)) + .try_reserve_in( + crate::fleet::resource::RetentionStage::Request, + ResourceCost::zero().with_retained_bytes(reservation_bytes), + ) .map_err(|error| match error { Error::Capacity(_) => Error::Capacity("node retained bytes"), error => error, diff --git a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs index cb49d4b7..5229e814 100644 --- a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs +++ b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs @@ -8,6 +8,20 @@ pub(super) const CHECKPOINT_COMMANDS: u64 = 215; const MAX_ROOT_AGE: std::time::Duration = std::time::Duration::from_secs(45); const MAX_MATERIALIZERS: usize = 8; +fn reserve_materialization( + resources: &crate::fleet::resource::ResourceLedger, + bytes: usize, + progress_needed: bool, +) -> crate::Result { + resources.try_reserve_materialization( + ResourceCost::zero().with_retained_bytes(bytes), + progress_needed, + ) +} + +#[cfg(test)] +mod tests; + pub(super) fn blocks_commands(active: &ActiveCell) -> bool { active.root_debt.as_ref().is_some_and(|debt| { debt.selected.proof.locator_count() >= CHECKPOINT_COMMANDS as usize @@ -234,8 +248,11 @@ pub(super) fn dispatch( .ok_or(Error::Capacity("checkpoint metadata admission")) }) .and_then(|bytes| { - pool.resource_ledger() - .try_reserve(ResourceCost::zero().with_retained_bytes(bytes)) + reserve_materialization( + &pool.resource_ledger(), + bytes, + available == MAX_MATERIALIZERS, + ) }); let Ok(reservation) = reservation else { continue; @@ -281,10 +298,12 @@ pub(super) fn dispatch( drop(reservation); // Checkpoint this exact cut. Later selected captures can retire // independently; a rebased proof waits for the native witness. - debt.durability.checkpoint_materialized(publisher.authority(), root).await?; + pool.telemetry_handle().publication_phase(crate::fleet::telemetry::NodePublicationPhase::CheckpointWait, + debt.durability.checkpoint_materialized(publisher.authority(), root)).await?; // Transfer pre-admitted metadata to the worker. No admission // can fail after the canonical root and checkpoint are joined. - pool.bind_bundle_materialized(cell, root, Arc::clone(&debt.selected), retained).await + pool.telemetry_handle().publication_phase(crate::fleet::telemetry::NodePublicationPhase::WorkerBinding, + pool.bind_bundle_materialized(cell, root, Arc::clone(&debt.selected), retained)).await }.await; joined.cancel(); publisher.record_publication_timing(crate::fleet::telemetry::PublicationTiming { diff --git a/crates/cellule-runtime/src/cell/actor/materialization/tests.rs b/crates/cellule-runtime/src/cell/actor/materialization/tests.rs new file mode 100644 index 00000000..bd3c8843 --- /dev/null +++ b/crates/cellule-runtime/src/cell/actor/materialization/tests.rs @@ -0,0 +1,52 @@ +use super::*; +use crate::fleet::resource::{ResourceCost, ResourceLedger, RetentionStage}; + +#[test] +fn working_roots_leave_room_for_foreground_and_existing_debt_can_progress() { + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(64 << 20)); + let publisher = resources + .try_reserve_in( + RetentionStage::Publisher, + ResourceCost::zero().with_retained_bytes(22 << 20), + ) + .unwrap(); + let mut selected = resources + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(10 << 20), + ) + .unwrap(); + let root = reserve_materialization(&resources, 5 << 20, true).unwrap(); + assert!( + reserve_materialization(&resources, 5 << 20, false).is_err(), + "parallel root buffers consumed foreground burst headroom" + ); + let commands = resources + .try_reserve_in( + RetentionStage::Request, + ResourceCost::zero().with_retained_bytes(8 << 20), + ) + .unwrap(); + let snapshot = resources.snapshot().unwrap(); + assert!( + snapshot.used.retained_bytes() + < crate::fleet::resource::retained_admission_threshold(snapshot.limit.retained_bytes()) + ); + drop(commands); + drop(root); + // Already-retained debt must not wait forever for the foreground margin. + let more_debt = resources + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(16 << 20), + ) + .unwrap(); + let cleanup = reserve_materialization(&resources, 5 << 20, true).unwrap(); + assert!(reserve_materialization(&resources, 5 << 20, false).is_err()); + drop(more_debt); + selected.shrink_retained(0).unwrap(); + drop(cleanup); + drop(publisher); + drop(selected); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); +} diff --git a/crates/cellule-runtime/src/cell/actor/requests.rs b/crates/cellule-runtime/src/cell/actor/requests.rs index fd3fae6d..78d5e2c7 100644 --- a/crates/cellule-runtime/src/cell/actor/requests.rs +++ b/crates/cellule-runtime/src/cell/actor/requests.rs @@ -324,7 +324,10 @@ pub(super) fn reserve_pending_publication( // Larger cuts still need their full charged admission. Returning unused // result capacity first makes it available without changing the node limit. pool.resource_ledger() - .try_reserve(ResourceCost::zero().with_retained_bytes(bytes)) + .try_reserve_in( + crate::fleet::resource::RetentionStage::Capture, + ResourceCost::zero().with_retained_bytes(bytes), + ) .map_err(|error| match error { Error::Capacity(_) => Error::Capacity("pending publication bytes"), error => error, diff --git a/crates/cellule-runtime/src/cell/actor/runtime.rs b/crates/cellule-runtime/src/cell/actor/runtime.rs index 849b4a25..a869d7d3 100644 --- a/crates/cellule-runtime/src/cell/actor/runtime.rs +++ b/crates/cellule-runtime/src/cell/actor/runtime.rs @@ -62,6 +62,13 @@ fn permille(used: u64, limit: u64) -> crate::Result { } impl CellRuntime { + /// Diagnostic retained ownership from one locked resource snapshot. + /// The returned stage counts reconcile exactly to its retained total; + /// they grant no admission or durability proof. + pub fn retained_work(&self) -> crate::Result { + self.inner.resources.snapshot() + } + pub(crate) fn session(&self) -> SessionId { self.inner.session } diff --git a/crates/cellule-runtime/src/fleet/resource.rs b/crates/cellule-runtime/src/fleet/resource.rs index b921020e..aa407ffa 100644 --- a/crates/cellule-runtime/src/fleet/resource.rs +++ b/crates/cellule-runtime/src/fleet/resource.rs @@ -23,6 +23,16 @@ pub(crate) const READ_REPLICA_NATIVE_BYTES: usize = 12 * 1024 * 1024; pub(crate) const READ_REPLICA_FILE_DESCRIPTORS: usize = 4; pub(crate) const HYDRATION_JOB_CAPACITY: usize = 2; +pub(crate) const fn retained_admission_threshold(limit: usize) -> usize { + limit.saturating_sub(limit / 4) +} + +const fn publication_working_threshold(limit: usize) -> usize { + // Keep the original quarter for publication progress, and another eighth + // below foreground shedding for bursts of admitted SQL/result buffers. + retained_admission_threshold(limit).saturating_sub(limit / 8) +} + /// Bounded resources owned by one runtime admission token. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct ResourceCost { @@ -310,6 +320,50 @@ impl ResourceCost { } } +/// Diagnostic ownership of retained work; stages do not change admission. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RetentionStage { + /// Reservations outside the instrumented publication/request paths. + Other, + /// Accepted operation and response bytes. + Request, + /// Captured data awaiting publication and local cleanup. + Capture, + /// The original publisher's working buffers and metadata cache. + Publisher, + /// Pre-admitted successor buffers and receipt allowance. + Pipeline, + /// Selected dependency proofs retained by original consumers. + Selection, + /// Root reconstruction and joined checkpoint work. + Materialization, +} + +impl RetentionStage { + /// All finite ownership classes, in snapshot order. + pub const ALL: [Self; 7] = [ + Self::Other, + Self::Request, + Self::Capture, + Self::Publisher, + Self::Pipeline, + Self::Selection, + Self::Materialization, + ]; + /// Stable diagnostic label, without Cell or request identity. + pub const fn label(self) -> &'static str { + match self { + Self::Other => "other", + Self::Request => "request", + Self::Capture => "capture", + Self::Publisher => "publisher", + Self::Pipeline => "pipeline", + Self::Selection => "selection", + Self::Materialization => "materialization", + } + } +} + /// Point-in-time usage and limits for one resource ledger. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ResourceSnapshot { @@ -317,6 +371,9 @@ pub struct ResourceSnapshot { pub used: ResourceCost, /// Cost the ledger admits. pub limit: ResourceCost, + /// Retained ownership, indexed by [`RetentionStage::ALL`]. Its sum equals + /// `used.retained_bytes()` in this same locked snapshot. + pub retained_by_stage: [usize; 7], } #[derive(Clone)] @@ -342,6 +399,7 @@ impl ResourceLedger { snapshot: Mutex::new(ResourceSnapshot { used: ResourceCost::zero(), limit, + retained_by_stage: [0; 7], }), released: tokio::sync::Notify::new(), }), @@ -353,6 +411,39 @@ impl ResourceLedger { } pub(crate) fn try_reserve(&self, cost: ResourceCost) -> Result { + self.try_reserve_in(RetentionStage::Other, cost) + } + + pub(crate) fn try_reserve_in( + &self, + stage: RetentionStage, + cost: ResourceCost, + ) -> Result { + self.try_reserve_inner(stage, cost, false) + } + + /// Optional overlap must leave the foreground's existing progress reserve. + /// Check and admission share the lock so concurrent receipts cannot defeat it. + pub(crate) fn try_reserve_pipeline(&self, cost: ResourceCost) -> Result { + self.try_reserve_inner(RetentionStage::Pipeline, cost, true) + } + + pub(crate) fn try_reserve_materialization( + &self, + cost: ResourceCost, + progress_needed: bool, + ) -> Result { + // The actor allows one cleanup when no original materializer is alive. + // Debt already above the working threshold must still be reducible. + self.try_reserve_inner(RetentionStage::Materialization, cost, !progress_needed) + } + + fn try_reserve_inner( + &self, + stage: RetentionStage, + cost: ResourceCost, + preserve_headroom: bool, + ) -> Result { let mut state = self .state .snapshot @@ -365,14 +456,32 @@ impl ResourceLedger { if !next.fits_within(state.limit) { return Err(Error::Capacity("resource ledger")); } + if preserve_headroom + && next.retained_bytes() >= publication_working_threshold(state.limit.retained_bytes()) + { + return Err(Error::Capacity(match stage { + RetentionStage::Materialization => "root materialization headroom", + _ => "bundle pipeline headroom", + })); + } state.used = next; + state.retained_by_stage[stage as usize] += cost.retained_bytes; Ok(ResourceReservation { ledger: self.clone(), cost, + stage, }) } pub(crate) async fn reserve(&self, cost: ResourceCost) -> Result { + self.reserve_in(RetentionStage::Other, cost).await + } + + pub(crate) async fn reserve_in( + &self, + stage: RetentionStage, + cost: ResourceCost, + ) -> Result { if !cost.fits_within(self.snapshot()?.limit) { return Err(Error::Capacity("resource ledger")); } @@ -380,7 +489,7 @@ impl ResourceLedger { // Register before checking capacity so a release between the // failed attempt and await cannot leave the waiter asleep. let released = self.state.released.notified(); - match self.try_reserve(cost) { + match self.try_reserve_in(stage, cost) { Ok(reservation) => return Ok(reservation), Err(Error::Capacity(_)) => released.await, Err(error) => return Err(error), @@ -476,11 +585,12 @@ impl ResourceLedger { Ok(()) } - fn release(&self, cost: ResourceCost) { + fn release(&self, stage: RetentionStage, cost: ResourceCost) { if let Ok(mut state) = self.state.snapshot.lock() && let Some(used) = state.used.checked_sub(cost) { state.used = used; + state.retained_by_stage[stage as usize] -= cost.retained_bytes; } } } @@ -490,9 +600,25 @@ impl ResourceLedger { pub(crate) struct ResourceReservation { ledger: ResourceLedger, cost: ResourceCost, + stage: RetentionStage, } impl ResourceReservation { + /// Reclassifies the original admitted lifetime without changing capacity. + pub(crate) fn set_stage(&mut self, stage: RetentionStage) -> Result<()> { + if stage != self.stage { + let mut state = self + .ledger + .state + .snapshot + .lock() + .map_err(|_| Error::Control("resource ledger lock poisoned"))?; + state.retained_by_stage[self.stage as usize] -= self.cost.retained_bytes; + state.retained_by_stage[stage as usize] += self.cost.retained_bytes; + self.stage = stage; + } + Ok(()) + } pub(crate) const fn retained_bytes(&self) -> usize { self.cost.retained_bytes } @@ -509,6 +635,7 @@ impl ResourceReservation { Ok(Self { ledger: self.ledger.clone(), cost: ResourceCost::zero().with_retained_bytes(bytes), + stage: self.stage, }) } @@ -520,8 +647,10 @@ impl ResourceReservation { .checked_sub(bytes) .ok_or(Error::Capacity("capture cleanup cannot grow admission"))?; self.cost.retained_bytes = bytes; - self.ledger - .release(ResourceCost::zero().with_retained_bytes(released)); + self.ledger.release( + self.stage, + ResourceCost::zero().with_retained_bytes(released), + ); self.ledger.state.released.notify_waiters(); Ok(()) } @@ -529,7 +658,7 @@ impl ResourceReservation { impl Drop for ResourceReservation { fn drop(&mut self) { - self.ledger.release(self.cost); + self.ledger.release(self.stage, self.cost); self.ledger.state.released.notify_waiters(); } } @@ -648,6 +777,48 @@ impl cellule_ltx::HostResourceAdmission for LedgerHostResourceAdmission { mod tests { use super::*; + #[test] + fn stage_transfer_reconciles_without_admitting_more_work() { + let limit = ResourceCost::zero().with_retained_bytes(128); + let ledger = ResourceLedger::new(limit); + let mut request = ledger + .try_reserve_in(RetentionStage::Request, limit) + .unwrap(); + let mut capture = request.split_retained(96).unwrap(); + capture.set_stage(RetentionStage::Capture).unwrap(); + let snapshot = ledger.snapshot().unwrap(); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Request as usize], + 32 + ); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Capture as usize], + 96 + ); + assert_eq!( + snapshot.retained_by_stage.iter().sum::(), + snapshot.used.retained_bytes() + ); + assert!( + ledger + .try_reserve(ResourceCost::zero().with_retained_bytes(1)) + .is_err() + ); + capture.shrink_retained(16).unwrap(); + capture.set_stage(RetentionStage::Selection).unwrap(); + drop(request); + let snapshot = ledger.snapshot().unwrap(); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Selection as usize], + 16 + ); + assert_eq!(snapshot.retained_by_stage.iter().sum::(), 16); + drop(capture); + let snapshot = ledger.snapshot().unwrap(); + assert_eq!(snapshot.used, ResourceCost::zero()); + assert_eq!(snapshot.retained_by_stage, [0; 7]); + } + #[test] fn retained_transfer_preserves_usage_until_each_owner_releases() { let cost = ResourceCost::active_cell().with_retained_bytes(128); diff --git a/crates/cellule-runtime/src/fleet/telemetry.rs b/crates/cellule-runtime/src/fleet/telemetry.rs index 6e965db4..cab1f3a6 100644 --- a/crates/cellule-runtime/src/fleet/telemetry.rs +++ b/crates/cellule-runtime/src/fleet/telemetry.rs @@ -183,6 +183,172 @@ pub struct SharedPublicationTiming { pub succeeded: bool, } +/// A bounded node-publication or materialization phase. Durations are +/// nested where documented; overlapping phases must not be summed as latency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NodePublicationPhase { + /// catalog wait observation. + CatalogWait, + /// preparation observation. + Preparation, + /// upload observation. + Upload, + /// verification observation. + Verification, + /// selection observation. + Selection, + /// Authority CAS inside a selection round (nested duration). + AuthorityCas, + /// receipt wait observation. + ReceiptWait, + /// materialization observation. + Materialization, + /// checkpoint observation. + Checkpoint, + /// Queueing and completion of an original materialized-root notification. + CheckpointWait, + /// worker binding observation. + WorkerBinding, +} +impl NodePublicationPhase { + /// Finite diagnostic values in index order. + pub const ALL: [Self; 11] = [ + Self::CatalogWait, + Self::Preparation, + Self::Upload, + Self::Verification, + Self::Selection, + Self::AuthorityCas, + Self::ReceiptWait, + Self::Materialization, + Self::Checkpoint, + Self::CheckpointWait, + Self::WorkerBinding, + ]; + /// Diagnostic label with no request or Cell identity. + pub const fn label(self) -> &'static str { + match self { + Self::CatalogWait => "catalog_wait", + Self::Preparation => "preparation", + Self::Upload => "upload", + Self::Verification => "verification", + Self::Selection => "selection", + Self::AuthorityCas => "authority_cas", + Self::ReceiptWait => "receipt_wait", + Self::Materialization => "materialization", + Self::Checkpoint => "checkpoint", + Self::CheckpointWait => "checkpoint_wait", + Self::WorkerBinding => "worker_binding", + } + } +} +/// Why the original actor refused work before dispatch. +/// A command refusal may still enter the separately bounded retry-resolution lane. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdmissionRefusal { + /// closed observation. + Closed, + /// fenced observation. + Fenced, + /// draining observation. + Draining, + /// publication backlog observation. + PublicationBacklog, + /// mailbox requests observation. + MailboxRequests, + /// mailbox bytes observation. + MailboxBytes, + /// node bytes observation. + NodeBytes, + /// invalid size observation. + InvalidSize, + /// other observation. + Other, +} +impl AdmissionRefusal { + /// Finite diagnostic values in index order. + pub const ALL: [Self; 9] = [ + Self::Closed, + Self::Fenced, + Self::Draining, + Self::PublicationBacklog, + Self::MailboxRequests, + Self::MailboxBytes, + Self::NodeBytes, + Self::InvalidSize, + Self::Other, + ]; + /// Diagnostic label with no request or Cell identity. + pub const fn label(self) -> &'static str { + match self { + Self::Closed => "closed", + Self::Fenced => "fenced", + Self::Draining => "draining", + Self::PublicationBacklog => "publication_backlog", + Self::MailboxRequests => "mailbox_requests", + Self::MailboxBytes => "mailbox_bytes", + Self::NodeBytes => "node_bytes", + Self::InvalidSize => "invalid_size", + Self::Other => "other", + } + } +} + +impl AdmissionRefusal { + pub(crate) fn from_error(error: &crate::Error) -> Self { + use crate::Error; + match error { + Error::Shared(source) => Self::from_error(source), + Error::RuntimeClosed => Self::Closed, + Error::Fenced => Self::Fenced, + Error::CellDraining => Self::Draining, + Error::Capacity("publication backlog") => Self::PublicationBacklog, + Error::Capacity("Cell mailbox requests") => Self::MailboxRequests, + Error::Capacity("Cell mailbox bytes") => Self::MailboxBytes, + Error::Capacity("node retained bytes") => Self::NodeBytes, + Error::Capacity("operation or result bytes" | "mailbox bytes") => Self::InvalidSize, + _ => Self::Other, + } + } +} + +/// Terminal phase outcome; cancellation records elapsed work without implying success. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PublicationPhaseOutcome { + /// The original phase completed successfully. + Succeeded, + /// The original phase returned an error. + Failed, + /// Its future exited without a terminal result. + Cancelled, +} + +/// A frozen native cohort; bytes exclude the catalog/framing overhead. +#[derive(Clone, Copy, Debug)] +pub struct NodePublicationCohort { + /// Complete captured assignments. + pub captures: usize, + /// Native frames in those assignments. + pub frames: usize, + /// Encoded native bytes retained by the captures. + pub bytes: usize, + /// Original materialized-root notifications included in this cohort. + pub checkpoints: usize, +} + +/// Why a ready predecessor did not overlap a successor. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PublicationPipelineFallback { + /// Materialized checkpoints require serial catalog preparation. + Checkpoints, + /// The original ledger could not pre-admit successor buffers and receipts. + Capacity, + /// No successor arrived before predecessor selection completed. + NoSuccessor, + /// Overlap would consume the existing foreground publication headroom. + Headroom, +} + /// Native follower timing for one dispatched append batch. /// One follower append attempt, measured on the blocking worker. /// @@ -492,6 +658,24 @@ pub trait CellTelemetry: Send + Sync { /// record the objects they did upload. fn publication_cost(&self, _objects: u64, _bytes: u64) {} + /// Begins one original phase; pair with its terminal observation to expose + /// in-flight work. Sinks must remain finite and nonblocking. + fn node_publication_started(&self, _phase: NodePublicationPhase) {} + /// Completes one phase, including failed and cancelled work. + fn node_publication_phase( + &self, + _phase: NodePublicationPhase, + _elapsed: Duration, + _outcome: PublicationPhaseOutcome, + ) { + } + /// Observes a frozen managed bundle's density. + fn node_publication_cohort(&self, _cohort: NodePublicationCohort) {} + /// Observes an ordered pipeline's serial fallback reason. + fn node_publication_fallback(&self, _reason: PublicationPipelineFallback) {} + /// Records the specific pre-dispatch refusal; it grants no outcome proof. + fn admission_refused(&self, _reason: AdmissionRefusal) {} + /// Records shared cohort work without Cell identity labels. fn shared_publication(&self, _timing: SharedPublicationTiming) {} @@ -699,6 +883,37 @@ impl CellTelemetryHandle { } } + pub(crate) async fn publication_phase( + &self, + phase: NodePublicationPhase, + work: impl std::future::Future>, + ) -> crate::Result { + let mut observation = PublicationObservation::new(self.clone(), phase); + let result = work.await; + observation.outcome = if result.is_ok() { + PublicationPhaseOutcome::Succeeded + } else { + PublicationPhaseOutcome::Failed + }; + result + } + + pub(crate) fn node_publication_cohort(&self, cohort: NodePublicationCohort) { + if let Some(sink) = self.inner.get() { + sink.node_publication_cohort(cohort); + } + } + pub(crate) fn node_publication_fallback(&self, reason: PublicationPipelineFallback) { + if let Some(sink) = self.inner.get() { + sink.node_publication_fallback(reason); + } + } + pub(crate) fn admission_refused(&self, reason: AdmissionRefusal) { + if let Some(sink) = self.inner.get() { + sink.admission_refused(reason); + } + } + pub(crate) fn shared_publication(&self, timing: SharedPublicationTiming) { if let Some(telemetry) = self.inner.get() { telemetry.shared_publication(timing); @@ -917,3 +1132,88 @@ mod tests { ); } } + +// This guard follows the polled original future, including cancellation. It +// owns no authority or work reservation and never changes the source error. +struct PublicationObservation { + sink: Option>, + phase: NodePublicationPhase, + started: std::time::Instant, + outcome: PublicationPhaseOutcome, +} +impl PublicationObservation { + fn new(telemetry: CellTelemetryHandle, phase: NodePublicationPhase) -> Self { + let sink = telemetry.inner.get().cloned(); + if let Some(sink) = &sink { + sink.node_publication_started(phase); + } + Self { + sink, + phase, + started: std::time::Instant::now(), + outcome: PublicationPhaseOutcome::Cancelled, + } + } +} +impl Drop for PublicationObservation { + fn drop(&mut self) { + if let Some(sink) = &self.sink { + sink.node_publication_phase(self.phase, self.started.elapsed(), self.outcome); + } + } +} + +#[cfg(test)] +mod publication_tests { + use super::*; + use std::sync::Mutex; + + #[derive(Default)] + struct Sink { + events: Mutex>, + } + impl CellTelemetry for Sink { + fn node_publication_phase( + &self, + phase: NodePublicationPhase, + _: Duration, + outcome: PublicationPhaseOutcome, + ) { + self.events.lock().unwrap().push((phase, outcome)); + } + } + + #[tokio::test] + async fn original_phase_reports_failure_and_cancellation_without_replacing_source() { + let sink = Arc::new(Sink::default()); + let handle = CellTelemetryHandle::from_sink(sink.clone()); + let result: crate::Result<()> = handle + .publication_phase(NodePublicationPhase::Upload, async { + Err(crate::Error::Capacity("original source")) + }) + .await; + assert!(matches!( + result, + Err(crate::Error::Capacity("original source")) + )); + let mut pending = Box::pin(handle.publication_phase( + NodePublicationPhase::ReceiptWait, + std::future::pending::>(), + )); + assert!(futures_util::poll!(pending.as_mut()).is_pending()); + drop(pending); + assert_eq!( + *sink.events.lock().unwrap(), + vec![ + ( + NodePublicationPhase::Upload, + PublicationPhaseOutcome::Failed + ), + ( + NodePublicationPhase::ReceiptWait, + PublicationPhaseOutcome::Cancelled + ), + ] + ); + } +} diff --git a/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs b/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs index 090e44dc..a2b9a86f 100644 --- a/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs @@ -38,8 +38,10 @@ const BODY_BYTES: usize = PREPARATION_BYTES impl BundlePreparation { pub(crate) fn reserve(resources: &ResourceLedger) -> Result { - let reservation = - resources.try_reserve(ResourceCost::zero().with_retained_bytes(PREPARATION_BYTES))?; + let reservation = resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Publisher, + ResourceCost::zero().with_retained_bytes(PREPARATION_BYTES), + )?; Ok(Self { scope: None, entries: std::iter::repeat_with(|| None) diff --git a/crates/cellule-runtime/src/node/bundle/selection.rs b/crates/cellule-runtime/src/node/bundle/selection.rs index 465b10be..e09640ff 100644 --- a/crates/cellule-runtime/src/node/bundle/selection.rs +++ b/crates/cellule-runtime/src/node/bundle/selection.rs @@ -381,48 +381,61 @@ impl NodeDirectory { if prepared.assignments.is_empty() { return Err(Error::Node("native bundle has no complete assignments")); } - let origin = origin::OriginBundle::load(&self.layout, prepared).await?; - let bindings = origin.selected_bindings(prepared)?; - let mut fresh = Vec::with_capacity(bindings.len()); - for binding in &bindings { - if !extension::verify( - &self.layout, - prepared, - lease, - binding, - prefixes, - limits, - &origin, - )? { - fresh.push(binding); - } - } - verification::verify_cohort( - &self.layout, - prepared.catalog.session, - prepared.catalog.epoch, - &fresh, - limits, - &origin, - ) - .await?; - let mut proofs = Vec::new(); - for binding in bindings { - let pin = binding - .control - .bundle_binding - .ok_or(Error::Node("selected binding lacks pin"))?; - proofs.push(BundleCoverageProof { - pin, - binding, - head: prepared.head, - session: prepared.catalog.session, - live: None, - }); - } + let mut proofs = self + .telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::Verification, + async { + let origin = origin::OriginBundle::load(&self.layout, prepared).await?; + let bindings = origin.selected_bindings(prepared)?; + let mut fresh = Vec::with_capacity(bindings.len()); + for binding in &bindings { + if !extension::verify( + &self.layout, + prepared, + lease, + binding, + prefixes, + limits, + &origin, + )? { + fresh.push(binding); + } + } + verification::verify_cohort( + &self.layout, + prepared.catalog.session, + prepared.catalog.epoch, + &fresh, + limits, + &origin, + ) + .await?; + let mut proofs = Vec::new(); + for binding in bindings { + let pin = binding + .control + .bundle_binding + .ok_or(Error::Node("selected binding lacks pin"))?; + proofs.push(BundleCoverageProof { + pin, + binding, + head: prepared.head, + session: prepared.catalog.session, + live: None, + }); + } + Ok(proofs) + }, + ) + .await?; lease.check()?; let selected = self - .select_native_catalog(observed, prepared, now_ms) + .telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::AuthorityCas, + self.select_native_catalog(observed, prepared, now_ms), + ) .await?; // Expiry during verification must not revive the original writer. lease.check()?; diff --git a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs index c1646a7f..9f15dbc4 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs @@ -324,6 +324,43 @@ async fn tight_original_budget_uses_one_cohort_and_returns_all_credit() { case.finish([first, second]).await; } +#[tokio::test] +async fn optional_overlap_preserves_foreground_headroom_with_live_selected_debt() { + use crate::fleet::resource::{ResourceCost, RetentionStage}; + let mut case = Case::new(64 << 20, false, false).await; + // Model other Cells' original selected receipts while this cohort's PUT + // is stalled. The optional successor must not make fresh SQL shed itself. + let selected_debt = case + .pool + .resource_ledger() + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(8 << 20), + ) + .unwrap(); + let first = case.first().await; + let second = case.second().await; + let overlapped = tokio::time::timeout( + Duration::from_millis(250), + case.store.second_done.notified(), + ) + .await + .is_ok(); + assert!( + !overlapped, + "optional overlap consumed foreground progress headroom" + ); + let snapshot = case.pool.resource_ledger().snapshot().unwrap(); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Pipeline as usize], + 0 + ); + assert!(snapshot.used.retained_bytes() < snapshot.limit.retained_bytes() * 3 / 4); + case.store.resume.notify_one(); + drop(selected_debt); + case.finish([first, second]).await; +} + #[tokio::test] async fn fenced_join_waits_for_both_accepted_uploads_after_caller_cancellation() { failed_join(false).await; diff --git a/crates/cellule-runtime/src/node/directory/mod.rs b/crates/cellule-runtime/src/node/directory/mod.rs index f13e8ede..c277627a 100644 --- a/crates/cellule-runtime/src/node/directory/mod.rs +++ b/crates/cellule-runtime/src/node/directory/mod.rs @@ -40,6 +40,7 @@ pub struct NodeDirectory { // maintenance continue to read their canonical records directly. reader_membership: Arc>>, inventory_scope: [u8; 16], + pub(super) telemetry: crate::fleet::telemetry::CellTelemetryHandle, } /// Request verifier bound to one mTLS-authenticated enrollment observation. @@ -106,9 +107,21 @@ impl NodeDirectory { recovery_scan: Arc::new(RwLock::new(None)), reader_membership: Arc::new(RwLock::new(None)), inventory_scope: rand::random(), + telemetry: crate::fleet::telemetry::CellTelemetryHandle::default(), } } + /// Uses the application's original runtime sink for bundle verification and + /// selection diagnostics. Observations grant no availability or authority. + #[must_use] + pub fn with_telemetry( + mut self, + telemetry: crate::fleet::telemetry::CellTelemetryHandle, + ) -> Self { + self.telemetry = telemetry; + self + } + /// Returns the fleet this directory is scoped to. #[must_use] pub const fn fleet(&self) -> Digest { diff --git a/crates/cellule-runtime/src/node/durability/mod.rs b/crates/cellule-runtime/src/node/durability/mod.rs index 6cde8051..f878b52b 100644 --- a/crates/cellule-runtime/src/node/durability/mod.rs +++ b/crates/cellule-runtime/src/node/durability/mod.rs @@ -367,7 +367,8 @@ impl NodeDurability { .ok_or(Error::PendingPublication)?; // Reserve before cloning locator metadata; the original cohort remains // separately charged until its other original consumers release it. - let memory = resources.try_reserve( + let memory = resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Selection, crate::fleet::resource::ResourceCost::zero() .with_retained_bytes(selected.proof.retained_metadata_bytes()?), )?; diff --git a/crates/cellule-runtime/src/node/durability/publication/mod.rs b/crates/cellule-runtime/src/node/durability/publication/mod.rs index fa21ed82..ff6b3c89 100644 --- a/crates/cellule-runtime/src/node/durability/publication/mod.rs +++ b/crates/cellule-runtime/src/node/durability/publication/mod.rs @@ -71,7 +71,8 @@ impl Publisher { .selection_resources .get() .ok_or(Error::PendingPublication)?; - resources.try_reserve( + resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Publisher, crate::fleet::resource::ResourceCost::zero() // The fifth buffer covers the fresh cohort origin read. After // matching the proposal, its allocation is released for 2 MiB @@ -221,6 +222,12 @@ async fn run( mut preparation: crate::node::bundle::BundlePreparation, lease: &NodeLeaseGuard, ) -> Result<()> { + let telemetry = durability + .upgrade() + .ok_or(Error::RuntimeClosed)? + .shipper + .telemetry() + .clone(); let mut prefixes = prefixes::Prefixes::new()?; let mut carry = None; let mut checkpoints_open = true; @@ -234,14 +241,14 @@ async fn run( } else if let Some(capture) = feed.try_recv() { Some(capture) } else if let Some(first) = first_checkpoint.take() { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; continue; } else { tokio::select! { capture = feed.recv() => capture, checkpoint = checkpoints.recv(), if checkpoints_open => { if let Some(first) = checkpoint { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } else { checkpoints_open = false; } continue; @@ -251,7 +258,7 @@ async fn run( }; let Some(first) = capture else { if let Some(first) = first_checkpoint.take() { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } break; }; @@ -264,6 +271,7 @@ async fn run( first_checkpoint, true, lease, + &telemetry, ) .await?; let original = durability.upgrade().ok_or(Error::RuntimeClosed)?; @@ -285,7 +293,13 @@ async fn run( }; let cohort = receipts::SelectedCaptures::new(&original, &captures, proofs)?; let memory = { - let admission = cohort.resources(&original)?.reserve(cohort.cost()); + let admission = telemetry.publication_phase( + crate::fleet::telemetry::NodePublicationPhase::ReceiptWait, + cohort.resources(&original)?.reserve_in( + crate::fleet::resource::RetentionStage::Selection, + cohort.cost(), + ), + ); tokio::pin!(admission); loop { tokio::select! { @@ -300,7 +314,7 @@ async fn run( // Root tasks retain their admission until this callback // joins. Servicing it while receipt credit is exhausted // releases credit without repeating the durable CAS. - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } else { checkpoints_open = false; } } } @@ -315,7 +329,7 @@ async fn run( } checkpoints.close(); while let Some(first) = checkpoints.recv().await { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } Ok(()) } @@ -331,10 +345,14 @@ async fn checkpoint_cohort( authority: &Arc, receiver: &mut mpsc::Receiver, first: CheckpointRequest, + telemetry: &crate::fleet::telemetry::CellTelemetryHandle, ) -> Result<()> { let cohort = checkpoints::Cohort::gather(receiver, Some(first), MAX_CAPTURES)?; - let result = authority - .checkpoint(cohort.values()) + let result = telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::Checkpoint, + authority.checkpoint(cohort.values()), + ) .await .map_err(Arc::new); cohort.complete(result.clone()); @@ -359,6 +377,7 @@ async fn assemble( mut first_checkpoint: Option, include_checkpoints: bool, lease: &NodeLeaseGuard, + telemetry: &crate::fleet::telemetry::CellTelemetryHandle, ) -> Result { let mut captures = vec![first]; let mut frames = captures[0].frames().len(); @@ -371,7 +390,7 @@ async fn assemble( if frames == MAX_FRAMES && let Some(first) = first_checkpoint.take() { - checkpoint_cohort(authority, checkpoints, first).await?; + checkpoint_cohort(authority, checkpoints, first, telemetry).await?; } // Original root tasks hold credit until their callback joins. Reserve // the entire ready cohort before native assembly can occupy its rows; @@ -411,6 +430,12 @@ async fn assemble( ready_checkpoints.extend(checkpoints, MAX_FRAMES - frames)?; } + telemetry.node_publication_cohort(crate::fleet::telemetry::NodePublicationCohort { + captures: captures.len(), + frames, + bytes, + checkpoints: ready_checkpoints.notification_count(), + }); Ok(CaptureCohort { captures, ready_checkpoints, diff --git a/crates/cellule-runtime/src/node/durability/publication/pipeline.rs b/crates/cellule-runtime/src/node/durability/publication/pipeline.rs index 881a8257..91ce4766 100644 --- a/crates/cellule-runtime/src/node/durability/publication/pipeline.rs +++ b/crates/cellule-runtime/src/node/durability/publication/pipeline.rs @@ -1,6 +1,9 @@ //! One lazy successor: pre-admitted overlap, ordered proof, joined I/O. use super::*; use crate::fleet::resource::{ResourceCost, ResourceReservation}; +use crate::fleet::telemetry::{ + NodePublicationPhase as Phase, PublicationPipelineFallback as Fallback, +}; use crate::node::bundle::PreparedNodeBundle; use std::sync::OnceLock; @@ -34,7 +37,7 @@ fn reserve( .selection_resources .get() .ok_or(Error::PendingPublication)? - .try_reserve(ResourceCost::zero().with_retained_bytes(bytes))?; + .try_reserve_pipeline(ResourceCost::zero().with_retained_bytes(bytes))?; Ok(Memory { first, second, @@ -72,10 +75,14 @@ pub(super) async fn publish( progress: &watch::Sender, lease: &NodeLeaseGuard, ) -> Result> { + let telemetry = original.shipper.telemetry(); let first_capture_count = first.captures.len(); let may_overlap = first.ready_checkpoints.values().is_empty(); let failure = OnceLock::new(); - let round = match authority.begin_round().await { + let round = match telemetry + .publication_phase(Phase::CatalogWait, authority.begin_round()) + .await + { Ok(round) => round, Err(error) => { let error = Arc::new(error); @@ -83,12 +90,15 @@ pub(super) async fn publish( return Err(Error::Shared(error)); } }; - let staged = match round - .stage( - None, - &first.captures, - first.ready_checkpoints.values(), - Some(preparation), + let staged = match telemetry + .publication_phase( + Phase::Preparation, + round.stage( + None, + &first.captures, + first.ready_checkpoints.values(), + Some(preparation), + ), ) .await { @@ -105,13 +115,20 @@ pub(super) async fn publish( async { let result = async { let publication = async { - let uploaded = round.upload(Arc::clone(&staged)).await?; + let uploaded = telemetry + .publication_phase(Phase::Upload, round.upload(Arc::clone(&staged))) + .await?; let prior = prefixes.for_captures(&first.captures); let borrowed = prior .iter() .map(|selected| &selected.proof) .collect::>(); - round.select(&uploaded, &borrowed, lease).await + telemetry + .publication_phase( + Phase::Selection, + round.select(&uploaded, &borrowed, lease), + ) + .await } .await; let publication = recorded(publication, &failure, lease).map_err(Arc::new); @@ -147,6 +164,7 @@ pub(super) async fn publish( async { let result = async { if !may_overlap { + telemetry.node_publication_fallback(Fallback::Checkpoints); let _ = receipt.send(None); return Ok(None); } @@ -161,12 +179,20 @@ pub(super) async fn publish( } }; let Some(next) = next else { + telemetry.node_publication_fallback(Fallback::NoSuccessor); let _ = receipt.send(None); return Ok(None); }; let mut memory = match reserve(original, authority.as_ref(), first_capture_count) { Ok(memory) => memory, + Err(Error::Capacity("bundle pipeline headroom")) => { + telemetry.node_publication_fallback(Fallback::Headroom); + *carry = Some(next); + let _ = receipt.send(None); + return Ok(None); + } Err(Error::Capacity(_)) => { + telemetry.node_publication_fallback(Fallback::Capacity); *carry = Some(next); let _ = receipt.send(None); return Ok(None); @@ -189,19 +215,25 @@ pub(super) async fn publish( None, false, lease, + telemetry, ) .await?; let upload = async { - let next = round - .stage( - Some(&staged), - &second.captures, - second.ready_checkpoints.values(), - Some(preparation), + let next = telemetry + .publication_phase( + Phase::Preparation, + round.stage( + Some(&staged), + &second.captures, + second.ready_checkpoints.values(), + Some(preparation), + ), ) .await?; lease.check()?; - round.upload(next).await + telemetry + .publication_phase(Phase::Upload, round.upload(next)) + .await } .await; match recorded(upload, &failure, lease) { @@ -245,8 +277,11 @@ pub(super) async fn publish( .iter() .map(|selected| &selected.proof) .collect::>(); - let result = round - .select(&successor.uploaded, &borrowed, lease) + let result = telemetry + .publication_phase( + Phase::Selection, + round.select(&successor.uploaded, &borrowed, lease), + ) .await .map_err(Arc::new); drop(borrowed); diff --git a/crates/cellule-runtime/src/node/durability/receipts.rs b/crates/cellule-runtime/src/node/durability/receipts.rs index 09d9bc83..9218309c 100644 --- a/crates/cellule-runtime/src/node/durability/receipts.rs +++ b/crates/cellule-runtime/src/node/durability/receipts.rs @@ -84,6 +84,7 @@ impl<'a> SelectedCaptures<'a> { // Transfer a single cohort admission before waking any sibling. Waiting // for credit may have joined older checkpoints, so revalidate the // original lease/gate against these exact live proofs after the wait. + memory.set_stage(crate::fleet::resource::RetentionStage::Selection)?; let memories = self .bytes .into_iter() diff --git a/crates/cellule-runtime/src/node/log_shipper/mod.rs b/crates/cellule-runtime/src/node/log_shipper/mod.rs index eabafc02..348a6369 100644 --- a/crates/cellule-runtime/src/node/log_shipper/mod.rs +++ b/crates/cellule-runtime/src/node/log_shipper/mod.rs @@ -213,6 +213,10 @@ pub struct NodeLogShipper { } impl NodeLogShipper { + pub(crate) fn telemetry(&self) -> &crate::fleet::telemetry::CellTelemetryHandle { + &self.telemetry + } + /// Starts one shipper for the exact ensemble owned by `gate`. pub fn new( gate: DurabilityGate, diff --git a/crates/cellule-runtime/src/publication/mod.rs b/crates/cellule-runtime/src/publication/mod.rs index ff412d0b..d8a7611f 100644 --- a/crates/cellule-runtime/src/publication/mod.rs +++ b/crates/cellule-runtime/src/publication/mod.rs @@ -117,27 +117,38 @@ impl CellPublisher { proof: &crate::node::bundle::BundleCoverageProof, next_due_ms: Option, ) -> Result { - self.check_node_lease()?; - if self.observed.value().bundle_binding != Some(proof.binding()) { - return Err(Error::Fenced); - } - let base = self.observed.value().ltx_root().ok_or(Error::Fenced)?; - if base.commit_sequence == proof.commit_sequence() && base.position == proof.position() { - return Ok(base); - } - let overlay = proof - .recovery_overlay_from(self.authority.layout(), self.replica.limits(), base) - .await?; - self.check_node_lease()?; - let (replica, confirmation) = lineage::replica(self.replica.clone(), &self.authority); - let prepared = replica - .prepare_recovered_overlay(&overlay, self.observed.value().schema) + let telemetry = self.telemetry.clone(); + telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::Materialization, + async { + self.check_node_lease()?; + if self.observed.value().bundle_binding != Some(proof.binding()) { + return Err(Error::Fenced); + } + let base = self.observed.value().ltx_root().ok_or(Error::Fenced)?; + if base.commit_sequence == proof.commit_sequence() + && base.position == proof.position() + { + return Ok(base); + } + let overlay = proof + .recovery_overlay_from(self.authority.layout(), self.replica.limits(), base) + .await?; + self.check_node_lease()?; + let (replica, confirmation) = + lineage::replica(self.replica.clone(), &self.authority); + let prepared = replica + .prepare_recovered_overlay(&overlay, self.observed.value().schema) + .await + .map_err(lineage::error)?; + self.lineage_confirmed = *confirmation + .lock() + .map_err(|_| Error::Peer("root lineage confirmation lock poisoned"))?; + self.publish_prepared(&prepared, next_due_ms).await + }, + ) .await - .map_err(lineage::error)?; - self.lineage_confirmed = *confirmation - .lock() - .map_err(|_| Error::Peer("root lineage confirmation lock poisoned"))?; - self.publish_prepared(&prepared, next_due_ms).await } pub(crate) fn with_shared_publication( diff --git a/docs/managed-bundle-pipeline.md b/docs/managed-bundle-pipeline.md index 7650d3be..9ee47207 100644 --- a/docs/managed-bundle-pipeline.md +++ b/docs/managed-bundle-pipeline.md @@ -23,7 +23,7 @@ The manual `select` method uses these same three phases. | Event | Managed behavior | | --- | --- | | First cohort starts | Keep the existing 20-MiB working reservation and 2-MiB shard cache | -| Another capture arrives during publication | Try to admit another 20 MiB plus conservative receipt bounds for both cohorts on the original ledger | +| Another capture arrives during publication | Try to admit another 20 MiB plus conservative receipt bounds for both cohorts on the original ledger, preserving publication progress and foreground burst headroom | | Extra credit unavailable | Keep that capture for the next serial round; do not wait while holding catalog ordering | | Successor PUT finishes first | Retain uploaded bytes; grant no coverage or response credit | | First selection succeeds | Confirm its original receipts and release its native capture credit independently of the second PUT | @@ -93,6 +93,51 @@ raw failed/passed logs and isolated snapshots for this integration are retained outside Git under `cellule-ios-parity-20261010/managed-bundle-pipeline`. +## Publication diagnostics + +The runtime's `CellTelemetry` sink observes managed bundle preparation, upload, +selection, receipt admission, materialization, checkpoint callbacks and worker +binding. Install the same `CellTelemetryHandle` on `NodeDirectory` with +`with_telemetry` to separate fresh dependency verification and the authority +CAS inside selection. Phase callbacks include success, source failure and +cancellation; the sink captured at phase entry receives both entry and exit. +Callbacks must remain bounded and nonblocking. + +The SQL example exports these observations under `node_publication`, with raw +cumulative histograms under `histograms.node_publication_*`. Selection includes +verification and CAS; materialization includes root reconstruction and its Cell +CAS. These nested durations and overlapping pipeline work cannot be summed as +request latency. `checkpoint_wait` includes the original queued notification; +`checkpoint` measures a standalone authority checkpoint. Native cohort counters +report captures, frames, bytes and included checkpoint notifications. Serial +fallback counters distinguish checkpoints, capacity, foreground headroom and an +absent successor. Optional overlap admission atomically stays below the same +75% retained-byte threshold used by ordinary write admission, reserving another +eighth of the budget below that threshold for foreground bursts. Parallel root +materializers use the same working threshold. When none is alive, one required +cleanup may use the original full budget so existing debt cannot starve its own +reduction. These rules do not increase the total budget or reduce receipt or +materialization bounds. + +`CellRuntime::retained_work` returns one locked ledger snapshot whose +`retained_by_stage` counts sum to its retained total. Ownership classes follow +the original reservation through split, reclassification, shrink and drop; +they do not alter admission or represent estimated heap use. Unclassified work +remains explicit as `other`. The separate native-log byte window still has its +own accounting. The example exports this snapshot under `retained_work`. + +Actor admission refusals are counted separately from application HTTP failures: +a backlog-refused mutation may still resolve an existing durable outcome. +The example's finite HTTP failure classes preserve the source chain and public +error envelope, including post-commit capacity failures. Neither counter grants +an outcome or recovery proof. Use raw request journals to reconcile offered, +successful, failed, dropped and late requests. + +The 45-second root age trigger is a materialization policy, not a guarantee +that bucket selection is delayed by 45 seconds. Compare issued and selected +node-log frontiers, retained ownership, root debt age and phase service rates +before attributing an old root to a slow bundle upload. + ## Verification All 13 contributor routes pass on an isolated snapshot, including 2,095 From f52aa55e5889c8967c23c94f0511e6ead6d4ffbd Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 11:30:09 -0700 Subject: [PATCH 03/31] perf(runtime): overlap bounded historical frame reads --- .../src/cell/actor/materialization/mod.rs | 13 + crates/cellule-runtime/src/fleet/resource.rs | 6 + .../cellule-runtime/src/node/bundle/proof.rs | 73 +++-- .../bundle/tests/materialization_reads.rs | 249 ++++++++++++++++++ .../src/node/bundle/tests/mod.rs | 1 + docs/managed-bundle-pipeline.md | 10 + 6 files changed, 337 insertions(+), 15 deletions(-) create mode 100644 crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs diff --git a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs index 5229e814..b7394dc7 100644 --- a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs +++ b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs @@ -217,6 +217,19 @@ pub(super) fn dispatch( if available == 0 { return; } + // Every root needs at least this original I/O body and prefix metadata. + // Avoid sorting the entire dirty fleet on each SQL event when no parallel + // job can fit. Exact admission below still arbitrates concurrent releases. + let minimum = crate::node::bundle::MAX_BUNDLE_BYTES as usize + + crate::node::bundle::MaterializedBundlePrefix::maximum_retained_bytes(); + if running != 0 + && pool + .resource_ledger() + .materialization_headroom() + .is_ok_and(|bytes| bytes <= minimum) + { + return; + } let now = std::time::Instant::now(); let mut ready: Vec<_> = cells .iter() diff --git a/crates/cellule-runtime/src/fleet/resource.rs b/crates/cellule-runtime/src/fleet/resource.rs index aa407ffa..e71cdc7e 100644 --- a/crates/cellule-runtime/src/fleet/resource.rs +++ b/crates/cellule-runtime/src/fleet/resource.rs @@ -438,6 +438,12 @@ impl ResourceLedger { self.try_reserve_inner(RetentionStage::Materialization, cost, !progress_needed) } + pub(crate) fn materialization_headroom(&self) -> Result { + let state = self.snapshot()?; + Ok(publication_working_threshold(state.limit.retained_bytes()) + .saturating_sub(state.used.retained_bytes())) + } + fn try_reserve_inner( &self, stage: RetentionStage, diff --git a/crates/cellule-runtime/src/node/bundle/proof.rs b/crates/cellule-runtime/src/node/bundle/proof.rs index 4f75fcb6..a45dec4a 100644 --- a/crates/cellule-runtime/src/node/bundle/proof.rs +++ b/crates/cellule-runtime/src/node/bundle/proof.rs @@ -3,6 +3,7 @@ use super::*; use crate::control::authority::{CellAuthority, VersionedControl}; use crate::node::NodeDirectory; use crate::node::directory::NodeRecord; +use futures_util::{StreamExt, stream::FuturesOrdered}; impl NodeDirectory { /// Reopens one authority-pinned selected Cell suffix from canonical origin. @@ -150,23 +151,65 @@ async fn verify_binding_into( ) -> Result<()> { verify_base(layout, binding, limits).await?; let mut chain = BindingChain::new(binding)?; - for locator in &binding.locators { - let object = locator - .object - .ok_or(Error::Node("bundle locator is unresolved"))?; - let end = locator - .offset - .checked_add(locator.bytes) - .ok_or(Error::Node("bundle locator overflow"))?; - let bytes = - origin::read_range(layout, session, epoch, object, locator.offset..end, origin).await?; - let frame = checked_frame(session, epoch, binding, locator, bytes, limits)?; - chain.accept(FrameStep::from_frame(&frame))?; - if let Some(frames) = &mut frames { - frames.push(frame); + let mut pending = binding.locators.iter(); + let mut reads = FuturesOrdered::new(); + for locator in pending.by_ref().take(verification::READ_CONCURRENCY) { + reads.push_back(read_frame( + layout, session, epoch, binding, locator, limits, origin, + )); + } + // The original suffix admission already covers all retained frame bodies. + // Overlap bounded GETs, consume facts in exact locator order, and join the + // accepted window after failure without dispatching any later reads. + let mut failure = None; + while let Some(result) = reads.next().await { + if failure.is_some() { + continue; + } + let result = result.and_then(|frame| { + chain.accept(FrameStep::from_frame(&frame))?; + if let Some(frames) = &mut frames { + frames.push(frame); + } + Ok(()) + }); + if let Err(error) = result { + failure = Some(error); + } else if let Some(locator) = pending.next() { + reads.push_back(read_frame( + layout, session, epoch, binding, locator, limits, origin, + )); } } - chain.finish(binding) + match failure { + Some(error) => Err(error), + None => chain.finish(binding), + } +} + +#[allow( + clippy::too_many_arguments, + reason = "one exact authenticated locator and origin scope" +)] +async fn read_frame( + layout: &cellule_ltx::CellStorageLayout, + session: SessionId, + epoch: u64, + binding: &Binding, + locator: &Locator, + limits: cellule_ltx::Limits, + origin: Option<&origin::OriginBundle>, +) -> Result { + let object = locator + .object + .ok_or(Error::Node("bundle locator is unresolved"))?; + let end = locator + .offset + .checked_add(locator.bytes) + .ok_or(Error::Node("bundle locator overflow"))?; + let bytes = + origin::read_range(layout, session, epoch, object, locator.offset..end, origin).await?; + checked_frame(session, epoch, binding, locator, bytes, limits) } /// Checked frame facts retained only within one verification operation. They diff --git a/crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs b/crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs new file mode 100644 index 00000000..27d0b430 --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs @@ -0,0 +1,249 @@ +use super::*; +use futures_util::stream::BoxStream; +use object_store::ObjectStore; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Notify; + +#[derive(Debug, Default)] +struct HeldFrameStore { + inner: InMemory, + armed: AtomicBool, + reads: AtomicUsize, + fail_second: AtomicBool, + failed_path: std::sync::Mutex>, + paths: std::sync::Mutex>, + first: Notify, + second: Notify, + third: Notify, + resume: Notify, + resume_third: Notify, +} +impl std::fmt::Display for HeldFrameStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HeldFrameStore") + } +} +#[async_trait::async_trait] +impl ObjectStore for HeldFrameStore { + async fn put_opts( + &self, + path: &Path, + payload: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + async fn put_multipart_opts( + &self, + path: &Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts( + &self, + path: &Path, + options: object_store::GetOptions, + ) -> object_store::Result { + if self.armed.load(Ordering::Acquire) + && options.range.is_some() + && path.as_ref().ends_with(".nodebundle") + { + self.paths.lock().unwrap().insert(path.to_string()); + if self.fail_second.load(Ordering::Acquire) + && self.failed_path.lock().unwrap().as_deref() == Some(path.as_ref()) + { + return Err(object_store::Error::Generic { + store: "HeldFrameStore", + source: std::io::Error::other("deliberate historical read failure").into(), + }); + } + match self.reads.fetch_add(1, Ordering::AcqRel) { + 0 => { + self.first.notify_one(); + self.resume.notified().await; + } + 1 => { + self.second.notify_one(); + if self.fail_second.load(Ordering::Acquire) { + *self.failed_path.lock().unwrap() = Some(path.to_string()); + return Err(object_store::Error::Generic { + store: "HeldFrameStore", + source: std::io::Error::other("deliberate historical read failure") + .into(), + }); + } + } + 2 if self.fail_second.load(Ordering::Acquire) => { + self.third.notify_one(); + self.resume_third.notified().await; + } + _ => {} + } + } + self.inner.get_opts(path, options).await + } + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +#[tokio::test] +async fn failed_historical_window_joins_accepted_reads_and_publishes_no_root() { + let store = Arc::new(HeldFrameStore::default()); + let mut f = Fixture::with_store(store.clone()).await; + let mut cell = f.cell(4).await; + let mut proof = None; + for commit in 2..=11 { + let (_, frames, assigned) = f.append(&mut cell, commit); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (node, mut proofs) = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + f.node = node; + proof = proofs.pop(); + } + let before = cell.control.value().encode().unwrap(); + let proof = proof.unwrap(); + let mut publisher = f.publisher(&cell); + store.fail_second.store(true, Ordering::Release); + store.armed.store(true, Ordering::Release); + let materialized = publisher.materialize_bundle(&proof); + tokio::pin!(materialized); + tokio::select! { + _ = store.third.notified() => {}, + _ = &mut materialized => panic!("accepted historical reads must remain joined"), + _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => panic!("historical window was not dispatched"), + } + store.resume.notify_one(); + tokio::select! { + _ = &mut materialized => panic!("failure escaped while an accepted read remained held"), + _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {}, + } + // Transport retries of the same missing frame are not another dispatched + // locator. The accepted window and its one earlier refill remain bounded. + let dispatched = store.paths.lock().unwrap().len(); + assert!(dispatched <= verification::READ_CONCURRENCY + 1); + store.resume_third.notify_one(); + let error = materialized.await.unwrap_err(); + assert_eq!(store.paths.lock().unwrap().len(), dispatched); + let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut found = false; + while let Some(error) = source { + found |= error + .to_string() + .contains("deliberate historical read failure"); + source = error.source(); + } + assert!(found, "original read failure was replaced: {error:?}"); + assert_eq!( + cell.authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap() + .value() + .encode() + .unwrap(), + before + ); +} + +#[tokio::test] +async fn materialization_overlaps_historical_reads_before_exact_cold_restore() { + let store = Arc::new(HeldFrameStore::default()); + let mut f = Fixture::with_store(store.clone()).await; + let mut cell = f.cell(4).await; + let mut proof = None; + for commit in [2, 3] { + let (_, frames, assigned) = f.append(&mut cell, commit); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (node, mut proofs) = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + f.node = node; + proof = proofs.pop(); + } + let proof = proof.unwrap(); + assert_eq!(proof.locator_count(), 2); + let mut publisher = f.publisher(&cell); + store.armed.store(true, Ordering::Release); + let materialized = publisher.materialize_bundle(&proof); + tokio::pin!(materialized); + tokio::select! { + _ = store.first.notified() => {}, + _ = &mut materialized => panic!("first historical read must remain held"), + } + let overlapped = tokio::select! { + _ = store.second.notified() => true, + _ = tokio::time::sleep(std::time::Duration::from_millis(250)) => false, + _ = &mut materialized => panic!("root cannot publish before its first frame"), + }; + assert!( + overlapped, + "historical reads remained serial under the original suffix admission" + ); + store.resume.notify_one(); + let root = materialized.await.unwrap(); + assert_eq!(root.commit_sequence, 3); + let restored = f.scratch.path().join("overlapped-restore.sqlite"); + cell.replica + .open_root(&root) + .await + .unwrap() + .restore(&restored) + .await + .unwrap(); + let db = rusqlite::Connection::open(restored).unwrap(); + let mut stmt = db + .prepare("SELECT request, result FROM outcomes ORDER BY request") + .unwrap(); + let rows = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + rows, + vec![ + ("request-2".into(), "result-2".into()), + ("request-3".into(), "result-3".into()), + ("seed".into(), "original".into()) + ] + ); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/mod.rs index c931b40d..1e469f1b 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/mod.rs @@ -23,6 +23,7 @@ mod faults; mod index; mod lifecycle; mod managed; +mod materialization_reads; mod pipeline; mod publication_hooks; mod ranges; diff --git a/docs/managed-bundle-pipeline.md b/docs/managed-bundle-pipeline.md index 13f37c99..61b6c9d8 100644 --- a/docs/managed-bundle-pipeline.md +++ b/docs/managed-bundle-pipeline.md @@ -43,6 +43,16 @@ preserves heartbeat and coverage changes observed after staging. ## Checkpoints and joined lifetime +Materialization overlaps at most eight original authenticated frame-range reads +within its existing `6 × suffix bytes + 4 MiB` working reservation. Verification +consumes frame facts in locator order. A failure stops new dispatch, joins the +accepted window, preserves the first source error and grants no root credit. +Base-origin verification remains fresh and precedes those reads. + +When a materializer is already alive and even the minimum working reservation +cannot fit, dispatch skips the repeated dirty-fleet walk and sort. This is only +a scheduling hint; the original atomic admission still decides each job. + A first cohort containing root checkpoint notifications uses serial publication. The lookahead cohort contains native captures only. Ready root notifications join the next first cohort or the existing idle checkpoint path. This preserves From 0ff943054da5f28fedfe033184dffc87c64911e6 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 12:02:30 -0700 Subject: [PATCH 04/31] fix(runtime): bound publication observation future frames --- crates/cellule-runtime/src/fleet/telemetry.rs | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/crates/cellule-runtime/src/fleet/telemetry.rs b/crates/cellule-runtime/src/fleet/telemetry.rs index cab1f3a6..ae7a2342 100644 --- a/crates/cellule-runtime/src/fleet/telemetry.rs +++ b/crates/cellule-runtime/src/fleet/telemetry.rs @@ -883,19 +883,26 @@ impl CellTelemetryHandle { } } - pub(crate) async fn publication_phase( + pub(crate) fn publication_phase( &self, phase: NodePublicationPhase, work: impl std::future::Future>, - ) -> crate::Result { - let mut observation = PublicationObservation::new(self.clone(), phase); - let result = work.await; - observation.outcome = if result.is_ok() { - PublicationPhaseOutcome::Succeeded - } else { - PublicationPhaseOutcome::Failed - }; - result + ) -> impl std::future::Future> { + // Box at construction, before the observation future can be embedded + // in another async frame. Publication work contains large bounded + // envelopes; inlining it through telemetry can exhaust a worker stack. + let work = Box::pin(work); + let handle = self.clone(); + async move { + let mut observation = PublicationObservation::new(handle, phase); + let result = work.await; + observation.outcome = if result.is_ok() { + PublicationPhaseOutcome::Succeeded + } else { + PublicationPhaseOutcome::Failed + }; + result + } } pub(crate) fn node_publication_cohort(&self, cohort: NodePublicationCohort) { @@ -1183,6 +1190,23 @@ mod publication_tests { } } + #[tokio::test] + async fn phase_observation_keeps_large_work_out_of_its_caller_frame() { + let handle = CellTelemetryHandle::default(); + let bytes = [0x5a_u8; 128 << 10]; + let work = async move { + tokio::task::yield_now().await; + Ok(bytes.iter().map(|byte| u64::from(*byte)).sum::()) + }; + let observed = handle.publication_phase(NodePublicationPhase::Materialization, work); + assert!( + std::mem::size_of_val(&observed) <= 1024, + "phase observation embeds {} bytes of accepted work in its caller", + std::mem::size_of_val(&observed), + ); + assert_eq!(observed.await.unwrap(), 0x5a * (128 << 10)); + } + #[tokio::test] async fn original_phase_reports_failure_and_cancellation_without_replacing_source() { let sink = Arc::new(Sink::default()); From 2178ba760b9937f93652153a7d720b7bef372672 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 12:21:01 -0700 Subject: [PATCH 05/31] perf(runtime): release cleanup admission when root I/O joins --- crates/cellule-axum/examples/sql.rs | 1 + .../src/cell/actor/materialization/mod.rs | 16 +---- .../src/cell/actor/materialization/tests.rs | 55 ++++++++++++++-- crates/cellule-runtime/src/fleet/resource.rs | 62 +++++++++++++++---- 4 files changed, 105 insertions(+), 29 deletions(-) diff --git a/crates/cellule-axum/examples/sql.rs b/crates/cellule-axum/examples/sql.rs index 0e757387..c8084504 100644 --- a/crates/cellule-axum/examples/sql.rs +++ b/crates/cellule-axum/examples/sql.rs @@ -521,6 +521,7 @@ async fn main() -> ExampleResult<()> { Ok(snapshot) => serde_json::json!({ "total": snapshot.used.retained_bytes(), "capacity": snapshot.limit.retained_bytes(), + "working_materializations": snapshot.working_materializations, "stages": cellule_runtime::fleet::resource::RetentionStage::ALL.iter() .map(|stage| (stage.label().to_owned(), snapshot.retained_by_stage[*stage as usize])) .collect::>(), diff --git a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs index b7394dc7..14a5adf2 100644 --- a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs +++ b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs @@ -11,12 +11,8 @@ const MAX_MATERIALIZERS: usize = 8; fn reserve_materialization( resources: &crate::fleet::resource::ResourceLedger, bytes: usize, - progress_needed: bool, ) -> crate::Result { - resources.try_reserve_materialization( - ResourceCost::zero().with_retained_bytes(bytes), - progress_needed, - ) + resources.try_reserve_materialization(ResourceCost::zero().with_retained_bytes(bytes)) } #[cfg(test)] @@ -226,7 +222,7 @@ pub(super) fn dispatch( && pool .resource_ledger() .materialization_headroom() - .is_ok_and(|bytes| bytes <= minimum) + .is_ok_and(|(bytes, working)| working && bytes <= minimum) { return; } @@ -260,13 +256,7 @@ pub(super) fn dispatch( ) .ok_or(Error::Capacity("checkpoint metadata admission")) }) - .and_then(|bytes| { - reserve_materialization( - &pool.resource_ledger(), - bytes, - available == MAX_MATERIALIZERS, - ) - }); + .and_then(|bytes| reserve_materialization(&pool.resource_ledger(), bytes)); let Ok(reservation) = reservation else { continue; }; diff --git a/crates/cellule-runtime/src/cell/actor/materialization/tests.rs b/crates/cellule-runtime/src/cell/actor/materialization/tests.rs index bd3c8843..6a924b81 100644 --- a/crates/cellule-runtime/src/cell/actor/materialization/tests.rs +++ b/crates/cellule-runtime/src/cell/actor/materialization/tests.rs @@ -1,6 +1,53 @@ use super::*; use crate::fleet::resource::{ResourceCost, ResourceLedger, RetentionStage}; +#[test] +fn checkpoint_metadata_does_not_hold_the_only_cleanup_admission() { + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(64 << 20)); + let publisher = resources + .try_reserve_in( + RetentionStage::Publisher, + ResourceCost::zero().with_retained_bytes(22 << 20), + ) + .unwrap(); + let selection = resources + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(10 << 20), + ) + .unwrap(); + let mut working = reserve_materialization(&resources, 10 << 20).unwrap(); + assert_eq!(resources.snapshot().unwrap().working_materializations, 1); + let checkpoint = working + .split_retained(crate::node::bundle::MaterializedBundlePrefix::maximum_retained_bytes()) + .unwrap(); + // The first actor is still materializing until its exact checkpoint joins, + // but its original root I/O and working buffers have already exited. + drop(working); + assert_eq!(resources.snapshot().unwrap().working_materializations, 0); + let next = reserve_materialization(&resources, 10 << 20) + .expect("a joined root I/O must release the single cleanup opportunity"); + assert!(reserve_materialization(&resources, 5 << 20).is_err()); + let commands = resources + .try_reserve_in( + RetentionStage::Request, + ResourceCost::zero().with_retained_bytes(4 << 20), + ) + .unwrap(); + let snapshot = resources.snapshot().unwrap(); + assert!( + snapshot.used.retained_bytes() + < crate::fleet::resource::retained_admission_threshold(snapshot.limit.retained_bytes()) + ); + drop(commands); + drop(next); + assert_eq!(resources.snapshot().unwrap().working_materializations, 0); + drop(checkpoint); + drop(selection); + drop(publisher); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); +} + #[test] fn working_roots_leave_room_for_foreground_and_existing_debt_can_progress() { let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(64 << 20)); @@ -16,9 +63,9 @@ fn working_roots_leave_room_for_foreground_and_existing_debt_can_progress() { ResourceCost::zero().with_retained_bytes(10 << 20), ) .unwrap(); - let root = reserve_materialization(&resources, 5 << 20, true).unwrap(); + let root = reserve_materialization(&resources, 5 << 20).unwrap(); assert!( - reserve_materialization(&resources, 5 << 20, false).is_err(), + reserve_materialization(&resources, 5 << 20).is_err(), "parallel root buffers consumed foreground burst headroom" ); let commands = resources @@ -41,8 +88,8 @@ fn working_roots_leave_room_for_foreground_and_existing_debt_can_progress() { ResourceCost::zero().with_retained_bytes(16 << 20), ) .unwrap(); - let cleanup = reserve_materialization(&resources, 5 << 20, true).unwrap(); - assert!(reserve_materialization(&resources, 5 << 20, false).is_err()); + let cleanup = reserve_materialization(&resources, 5 << 20).unwrap(); + assert!(reserve_materialization(&resources, 5 << 20).is_err()); drop(more_debt); selected.shrink_retained(0).unwrap(); drop(cleanup); diff --git a/crates/cellule-runtime/src/fleet/resource.rs b/crates/cellule-runtime/src/fleet/resource.rs index e71cdc7e..739a3a54 100644 --- a/crates/cellule-runtime/src/fleet/resource.rs +++ b/crates/cellule-runtime/src/fleet/resource.rs @@ -374,6 +374,17 @@ pub struct ResourceSnapshot { /// Retained ownership, indexed by [`RetentionStage::ALL`]. Its sum equals /// `used.retained_bytes()` in this same locked snapshot. pub retained_by_stage: [usize; 7], + /// Admitted root I/O lifetimes whose working buffers have not yet exited. + /// Joined roots waiting for their exact checkpoint retain bytes but no + /// working admission. + pub working_materializations: usize, +} + +#[derive(Clone, Copy)] +enum WorkingAdmission { + Unrestricted, + Optional, + Materialization, } #[derive(Clone)] @@ -400,6 +411,7 @@ impl ResourceLedger { used: ResourceCost::zero(), limit, retained_by_stage: [0; 7], + working_materializations: 0, }), released: tokio::sync::Notify::new(), }), @@ -419,36 +431,40 @@ impl ResourceLedger { stage: RetentionStage, cost: ResourceCost, ) -> Result { - self.try_reserve_inner(stage, cost, false) + self.try_reserve_inner(stage, cost, WorkingAdmission::Unrestricted) } /// Optional overlap must leave the foreground's existing progress reserve. /// Check and admission share the lock so concurrent receipts cannot defeat it. pub(crate) fn try_reserve_pipeline(&self, cost: ResourceCost) -> Result { - self.try_reserve_inner(RetentionStage::Pipeline, cost, true) + self.try_reserve_inner(RetentionStage::Pipeline, cost, WorkingAdmission::Optional) } pub(crate) fn try_reserve_materialization( &self, cost: ResourceCost, - progress_needed: bool, ) -> Result { - // The actor allows one cleanup when no original materializer is alive. - // Debt already above the working threshold must still be reducible. - self.try_reserve_inner(RetentionStage::Materialization, cost, !progress_needed) + self.try_reserve_inner( + RetentionStage::Materialization, + cost, + WorkingAdmission::Materialization, + ) } - pub(crate) fn materialization_headroom(&self) -> Result { + pub(crate) fn materialization_headroom(&self) -> Result<(usize, bool)> { let state = self.snapshot()?; - Ok(publication_working_threshold(state.limit.retained_bytes()) - .saturating_sub(state.used.retained_bytes())) + Ok(( + publication_working_threshold(state.limit.retained_bytes()) + .saturating_sub(state.used.retained_bytes()), + state.working_materializations != 0, + )) } fn try_reserve_inner( &self, stage: RetentionStage, cost: ResourceCost, - preserve_headroom: bool, + admission: WorkingAdmission, ) -> Result { let mut state = self .state @@ -462,6 +478,15 @@ impl ResourceLedger { if !next.fits_within(state.limit) { return Err(Error::Capacity("resource ledger")); } + let working = matches!(admission, WorkingAdmission::Materialization); + // One root I/O may reduce debt beyond the working threshold. Its + // original buffers release this opportunity before checkpoint waiting; + // retained prefix metadata cannot serialize unrelated root I/O. + let preserve_headroom = match admission { + WorkingAdmission::Unrestricted => false, + WorkingAdmission::Optional => true, + WorkingAdmission::Materialization => state.working_materializations != 0, + }; if preserve_headroom && next.retained_bytes() >= publication_working_threshold(state.limit.retained_bytes()) { @@ -470,12 +495,18 @@ impl ResourceLedger { _ => "bundle pipeline headroom", })); } + let working_materializations = state + .working_materializations + .checked_add(usize::from(working)) + .ok_or(Error::Capacity("materialization admission arithmetic"))?; state.used = next; + state.working_materializations = working_materializations; state.retained_by_stage[stage as usize] += cost.retained_bytes; Ok(ResourceReservation { ledger: self.clone(), cost, stage, + working, }) } @@ -591,11 +622,15 @@ impl ResourceLedger { Ok(()) } - fn release(&self, stage: RetentionStage, cost: ResourceCost) { + fn release(&self, stage: RetentionStage, cost: ResourceCost, working: bool) { if let Ok(mut state) = self.state.snapshot.lock() && let Some(used) = state.used.checked_sub(cost) + && let Some(working_materializations) = state + .working_materializations + .checked_sub(usize::from(working)) { state.used = used; + state.working_materializations = working_materializations; state.retained_by_stage[stage as usize] -= cost.retained_bytes; } } @@ -607,6 +642,7 @@ pub(crate) struct ResourceReservation { ledger: ResourceLedger, cost: ResourceCost, stage: RetentionStage, + working: bool, } impl ResourceReservation { @@ -642,6 +678,7 @@ impl ResourceReservation { ledger: self.ledger.clone(), cost: ResourceCost::zero().with_retained_bytes(bytes), stage: self.stage, + working: false, }) } @@ -656,6 +693,7 @@ impl ResourceReservation { self.ledger.release( self.stage, ResourceCost::zero().with_retained_bytes(released), + false, ); self.ledger.state.released.notify_waiters(); Ok(()) @@ -664,7 +702,7 @@ impl ResourceReservation { impl Drop for ResourceReservation { fn drop(&mut self) { - self.ledger.release(self.stage, self.cost); + self.ledger.release(self.stage, self.cost, self.working); self.ledger.state.released.notify_waiters(); } } From 5fcc00a1cbf3bb5235d6fc55290539f9cf8466e8 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 13:02:41 -0700 Subject: [PATCH 06/31] fix(ci): verify the current active-owner startup refusal --- scripts/bench-axum-rustfs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/bench-axum-rustfs.py b/scripts/bench-axum-rustfs.py index a76563d0..8d634218 100644 --- a/scripts/bench-axum-rustfs.py +++ b/scripts/bench-axum-rustfs.py @@ -294,8 +294,8 @@ def steady_reads(args, directory, service, concurrency, acknowledged): def verify_active_owner_refused(binary, directory, prefix, cells, workers): env = os.environ.copy() - # Exercise the same Cell authority refusal without trying to enroll another - # boot for the fixture's already-live physical leader identity. + # Without directory enrollment this boot cannot prove takeover of the + # fixture's active predecessor. Check that precise startup refusal. env.pop("CELLULE_AXUM_FLEET_DIR", None) env.pop("CELLULE_AXUM_FOLLOWER", None) env.update(CELLULE_TEST_PREFIX=prefix, CELLULE_AXUM_BIND="127.0.0.1:0", @@ -304,7 +304,7 @@ def verify_active_owner_refused(binary, directory, prefix, cells, workers): with log.open("w") as output: candidate = subprocess.run([str(binary)], env=env, stdout=output, stderr=subprocess.STDOUT, timeout=120) require(candidate.returncode != 0, "second service took an active Cell") - require("idle acquisition requires a published idle control" in log.read_text(), "second service failed for an unexpected reason") + require('Node("crash takeover requires real directory enrollment")' in log.read_text(), "second service failed for an unexpected reason") def point(args, directory, repeat, concurrency, cells): From 4ee7cf523e8b28f62c97601074db2752e42cf763 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 13:05:33 -0700 Subject: [PATCH 07/31] perf(runtime): account retained control allocations by capacity --- crates/cellule-runtime/src/control/mod.rs | 23 +++++++++++ crates/cellule-runtime/src/node/bundle/mod.rs | 7 +++- .../src/node/bundle/selection.rs | 3 +- .../src/node/bundle/tests/ranges.rs | 39 +++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/crates/cellule-runtime/src/control/mod.rs b/crates/cellule-runtime/src/control/mod.rs index a84680e9..34a336bc 100644 --- a/crates/cellule-runtime/src/control/mod.rs +++ b/crates/cellule-runtime/src/control/mod.rs @@ -217,6 +217,29 @@ pub enum Transition { } impl Control { + /// Heap storage retained by an already decoded control. Inline fields are + /// charged by the containing proof; codec buffers have working admission. + pub(crate) fn retained_heap_bytes(&self) -> usize { + // Keep this exhaustive: adding a persisted field requires deciding + // whether its allocation belongs to retained proof admission. + let Self { + cell: _, + incarnation: _, + epoch: _, + revision: _, + progress: _, + state: _, + owner, + root: _, + recovery: _, + bundle_binding: _, + code: _, + schema: _, + next_due_ms: _, + } = self; + owner.as_ref().map_or(0, |owner| owner.endpoint.capacity()) + } + /// Returns this record's incarnation/epoch identity, without granting authority. #[must_use] pub const fn owner_fence(&self) -> OwnerFence { diff --git a/crates/cellule-runtime/src/node/bundle/mod.rs b/crates/cellule-runtime/src/node/bundle/mod.rs index e08cee6b..f0ef3284 100644 --- a/crates/cellule-runtime/src/node/bundle/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/mod.rs @@ -292,7 +292,10 @@ impl BundleCoverageProof { } pub(crate) fn retained_metadata_bytes(&self) -> Result { - let control = self.binding.control.encode()?.len(); + // This proof retains one decoded control, never serialized copies. + // Charge its endpoint allocation by capacity and its inline fields + // through Self below. Encoding scratch belongs to publication work. + let control_heap = self.binding.control.retained_heap_bytes(); self.binding .locators .capacity() @@ -304,7 +307,7 @@ impl BundleCoverageProof { .map_or(0, |live| live.retained_metadata_bytes()), ) }) - .and_then(|bytes| bytes.checked_add(control.checked_mul(4)?)) + .and_then(|bytes| bytes.checked_add(control_heap)) .and_then(|bytes| bytes.checked_add(std::mem::size_of::() + 256)) .ok_or(Error::Capacity("selected bundle metadata")) } diff --git a/crates/cellule-runtime/src/node/bundle/selection.rs b/crates/cellule-runtime/src/node/bundle/selection.rs index e09640ff..63142ec0 100644 --- a/crates/cellule-runtime/src/node/bundle/selection.rs +++ b/crates/cellule-runtime/src/node/bundle/selection.rs @@ -126,7 +126,8 @@ impl NodeDirectory { } // Cloned locator arrays cannot exceed the validated history bound. // Collected live assignments grow geometrically within the 64-frame - // cohort. Control-owned strings/maps retain their existing 4x charge. + // cohort. Preserve the conservative codec/endpoint preflight allowance; + // transfer only concrete retained allocations after verification. let origin = self.layout.node_path(&[0; 16]).to_string(); let each = MAX_LOCATORS * std::mem::size_of::() + MAX_FRAMES * std::mem::size_of::() diff --git a/crates/cellule-runtime/src/node/bundle/tests/ranges.rs b/crates/cellule-runtime/src/node/bundle/tests/ranges.rs index 1f38b6a8..3afdb445 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/ranges.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/ranges.rs @@ -1,6 +1,45 @@ use super::*; use object_store::ObjectStoreExt as _; +#[tokio::test] +async fn retained_proof_charges_endpoint_capacity_without_serialized_control_copies() { + let mut f = Fixture::new().await; + let mut cell = f.cell(4).await; + let (_, frames, assigned) = f.append(&mut cell, 2); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (_, mut proofs) = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + let mut proof = proofs.pop().unwrap(); + let original = proof.retained_metadata_bytes().unwrap(); + let owner = proof.binding.control.owner.as_mut().unwrap(); + let original_capacity = owner.endpoint.capacity(); + // Identical authority bytes can retain a larger allocation. Accounting + // must observe capacity, rather than infer storage from serialized length. + let encoded = owner.endpoint.clone(); + owner.endpoint.reserve_exact(8192); + let extra = owner.endpoint.capacity() - original_capacity; + assert_eq!(owner.endpoint, encoded); + assert_eq!(proof.retained_metadata_bytes().unwrap(), original + extra); + + // One small selected proof should not retain transient codec buffers. + // This is a footprint ceiling, independent of the accounting formula. + assert!( + original < 2048, + "selected proof retains serialized control scratch" + ); + assert!( + proof.retained_metadata_bytes().unwrap() + <= f.directory.bundle_receipt_memory_bound(1).unwrap() + ); +} + #[tokio::test] async fn missing_base_chunk_cannot_mint_a_selected_reconstruction_proof() { let mut f = Fixture::new().await; From 03d2eba21c64105841c5ff0d2cb671c4148edf53 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 13:33:56 -0700 Subject: [PATCH 08/31] fix(runtime): join authority activation outside proof waiter lifetimes --- .../src/node/durability/activation.rs | 106 +++++++++ .../src/node/durability/mod.rs | 54 ++--- .../durability/tests/activation_lifetime.rs | 213 ++++++++++++++++++ .../src/node/durability/tests/mod.rs | 1 + 4 files changed, 345 insertions(+), 29 deletions(-) create mode 100644 crates/cellule-runtime/src/node/durability/activation.rs create mode 100644 crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs diff --git a/crates/cellule-runtime/src/node/durability/activation.rs b/crates/cellule-runtime/src/node/durability/activation.rs new file mode 100644 index 00000000..8a9e9768 --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/activation.rs @@ -0,0 +1,106 @@ +//! One epoch-owned activation callback, independent of proof waiters. +use std::sync::{Arc, Mutex}; + +use futures_util::FutureExt; +use futures_util::future::{BoxFuture, Shared}; + +use super::NodeLogAuthority; +use crate::node::lease::NodeLeaseGuard; +use crate::node::log::DurabilityGate; +use crate::{Error, Result}; + +type Attempt = std::result::Result<(), Arc>; +type Completion = Shared>>>; + +#[derive(Default)] +pub(super) struct Activation { + state: Mutex, +} + +#[derive(Default)] +struct State { + completion: Option, + joining: bool, +} + +impl Activation { + pub(super) async fn wait( + &self, + epoch: u64, + gate: &DurabilityGate, + authority: &Arc, + lease: &NodeLeaseGuard, + ) -> Result<()> { + let completion = { + let mut state = self + .state + .lock() + .map_err(|_| Error::Node("node activation lock poisoned"))?; + if !state.joining + && state + .completion + .as_ref() + .is_some_and(|completion| matches!(completion.peek(), Some(Ok(Err(_))))) + { + // A completed failed attempt can be retried, but never overlap + // another callback or restart after shutdown begins joining. + state.completion = None; + } + if let Some(completion) = &state.completion { + completion.clone() + } else { + if state.joining { + return Err(Error::RuntimeClosed); + } + let runtime = tokio::runtime::Handle::try_current().map_err(Error::RuntimeStart)?; + let gate = gate.clone(); + let authority = Arc::clone(authority); + let lease = lease.clone(); + // Dropping a proof waiter (including an Object proof winning) + // cannot drop an accepted CAS before the adapter joins its reply + // and updates the version used by later authority mutations. + let worker = runtime.spawn(async move { + lease.check()?; + tokio::select! { + result = authority.activate(epoch) => result?, + () = lease.wait_fenced() => return Err(Error::Fenced), + } + lease.check()?; + gate.activate_fleet() + }); + let completion = async move { + worker + .await + .map(|result| result.map_err(Arc::new)) + .map_err(|error| Arc::new(Error::FollowerWorkerJoin(error))) + } + .boxed() + .shared(); + state.completion = Some(completion.clone()); + completion + } + }; + completion + .await + .map_err(Error::Shared)? + .map_err(Error::Shared) + } + + pub(super) async fn join(&self) -> Result<()> { + let completion = { + let mut state = self + .state + .lock() + .map_err(|_| Error::Node("node activation lock poisoned"))?; + state.joining = true; + state.completion.clone() + }; + if let Some(completion) = completion { + // Callback failure grants no Fleet proof. Shutdown may still close + // through complete Object coverage, which it checks separately. + // Worker failure remains a source error: its callback did not join. + let _attempt = completion.await.map_err(Error::Shared)?; + } + Ok(()) + } +} diff --git a/crates/cellule-runtime/src/node/durability/mod.rs b/crates/cellule-runtime/src/node/durability/mod.rs index f878b52b..a5d99730 100644 --- a/crates/cellule-runtime/src/node/durability/mod.rs +++ b/crates/cellule-runtime/src/node/durability/mod.rs @@ -15,6 +15,7 @@ use crate::node::log_shipper::{NodeLogShipper, NodeLogSubmission}; use crate::node::log_transport::NodeLogTransport; use crate::{Error, Result}; +mod activation; mod object_coverage; use object_coverage::ObjectCoverage; mod publication; @@ -195,7 +196,7 @@ pub struct NodeDurability { authority: Arc, transport: Arc, node_lease: NodeLeaseGuard, - activated: OnceCell<()>, + activation: activation::Activation, object_coverage: ObjectCoverage, shutdown: Mutex<()>, retirement: std::sync::Mutex>>, @@ -245,7 +246,7 @@ impl NodeDurability { authority, transport, node_lease, - activated: OnceCell::new(), + activation: activation::Activation::default(), object_coverage: ObjectCoverage::default(), shutdown: Mutex::new(()), retirement: std::sync::Mutex::new(None), @@ -487,18 +488,7 @@ impl NodeDurability { result = self.gate.wait_followers(ticket) => result?, () = self.node_lease.wait_fenced() => return Err(Error::Fenced), } - self.activated - .get_or_try_init(|| async { - self.node_lease.check()?; - tokio::select! { - result = self.authority.activate(ticket.log_epoch()) => result?, - () = self.node_lease.wait_fenced() => return Err(Error::Fenced), - } - self.node_lease.check()?; - self.gate.activate_fleet()?; - Ok::<(), Error>(()) - }) - .await?; + self.activate_fleet(ticket.log_epoch()).await?; let proof = tokio::select! { result = self.gate.prove(ticket) => result?, () = self.node_lease.wait_fenced() => return Err(Error::Fenced), @@ -517,18 +507,7 @@ impl NodeDurability { self.node_lease.check()?; let activate = async { self.gate.wait_followers(ticket).await?; - self.activated - .get_or_try_init(|| async { - self.node_lease.check()?; - tokio::select! { - result = self.authority.activate(ticket.log_epoch()) => result?, - () = self.node_lease.wait_fenced() => return Err(Error::Fenced), - } - self.node_lease.check()?; - self.gate.activate_fleet()?; - Ok::<(), Error>(()) - }) - .await?; + self.activate_fleet(ticket.log_epoch()).await?; Ok::<(), Error>(()) }; tokio::pin!(activate); @@ -544,6 +523,15 @@ impl NodeDurability { Ok(proof) } + async fn activate_fleet(&self, epoch: u64) -> Result<()> { + let activation = self + .activation + .wait(epoch, &self.gate, &self.authority, &self.node_lease) + .await; + self.node_lease.check()?; + activation + } + /// Records an already-published object root and persists its contiguous watermark. /// /// Callers must complete the exact Cell root CAS before invoking this method. @@ -679,12 +667,20 @@ impl NodeDurability { return Ok(proof); } let shipping = self.shipper.shutdown().await; - if let Some(publisher) = self.publisher.get() { + let publication = if let Some(publisher) = self.publisher.get() { // Join both tasks even when fencing stopped the shipper. Preserve // the producer's original cause rather than its secondary fence. - publisher.join().await?; - } + publisher.join().await + } else { + Ok(()) + }; + // Activation may outlive its original waiter. Join it before coverage + // and closure use the authority adapter's acknowledged version, even + // when shipping or publication has already failed. + let activation = self.activation.join().await; + publication?; shipping?; + activation?; self.object_coverage .flush(&self.gate, self.authority.as_ref(), &self.node_lease, &[]) .await?; diff --git a/crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs b/crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs new file mode 100644 index 00000000..c9453696 --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs @@ -0,0 +1,213 @@ +use super::*; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +#[derive(Default)] +struct CommittedActivation { + state: tokio::sync::Mutex<()>, + remote: AtomicBool, + local: AtomicBool, + calls: AtomicUsize, + committed: Notify, + resume: Notify, + joined: Notify, +} + +impl NodeLogAuthority for CommittedActivation { + fn activate<'a>(&'a self, _epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let _state = self.state.lock().await; + self.calls.fetch_add(1, Ordering::AcqRel); + // The provider has accepted the CAS, but its response and the + // adapter's acknowledged version have not yet joined. + self.remote.store(true, Ordering::Release); + self.committed.notify_one(); + self.resume.notified().await; + self.local.store(true, Ordering::Release); + self.joined.notify_one(); + Ok(()) + }) + } + + fn advance_coverage<'a>(&'a self, _epoch: u64, _through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let _state = self.state.lock().await; + if self.remote.load(Ordering::Acquire) != self.local.load(Ordering::Acquire) { + return Err(Error::Node( + "coverage CAS used a cancelled activation version", + )); + } + Ok(()) + }) + } + + fn close<'a>( + &'a self, + _retirement: &'a crate::node::log::NodeLogRetirementObservation, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn cancelled_proof_waiter_keeps_accepted_activation_joined() { + for fleet_only in [true, false] { + cancelled_waiter(fleet_only).await; + } +} + +fn runtime(authority: Arc) -> Arc { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); + let transport: Arc = Arc::new(ImmediateTransport); + let shipper = NodeLogShipper::new( + gate.clone(), + Arc::clone(&transport), + cellule_ltx::Limits::default(), + ) + .unwrap(); + Arc::new(NodeDurability::new( + gate, + shipper, + authority, + transport, + lease(), + )) +} + +async fn cancelled_waiter(fleet_only: bool) { + let authority = Arc::new(CommittedActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + let waiter = tokio::spawn({ + let durability = durability.clone(); + async move { + if fleet_only { + durability.prove_fleet(ticket).await + } else { + durability.prove(ticket).await + } + } + }); + tokio::time::timeout( + std::time::Duration::from_secs(1), + authority.committed.notified(), + ) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + authority.resume.notify_one(); + tokio::time::timeout( + std::time::Duration::from_millis(250), + authority.joined.notified(), + ) + .await + .expect("accepted authority I/O was cancelled with its proof waiter"); + assert!(authority.local.load(Ordering::Acquire)); + assert_eq!( + durability.prove_fleet(ticket).await.unwrap().source(), + DurabilitySource::Fleet + ); + assert_eq!(authority.calls.load(Ordering::Acquire), 1); + durability.prove_object(ticket).await.unwrap(); + durability.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn shutdown_joins_cancelled_waiters_activation_before_coverage() { + let authority = Arc::new(CommittedActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + let waiter = tokio::spawn({ + let durability = durability.clone(); + async move { durability.prove(ticket).await } + }); + tokio::time::timeout( + std::time::Duration::from_secs(1), + authority.committed.notified(), + ) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let mut shutdown = tokio::spawn({ + let durability = durability.clone(); + async move { durability.shutdown().await } + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), &mut shutdown) + .await + .is_err() + ); + authority.resume.notify_one(); + // Joining the callback does not invent bucket coverage for the Fleet tail. + assert!(matches!( + shutdown.await.unwrap(), + Err(Error::PendingPublication) + )); + assert!(authority.local.load(Ordering::Acquire)); + durability.prove_object(ticket).await.unwrap(); + durability.shutdown().await.unwrap(); + assert_eq!(authority.calls.load(Ordering::Acquire), 1); +} + +#[derive(Default)] +struct RetryActivation(AtomicUsize); + +impl NodeLogAuthority for RetryActivation { + fn activate<'a>(&'a self, _epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async { + if self.0.fetch_add(1, Ordering::AcqRel) == 0 { + Err(Error::Node("injected activation failure")) + } else { + Ok(()) + } + }) + } + + fn advance_coverage<'a>(&'a self, _epoch: u64, _through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } + + fn close<'a>( + &'a self, + _retirement: &'a crate::node::log::NodeLogRetirementObservation, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn completed_activation_failure_can_retry_without_reissuing_the_ticket() { + let authority = Arc::new(RetryActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + let error = durability.prove_fleet(ticket).await.unwrap_err(); + assert!( + matches!(error, Error::Shared(source) if matches!(source.as_ref(), Error::Node("injected activation failure"))) + ); + assert_eq!( + durability.prove_fleet(ticket).await.unwrap().source(), + DurabilitySource::Fleet + ); + assert_eq!(authority.0.load(Ordering::Acquire), 2); + durability.prove_object(ticket).await.unwrap(); + durability.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn failed_activation_can_drain_through_complete_object_coverage() { + let authority = Arc::new(RetryActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + durability.prove_fleet(ticket).await.unwrap_err(); + assert_eq!( + durability.prove_object(ticket).await.unwrap().source(), + DurabilitySource::Object + ); + durability.shutdown().await.unwrap(); + assert_eq!(authority.0.load(Ordering::Acquire), 1); +} diff --git a/crates/cellule-runtime/src/node/durability/tests/mod.rs b/crates/cellule-runtime/src/node/durability/tests/mod.rs index bfec28d1..fca3a397 100644 --- a/crates/cellule-runtime/src/node/durability/tests/mod.rs +++ b/crates/cellule-runtime/src/node/durability/tests/mod.rs @@ -11,6 +11,7 @@ use crate::node::log_transport::{ AppendRequest, NodeLogTransport, RetireRequest, SealRequest, TailRequest, }; +mod activation_lifetime; mod closure; #[derive(Default)] From 7efda00804bd416942623f406a811dba3494b96d Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 14:32:04 -0700 Subject: [PATCH 09/31] docs: record fresh publication measurements and capacity priorities --- .../docs/write-performance-design.md | 31 ++- docs/bundle-coverage-implementation.md | 10 +- docs/managed-bundle-pipeline.md | 79 ++++-- docs/publication-progress-measurements.md | 248 ++++++++++++++++++ 4 files changed, 334 insertions(+), 34 deletions(-) create mode 100644 docs/publication-progress-measurements.md diff --git a/crates/cellule-runtime/docs/write-performance-design.md b/crates/cellule-runtime/docs/write-performance-design.md index 8ad08df0..9d8b0bc4 100644 --- a/crates/cellule-runtime/docs/write-performance-design.md +++ b/crates/cellule-runtime/docs/write-performance-design.md @@ -1,16 +1,25 @@ # Node write and read performance design -The latest completed production mixed-load diagnostic reaches 1,249.90 Fleet -writes/s and 12,580.70 reads/s across 2,000 Cells on an 8-vCPU/16-GiB owner. -Write request p99 is 240.5 ms; read request p99 is 48.1 ms. Queued offers are -dropped and root debt grows. A checkpoint-overlap experiment completes fewer -requests and grows more debt in its complete pair, so production retains the -checkpoint serial guard. All 402,167 ACKs across three completed cases pass -independent warm/cold state and original retry audits. The final baseline repeat -fails with VM filesystem I/O errors and is excluded. See the -[current managed-pipeline measurements](../../../docs/managed-bundle-pipeline.md#current-application-measurement). -These 60-second tmpfs diagnostics do not qualify capacity or performance parity; -the acceptance gates below remain unchanged. +A fresh five-minute public-application run at 2,000 Cells sustains 500 offered +Fleet writes/s with zero errors, drops or admission refusals under the unchanged +64-MiB retained-work budget. A fresh repeat after joining authority activation +also has zero errors/drops: all 167,001 ACKs pass warm/cold readback and exact +retries; drain takes 39.25 seconds. Scheduled write p99 is 16.0 ms, and root age +stays around 42–49 seconds. These are development diagnostics with tmpfs local +state; they do not qualify the 2K-write/20K-read target or +physical-media durability. See [fresh publication-progress measurements](../../../docs/publication-progress-measurements.md) +for source identity, actual resources, phase observations and retained failures. +The acceptance gates below remain unchanged. + +The fresh 60-second mixed diagnostic completes 971.25 writes/s and 18,529.95 +reads/s at 2K/20K offered load. Writes have 53,037 errors and 8,662 drops; reads +have zero returned errors and 88,152 drops. Publication backlog reaches the +foreground threshold, root age grows 57.7→88.7 seconds, and checkpoints force +serial publication in 1,019 of 1,022 rounds. A read-only window reaches 15,806.40/s +with zero returned errors and 251,484 drops under sampled competing compiler +load. All 90,562 ACKs pass warm/cold reads and exact retries; drain takes 38.77 +seconds. See the same fresh report for actual host/storage differences and phase +attribution. The full mixed/read targets remain unqualified. The following measurements remain historical observations. diff --git a/docs/bundle-coverage-implementation.md b/docs/bundle-coverage-implementation.md index 2fbf1be1..295b748c 100644 --- a/docs/bundle-coverage-implementation.md +++ b/docs/bundle-coverage-implementation.md @@ -1,6 +1,11 @@ # Bundle coverage implementation -The current paired 2,000-Cell measurements +Fresh Node Bundle v1 measurements pass a five-minute 500-offered-write/s +milestone; the separate 2K-write/20K-read target remains unqualified. See +[publication-progress measurements](publication-progress-measurements.md) for +the current source, resources, errors, drops and complete ACK audits. + +The historical paired 2,000-Cell measurements complete 502.28 Fleet writes/s versus 455.73 before and 1,999.83 for fresh celld. Metadata windows reduce observed GET/range work, but successful scheduled write p99 worsens 827.14→1,454.42 ms and returned errors increase. Read-only @@ -236,7 +241,8 @@ reverified. The 215-command checkpoint density is now represented and measured for a small-image fixture; total lifecycle cost is not qualified. Host memory, native-job admission and materializer progress still require qualification. The actor now owns bounded admitted scheduling and joins accepted jobs; its -latest application measurement fails availability. See the +fresh mixed application measurement fails delivery at the full offered targets, +while the smaller 500-write/s milestone passes. See the [node performance design](../crates/cellule-runtime/docs/write-performance-design.md). ## Verification and remaining delivery diff --git a/docs/managed-bundle-pipeline.md b/docs/managed-bundle-pipeline.md index 61b6c9d8..09264286 100644 --- a/docs/managed-bundle-pipeline.md +++ b/docs/managed-bundle-pipeline.md @@ -1,16 +1,17 @@ # Managed bundle publication pipeline -October 10, 2026. The runtime-owned producer can stage and upload one successor +October 11, 2026. The runtime-owned producer can stage and upload one successor while its predecessor is publishing. Selection and receipt confirmation remain -ordered. The latest completed production diagnostic reaches 1,249.90 writes/s -and 12,580.70 reads/s, with 240.5-ms write request p99, dropped offers and growing -root-materialization debt. Removing the checkpoint serial guard did not improve -completed throughput in its complete pair and remains an unadopted experiment. -**The 2,000-Cell mixed-load goal remains unmet; these short runs do not qualify -sustainable capacity.** +ordered. Stage accounting and phase telemetry now identify retained-byte pressure +and root work separately. Optional overlap and parallel materialization preserve +foreground headroom; historical frame reads overlap within the original admitted +suffix bound. -These measurements predate the [Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds). -They are historical evidence, not measurements of the new format. +Fresh 2,000-Cell runs with the current format are recorded in +[publication-progress measurements](publication-progress-measurements.md). +The historical application table below predates the +[Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds); +it does not qualify the current implementation or the mixed-load goal. ## Publication and admission @@ -36,6 +37,12 @@ bounds. Receipt admission covers the maximum native history, live assignments, bounded control allocation and origin path. Confirmation still checks actual costs before transferring reservations or waking any capture. +Retained receipts charge locator and assignment capacities, the origin path, +the control's endpoint capacity, and the inline proof and wrapper storage. +The decoded control holds no serialized copies; those transient buffers belong +to the unchanged publication working reservation. An exhaustive control-field +match keeps future heap-owning fields part of this accounting decision. + The two-cohort window is lazy: a capture can arrive after the first PUT has started. There is no requirement to collect both cohorts before dispatching the first. Selection advances only the original complete assignments and @@ -49,7 +56,7 @@ consumes frame facts in locator order. A failure stops new dispatch, joins the accepted window, preserves the first source error and grants no root credit. Base-origin verification remains fresh and precedes those reads. -When a materializer is already alive and even the minimum working reservation +When root I/O is already alive and even the minimum working reservation cannot fit, dispatch skips the repeated dirty-fleet walk and sort. This is only a scheduling hint; the original atomic admission still decides each job. @@ -69,6 +76,14 @@ or checkpoint I/O. Cancelling a shutdown caller leaves the original worker alive; a later shutdown joins that worker and observes its terminal result. Manual callers continue to own their admission and accepted I/O lifetimes. +The first follower proof also owns a node authority activation obligation. +One epoch-owned driver coalesces activation waiters; cancellation or an Object +proof winning cannot abandon an accepted CAS before the adapter joins its reply. +Shutdown freezes further activation starts and joins that driver before coverage +or closure, including earlier shipping/publication failures. Lease fencing remains +terminal. A failed callback grants no Fleet proof; complete Object coverage can +still drain the epoch. + ## Regression evidence The serving regression submits a second Cell only after the first immutable PUT @@ -98,7 +113,7 @@ compatibility, dependency verification and two-preparation model checks. Those model checks do not establish host memory accounting or adapter liveness; the managed tests cover those lifetimes separately. -## Current application measurement +## Historical application measurement The completed production baseline uses `4712c4c`, whose Rust/Cargo sources match `ea4903d` on main. One complete pair and an unpaired candidate repeat use an @@ -132,11 +147,11 @@ Builds, failed/passed logs and snapshots are retained outside Git under `3e62e448476193424d8c555ca9b021e130819f01f576ff180ff089bff2869b85`. These diagnostics establish neither sustainable capacity nor celld parity. -CNB3 stores a 32-KiB authenticated header, changed catalog shards, new native +The historical format stored a 32-KiB authenticated header, changed catalog shards, new native frames and detached Cell histories in one immutable upload. Unchanged extents -remain exact references to earlier objects. The -[format and recovery contract](bundle-coverage-implementation.md) describe its -digest chain and CNB1/CNB2 reader compatibility. The experiment changes no bytes. +remained exact references to earlier objects. The experiment changed no bytes. +The current [format and recovery contract](bundle-coverage-implementation.md) +uses Node Bundle v1 exclusively; legacy bundle readers were removed at cutover. ## Publication diagnostics @@ -159,17 +174,23 @@ fallback counters distinguish checkpoints, capacity, foreground headroom and an absent successor. Optional overlap admission atomically stays below the same 75% retained-byte threshold used by ordinary write admission, reserving another eighth of the budget below that threshold for foreground bursts. Parallel root -materializers use the same working threshold. When none is alive, one required -cleanup may use the original full budget so existing debt cannot starve its own -reduction. These rules do not increase the total budget or reduce receipt or -materialization bounds. +materializers use the same working threshold. When no root I/O is alive, one +required cleanup may use the original full budget so existing debt cannot starve +its own reduction. The working reservation releases this opportunity when root +I/O and its Cell CAS join. Prefix metadata remains admitted through checkpoint +and worker binding, without serializing another Cell's root I/O with that wait. +The original eight-task limit includes both working and checkpoint phases. +These rules do not increase the total budget or reduce receipt or materialization +bounds. `CellRuntime::retained_work` returns one locked ledger snapshot whose `retained_by_stage` counts sum to its retained total. Ownership classes follow the original reservation through split, reclassification, shrink and drop; -they do not alter admission or represent estimated heap use. Unclassified work +they report admitted reservations rather than sampled heap use. Unclassified work remains explicit as `other`. The separate native-log byte window still has its own accounting. The example exports this snapshot under `retained_work`. +Its `working_materializations` count reports original root I/O admissions in the +same locked snapshot; retained checkpoint metadata does not hold that count. Actor admission refusals are counted separately from application HTTP failures: a backlog-refused mutation may still resolve an existing durable outcome. @@ -185,7 +206,23 @@ before attributing an old root to a slow bundle upload. ## Verification -All 13 contributor routes pass on an isolated snapshot, including 2,095 +The publication-progress changes pass 2,112 workspace tests and 15 SQL example +tests on an isolated snapshot, plus strict all-target/all-feature Clippy, +formatting, boundary/layout checks and document/SQL validators. Focused tests +reproduce the original pipeline and materializer headroom failures, then verify +foreground admission and continued cleanup against the same ledger. A held-first +frame fixture reproduces serial historical reads; its bounded-overlap replacement +cold-restores the exact outcomes. The failure-window fixture verifies joined +accepted reads, preserved source errors and unchanged root authority. +Additional regressions distinguish joined root working admission from retained +checkpoint metadata, charge spare endpoint allocation without retaining codec +scratch, join accepted activation after proof-waiter cancellation, and bound the telemetry wrapper's async frame. The latter +reproduces a large inline child frame; boxing work at construction preserves phase +outcomes and avoids the webhook scenario's default-worker stack overflow. Its +unoptimized demo and complete process journey pass without enlarging stacks or +changing the original fault assertions. + +At the earlier pipeline milestone, all 13 contributor routes passed on an isolated snapshot, including 2,095 workspace tests with 43 environment-dependent cases ignored, strict Clippy, all targets/features, API docs, local LTX, boundary/layout checks and document validators. The focused bundle suite passes 125 cases. The seven TLS-dependent diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md new file mode 100644 index 00000000..e041ad73 --- /dev/null +++ b/docs/publication-progress-measurements.md @@ -0,0 +1,248 @@ +# Publication-progress measurements + +October 11, 2026. These fresh public-application diagnostics use Node Bundle v1, +two required followers, and the existing 64-MiB retained-work budget. They establish +a smaller 500-write/s acceptance milestone; the separate 2K-write/20K-read +qualification remains open. + +## Workload and resources + +| Setting | Actual configuration | +| --- | --- | +| Workload | 2,000 uniformly active Cells; distinct 96-byte SQL ledger INSERT plus SELECT; durable outcome retained for two hours | +| Requests | 30-second warmup; 128 in-flight requests and 128 queued offers; no timed-request retry | +| Serving owner | 8 vCPUs, 16 GiB; pinned CPUs 0–7 | +| Each follower | 1 vCPU, 2 GiB | +| Object provider | RustFS, 2 vCPUs, 2 GiB; fresh provider volume | +| Client/auditor | 4 vCPUs, 2 GiB | +| Linux VM | 12 vCPUs, 24 GiB; support roles share CPUs 8–11 | +| Host | Mac14,13; 12 physical/logical CPUs; 32 GiB RAM | +| Local database/follower storage | tmpfs; provider data on the VM disk | +| Evidence | Every ACK is read back and retried with the original request before and after joined drain/cold restore | + +The v6 and v7 write-only VM disks were under `/Users/haipingfu/.colima`, on the internal host volume. +The installed Colima CLI ignored the attempted `COLIMA_HOME` override; no +external-to-internal disk relocation occurred. The earlier failing profile and +its disk images remain preserved. Host process samples retain competing compiler +load. This shared development host is not a dedicated capacity-qualification +machine; these runs do not establish causal speedup percentages, physical disk +survival, maximum TPS, or celld parity. The runtime reads in these diagnostics +use the current live-owner lease mode; they do not implement a new per-read +follower barrier protocol. + +## Five-minute 500-write/s milestone + +Source `4ee7cf523e8b28f62c97601074db2752e42cf763`; all eight framework packages +were cleaned before building with pinned Rust 1.98.1. The SQL binary SHA-256 is +`5299a7dc2ce33b1cca3992f7f2e0001e5f2dd5ab14dec6f9f3617b4c88460dee`. + +| Measurement | Result | +| --- | ---: | +| Planned/generated writes | 150,000 / 150,000 | +| Completed within 300 seconds | 149,997 (499.99/s) | +| Completed including final in-flight requests | 150,000 | +| Errors / queue drops / unissued offers | 0 / 0 / 0 | +| Warmup errors / drops | 0 / 0 | +| Scheduled latency p50 / p95 / p99 | 6.6 / 10.3 / 16.1 ms | +| Request latency p50 / p95 / p99 | 5.8 / 9.1 / 13.9 ms | +| ACKs checked warm and cold, including setup/warmup/probe | 167,001 | +| Warm/cold readback or exact-retry failures | 0 / 0 | +| Joined owner/follower drain | 47.28 s | +| Cold startup | 11.84 s | + +Three offers complete just after the measurement boundary; the delivery test +counts them separately. Minute snapshots report zero admission refusals. + +| Window sample | Retained reservation bytes | Oldest root age | Issued minus bucket-covered frames | Pending root publications | +| --- | ---: | ---: | ---: | ---: | +| Start | 32,664,702 | 45.019 s | 8 | 1,902 | +| Minute 1 | 28,860,412 | 41.999 s | 6 | 1,905 | +| Minute 2 | 29,162,452 | 44.588 s | 8 | 1,980 | +| Minute 3 | 29,281,360 | 43.721 s | 25 | 1,860 | +| Minute 4 | 29,607,202 | 42.818 s | 9 | 1,851 | +| End | 39,664,838 | 46.113 s | 14 | 1,969 | + +Root age and population remain bounded around the existing 45-second policy +trigger. The end reservation includes 11,305,976 bytes of active materialization; +selection retains 5,242,400 bytes, and the publisher reserves 23,068,672 bytes. +These are ledger reservations, not a measurement of total process heap. + +| Completed phase | Count | Mean duration | +| --- | ---: | ---: | +| Preparation | 18,683 | 5.04 ms | +| Upload | 18,683 | 3.97 ms | +| Selection | 18,682 | 4.42 ms | +| Fresh verification inside selection | 18,682 | 1.51 ms | +| Authority CAS inside selection | 18,682 | 2.55 ms | +| Root materialization | 12,397 | 17.70 ms | +| Checkpoint notification wait | 12,395 | 32.64 ms | + +Every completed phase in this window reports zero failure/cancellation. +Nested and overlapping durations cannot be summed into request latency. Serial +fallback counts are 12,525 for foreground headroom, 6,127 for checkpoints and +31 for capacity; fallback preserves progress rather than rejecting the request. + +Raw artifacts are outside Git under +`cellule-publication-progress-20261011-memory-v6`, case +`cellule-fleet-parity-goal-memory-v6-internal-r1`. Build/source manifests, +request journals, minute snapshots, independent audits, process exits and +`internal-publication-analysis.json` are retained together. + +## Repeat after joining activation lifetimes + +Source `03d2eba21c64105841c5ff0d2cb671c4148edf53`, with SQL binary SHA-256 +`902ed4cd327ff514f79f5e6a47c2798a75a6894a5245015625823f79fb2fac70`. +The same five-minute workload and limits pass again after the authority activation +callback moves outside proof-waiter lifetimes. All eight framework packages were +cleaned for this fresh binary; its source manifest has no Rust/Cargo differences +from the committed implementation. + +| Measurement | Result | +| --- | ---: | +| Planned/generated/completed writes | 150,000 / 150,000 / 150,000 | +| Completed within the window | 149,990 (499.97/s) | +| Errors / queue drops / unissued offers | 0 / 0 / 0 | +| Warmup errors / drops | 0 / 0 | +| Scheduled latency p50 / p95 / p99 | 6.3 / 9.7 / 16.0 ms | +| Request latency p50 / p95 / p99 | 5.5 / 8.6 / 13.5 ms | +| All-ACK warm/cold readback and original retry checks | 167,001 each; zero failures | +| Joined fleet drain | 39.25 s | +| Cold startup | 9.73 s | + +Ten writes finish just after the window. Minute snapshots show zero admission +refusals, root age 42.0–48.9 seconds and 1,858–1,954 pending roots; the end age is +46.015 seconds. Retained reservations start at 28,399,209 bytes and end at +39,737,580 bytes, including root materialization in progress. Bucket frame lag +ranges from 6 to 54 in these samples, independently of root age. + +Completed selection averages 3.94 ms across 21,272 rounds; preparation 4.20 ms, +upload 3.44 ms, materialization 15.52 ms and checkpoint wait 29.16 ms. These are +nested/overlapping phase observations, not an additive response-time model or +an attributable speedup comparison. Host disk headroom decreased under swap +during the run. The next workload was held until the completed task-owned VM +could be stopped and its disks copied and verified on Workspace. + +Artifacts are under `cellule-publication-progress-20261011-activation-v7`, case +`cellule-fleet-parity-goal-activation-v7-corrected-r1`; +`acceptance-publication-analysis.json` pins the raw summary and snapshot hashes. +The first v7 attempt stopped before timed work because the copied harness +omitted its unchanged seed helper. Its logs and provider volume remain retained; +the corrected run uses the v6 helper byte-for-byte. + +## Fresh mixed and read-only windows + +The same v7 binary runs after the completed task-owned VM is stopped and both +images are copied byte-for-byte onto Workspace. `vm-relocation.json` records +matching pre-restart source/destination SHA-256 hashes, actual paths and the +unchanged 12-vCPU/24-GiB VM. Original provider volumes remain retained in that +image; the new case uses a fresh provider volume. The primary Colima profile is +untouched. This physical move occurred after the write-only repeat; it must not +be confused with the earlier ignored `COLIMA_HOME` override. + +Each window has a 30-second warmup and 60-second measurement with the same +2,000 Cells, SQL ledger, 64-MiB budget and 128/128 client limits. + +| Window | Offered/s | Successful/s in window | Returned errors | Dropped offers | Request p50 / p95 / p99 | Scheduled p50 / p95 / p99 | +| --- | ---: | ---: | ---: | ---: | --- | --- | +| Mixed writes | 2,000 | 971.25 | 53,037 | 8,662 | 6.2 / 18.5 / 36.6 ms | 7.7 / 20.9 / 39.3 ms | +| Mixed reads | 20,000 | 18,529.95 | 0 | 88,152 | 0.6 / 3.6 / 9.5 ms | 2.0 / 6.5 / 12.7 ms | +| Read-only reads | 20,000 | 15,806.40 | 0 | 251,484 | 0.5 / 4.2 / 17.0 ms | 1.9 / 10.8 / 38.5 ms | + +Every planned offer is generated: 120,000 mixed writes and 1,200,000 reads per +window. Mixed completions including in-flight drain are 58,301 writes and +1,111,848 reads; read-only completions are 948,516. Warmup mixed writes have +23,837 errors and 5,903 drops, mixed reads 61,229 drops, and read-only reads +54,460 drops. Warmup read errors are zero. These windows fail delivery at the +offered targets; successful-request latency alone does not establish capacity. + +All 90,562 ACKs, including setup/warmup/probes, pass both warm and cold readback +and original-request retry checks with zero errors. Joined fleet drain takes +38.77 seconds and cold startup 7.48 seconds. Correct ACK recovery and sustained +offered throughput remain separate observations. + +### Where the mixed window stalls + +Window telemetry classifies 53,035 new write failures as publication-backlog +refusals, with no new failures in the other finite classes. Client counts include +the final in-flight completions beyond the snapshot boundary. Retained +reservations stay near the 48-MiB foreground threshold: + +| Stage | Start | End | +| --- | ---: | ---: | +| Capture | 20,676,012 B | 13,796,358 B | +| Selection | 6,131,764 B | 8,023,906 B | +| Materialization | 455,328 B | 5,319,898 B | +| Publisher | 23,068,672 B | 23,068,672 B | +| Total, including small request/other reservations | 50,331,776 B | 50,244,114 B | + +Oldest root age grows 57.700→88.718 seconds. Issued-minus-bucket-covered frame +lag is 6,154→4,046, while follower proof ends only 21 frames behind issuance at +the end. The publisher completes 1,022 cohorts, including 60,234 captures and +63,892 frames: about 58.9 captures and 62.5 frames per cohort, near the existing +64-frame bound. Of these rounds, 1,019 use serial fallback for checkpoints. +There are two capacity fallbacks and no headroom fallbacks in the measured delta. + +| Completed phase | Mean duration | +| --- | ---: | +| Preparation | 24.52 ms | +| Upload | 22.06 ms | +| Selection, including fresh verification and CAS | 11.13 ms | +| Fresh verification | 4.91 ms | +| Authority CAS | 5.56 ms | +| Root materialization | 31.99 ms | +| Checkpoint notification wait | 100.02 ms | + +All completed phases have zero failure/cancellation. This points to shared +publication and checkpoint service under retained-byte pressure rather than +follower lag. It does not justify raising the budget or removing the checkpoint +ordering guard. The following read-only start/end snapshots show no pending +publication, no capture credit and zero bucket lag. + +The read-only window contains a sampled competing compiler load of 335.1% CPU; +the two mixed-window samples report zero. Read-only request maximum is 2.52 +seconds. The slower read-only result cannot establish a repeatable regression, +maximum throughput or a causal explanation; isolate generator/host stalls from +server execution in a controlled repeat. + +Artifacts: case `cellule-fleet-parity-read-mixed-activation-v7-workspace-r1` +under the v7 directory, with `mixed-publication-analysis.json`, +`read-only-publication-analysis.json` and `host-load-window-analysis.json`. + +## Next capacity work + +1. Compose ready root checkpoints and native captures in one verified canonical + catalog update, or otherwise restore safe overlap. The measured mixed path + falls back to serial publication in 1,019 of 1,022 rounds. Preserve root credit, + exact dependency verification and the authority CAS; retain the current guard + until the replacement protocol and fault tests prove the same ordering. +2. Reduce preparation/verification work and bytes per cohort. Use the new phase + and stage measurements to check actual allocation bounds and repeated catalog + work. Ramp fresh five-minute offers through 750, 1,000, 1,500 and 2,000 writes/s, + requiring zero refusals/drops, stable debt and every-ACK warm/cold checks under + the same 64-MiB budget. Smaller component timings are insufficient. +3. Repeat read and mixed windows on a controlled host, separating client schedule + stalls, synchronous SQL execution and support-role contention. Qualify hot-Cell + fairness and long-query isolation rather than extrapolating uniform warm reads. +4. Extend physical-media, crash/partition and maintenance qualification. The + activation-cancellation regression and process audits cover concrete runtime + lifetimes; tmpfs tests and joined cold restore cannot certify device loss. + +The hosted workspace run also retained two `Fenced` follower-maintenance example +failures. Both exact cases pass all-feature local execution and five repeated +runs. The full isolated public controller suite passes all 382 cases, including +both original failures. The original CI logs remain and the job is retried +without weaker assertions or longer deadlines; a passing repeat does not identify +or fix the intermittent failure's cause. + +## Retained failed attempts + +| Attempt | Outcome | +| --- | --- | +| Working admission v2, 500/s for five minutes | Zero returned errors but 144 drops; root age grew; not accepted | +| Ordered frame reads v3, two five-minute repeats | Errors/drops remained under competing compiler load; both completed all-ACK warm/cold audits and drained within 120 seconds | +| Working admission v5, 500/s for five minutes | 482.84/s; 3,746 errors and 1,398 drops; growing root age; all 161,857 ACKs passed warm/cold checks and drain took 51.92 seconds | +| Earlier profile/provider filesystem failure | Guest-wide I/O errors before timed work; no throughput result; stopped disks and failure artifacts preserved | + +No budget, qualification profile, evidence requirement, fault assertion or +deadline was increased to make these failures pass. Correct readback of ACKs +and sustainable offered throughput are separate requirements. From 1aba65ab62c38e03029d26e7d413435beca0b5a2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 15:36:11 -0700 Subject: [PATCH 10/31] perf(runtime): overlap checkpoint cohorts within bounded shared scratch --- crates/cellule-runtime/src/node/bundle/mod.rs | 38 ++ .../cellule-runtime/src/node/bundle/store.rs | 38 +- .../node/bundle/tests/checkpoint_pipeline.rs | 416 ++++++++++++++++++ .../src/node/bundle/tests/composed.rs | 16 +- .../src/node/bundle/tests/index/encoding.rs | 37 +- .../src/node/bundle/tests/index/pipeline.rs | 11 + .../src/node/bundle/tests/mod.rs | 1 + .../src/node/bundle/tests/pipeline.rs | 47 +- .../src/node/durability/publication/mod.rs | 10 +- .../node/durability/publication/pipeline.rs | 87 ++-- docs/managed-bundle-pipeline.md | 33 +- 11 files changed, 663 insertions(+), 71 deletions(-) create mode 100644 crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs diff --git a/crates/cellule-runtime/src/node/bundle/mod.rs b/crates/cellule-runtime/src/node/bundle/mod.rs index f0ef3284..c40fa51a 100644 --- a/crates/cellule-runtime/src/node/bundle/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/mod.rs @@ -196,6 +196,44 @@ impl StagedNodeBundle { pub const fn head(&self) -> NodeBundleHead { self.head } + + fn retained_metadata_bytes(&self) -> Result { + if self.catalog.index.is_some() { + return Err(Error::Node("staged bundle retains preparation index")); + } + let assignments = self + .assignments + .capacity() + .checked_mul(std::mem::size_of::()) + .ok_or(Error::Capacity("staged bundle metadata"))?; + let fixed = self + .catalog + .bindings + .capacity() + .checked_mul(std::mem::size_of::()) + .and_then(|bytes| bytes.checked_add(assignments)) + .and_then(|bytes| bytes.checked_add(std::mem::size_of::() + 256)) + .ok_or(Error::Capacity("staged bundle metadata"))?; + self.catalog + .bindings + .iter() + .try_fold(fixed, |bytes, binding| { + binding + .locators + .capacity() + .checked_mul(std::mem::size_of::()) + .and_then(|locators| bytes.checked_add(locators)) + .and_then(|bytes| bytes.checked_add(binding.control.retained_heap_bytes())) + .and_then(|bytes| bytes.checked_add(256)) + .ok_or(Error::Capacity("staged bundle metadata")) + }) + } + + pub(crate) fn retained_work_bytes(&self) -> Result { + self.retained_metadata_bytes()? + .checked_add(self.body.len()) + .ok_or(Error::Capacity("staged bundle memory")) + } } /// Selected, dependency-verified coverage of one exact Cell writer. diff --git a/crates/cellule-runtime/src/node/bundle/store.rs b/crates/cellule-runtime/src/node/bundle/store.rs index 4c7451ac..8a88e092 100644 --- a/crates/cellule-runtime/src/node/bundle/store.rs +++ b/crates/cellule-runtime/src/node/bundle/store.rs @@ -33,13 +33,47 @@ impl NodeDirectory { digest, selected_through: catalog.selected_through, }; - Ok(std::sync::Arc::new(StagedNodeBundle { + // Encoding and self-verification consumed the preparation snapshot. + // Successor assembly reads its immutable bytes; live selection needs + // only the new native bindings. Keep neither sibling rows nor their + // duplicate loaded histories across upload/selection lifetimes. + catalog.index = None; + let rows = std::mem::take(&mut catalog.bindings); + let count = rows + .iter() + .filter(|binding| { + binding + .locators + .iter() + .any(|locator| locator.object.is_none()) + }) + .count(); + let mut selected = Vec::with_capacity(count); + for binding in rows { + if binding + .locators + .iter() + .any(|locator| locator.object.is_none()) + { + selected.push(binding); + } + } + catalog.bindings = selected; + let staged = StagedNodeBundle { original, catalog, body, head, assignments, - })) + }; + // A retained proposal is at most one encoded object plus one bounded + // native metadata allocation. Shared preparation/verification scratch + // can account for that concrete lifetime independently of temporary + // decoder/encoder allocations, which have already joined above. + if staged.retained_metadata_bytes()? > MAX_BUNDLE_BYTES as usize { + return Err(Error::Capacity("staged bundle metadata")); + } + Ok(std::sync::Arc::new(staged)) } /// Uploads already-encoded immutable bytes, without selecting authority or diff --git a/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs new file mode 100644 index 00000000..3f9974f4 --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs @@ -0,0 +1,416 @@ +//! A checkpoint-bearing predecessor can overlap a native successor safely. +use super::actor::{Authority, transport}; +use super::composed::HeldSelection; +use super::pipeline::ManagedStore; +use super::receipt_pressure::submission; +use super::*; +use crate::cell::worker::SqlWorkerPool; +use crate::node::durability::NodeDurability; +use crate::node::log_shipper::NodeLogShipper; +use futures_util::poll; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +#[tokio::test] +async fn checkpoint_predecessor_overlaps_successor_and_cold_restores_both_exact_histories() { + checkpoint_case(128 << 20, Outcome::Complete).await; +} + +#[tokio::test] +async fn checkpoint_predecessor_overlaps_with_original_64_mib_budget() { + checkpoint_case(64 << 20, Outcome::Complete).await; +} + +#[tokio::test] +async fn both_cohorts_compose_checkpoints_and_credit_their_own_ordered_selections() { + checkpoint_case(64 << 20, Outcome::SuccessorCheckpoint).await; +} + +#[tokio::test] +async fn failed_checkpoint_predecessor_joins_accepted_successor_and_preserves_upload_source() { + checkpoint_case(64 << 20, Outcome::FailedUpload).await; +} + +#[tokio::test] +async fn fenced_checkpoint_predecessor_joins_uploads_after_shutdown_waiter_cancellation() { + checkpoint_case(64 << 20, Outcome::Fenced).await; +} + +#[tokio::test] +async fn checkpoint_base_missing_after_upload_prevents_both_selections_and_joins_successor() { + checkpoint_case(64 << 20, Outcome::MissingRoot).await; +} + +#[derive(Clone, Copy)] +enum Outcome { + Complete, + SuccessorCheckpoint, + FailedUpload, + Fenced, + MissingRoot, +} + +async fn checkpoint_case(retained: usize, outcome: Outcome) { + let store = Arc::new(ManagedStore::default()); + let mut f = Fixture::with_store(store.clone()).await; + super::coverage::enroll(&mut f).await; + let mut cells = [f.cell(4).await, f.cell(5).await]; + let original = Arc::new(Authority { + directory: f.directory.clone(), + observed: tokio::sync::Mutex::new(f.node.clone()), + }); + let authority = Arc::new(HeldSelection { + original: original.clone(), + calls: AtomicUsize::new(0), + combined: AtomicUsize::new(0), + largest_combined: AtomicUsize::new(0), + hold_after: 2, + entered: tokio::sync::Notify::new(), + resume: tokio::sync::Notify::new(), + }); + let peers = transport(&f, true); + let shipper = NodeLogShipper::new(f.gate.clone(), peers.clone(), Limits::default()).unwrap(); + let durability = Arc::new(NodeDurability::new( + f.gate.clone(), + shipper, + original, + peers, + f.lease.clone(), + )); + let pool = SqlWorkerPool::new(1, 1).unwrap(); + pool.configure_retained_capacity(retained).unwrap(); + durability + .attach_selection_resources(pool.resource_ledger()) + .unwrap(); + durability + .start_bundle_publication(authority.clone()) + .unwrap(); + + let seed = durability + .submit_capture(submission(&mut cells[0], 2)) + .await + .unwrap(); + let seed_selected = seed.selection.as_ref().unwrap().selected().await.unwrap(); + let root = f + .publisher(&cells[0]) + .materialize_bundle(&seed_selected.proof) + .await + .unwrap(); + // Joining the original catalog lock proves the seed round has returned; + // a ready receipt alone can precede the round's final cleanup. + drop(authority.original.observed.lock().await); + store.armed.store(true, Ordering::Release); + let filler = durability + .submit_capture(submission(&mut cells[1], 2)) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), store.entered.notified()) + .await + .unwrap(); + // Fill the prior round's successor before queueing the checkpoint. Without + // this barrier the checkpoint can become that successor, and this test + // would measure the intentional two-cohort bound instead of its guard. + let prior_successor = durability + .submit_capture(submission(&mut cells[1], 3)) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), store.second_done.notified()) + .await + .unwrap(); + store.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(3), authority.entered.notified()) + .await + .unwrap(); + let successor_checkpoint = matches!(outcome, Outcome::SuccessorCheckpoint); + let second_base = if successor_checkpoint { + let selected = filler.selection.as_ref().unwrap().selected().await.unwrap(); + let root = f + .publisher(&cells[1]) + .materialize_bundle(&selected.proof) + .await + .unwrap(); + Some((root, selected)) + } else { + None + }; + let first = durability + .submit_capture(submission(&mut cells[0], 3)) + .await + .unwrap(); + let mut checkpoint = Box::pin(tokio::task::unconstrained( + durability.checkpoint_materialized(cells[0].authority.clone(), root, seed_selected.clone()), + )); + assert!(poll!(checkpoint.as_mut()).is_pending()); + + // Queue the original callback and native work behind the prior selection; + // the held PUT therefore belongs to their combined catalog proposal. + // Both prior PUTs joined before resetting this task-local fault counter. + store.puts.store(0, Ordering::Release); + authority.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(3), store.entered.notified()) + .await + .unwrap(); + let fault = matches!( + outcome, + Outcome::FailedUpload | Outcome::Fenced | Outcome::MissingRoot + ); + store + .hold_second + .store(fault || successor_checkpoint, Ordering::Release); + store + .fail_first + .store(matches!(outcome, Outcome::FailedUpload), Ordering::Release); + let mut second_checkpoint = second_base.as_ref().map(|(root, selected)| { + Box::pin(tokio::task::unconstrained( + durability.checkpoint_materialized(cells[1].authority.clone(), *root, selected.clone()), + )) + }); + if let Some(callback) = &mut second_checkpoint { + assert!(poll!(callback.as_mut()).is_pending()); + } + let second = durability + .submit_capture(submission(&mut cells[1], 4)) + .await + .unwrap(); + let successor_entered = if fault || successor_checkpoint { + &store.second_entered + } else { + &store.second_done + }; + let overlap = tokio::time::timeout(Duration::from_secs(2), successor_entered.notified()) + .await + .is_ok(); + assert!(!first.selection.as_ref().unwrap().is_ready()); + assert!(!second.selection.as_ref().unwrap().is_ready()); + assert!( + poll!(checkpoint.as_mut()).is_pending(), + "upload supplies no checkpoint credit" + ); + assert_eq!( + durability.progress().unwrap().tiered_through, + prior_successor.assignment.ticket().last_sequence() + ); + if fault { + let prior_head = f + .directory + .load(SessionId::from_bytes([1; 16]), NOW) + .await + .unwrap() + .unwrap() + .advertisement() + .bundle_head(); + if matches!(outcome, Outcome::Fenced) { + f.lease.fence(); + } + let root_path = f.layout.incarnation_object_path( + &root.cell, + &root.incarnation, + &root.digest, + cellule_ltx::CellObjectKind::Root, + ); + if matches!(outcome, Outcome::MissingRoot) { + f.count.block_body_reads_for(&root_path); + } + let mut joined = Box::pin(durability.shutdown()); + assert!(poll!(joined.as_mut()).is_pending()); + store.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(3), f.lease.wait_fenced()) + .await + .unwrap(); + let checkpoint_error = tokio::time::timeout(Duration::from_secs(3), checkpoint.as_mut()) + .await + .unwrap() + .unwrap_err(); + if matches!(outcome, Outcome::FailedUpload) { + assert!(format!("{checkpoint_error:?}").contains("first bundle PUT failed")); + } + assert!( + poll!(joined.as_mut()).is_pending(), + "accepted successor PUT was abandoned" + ); + assert_eq!( + durability.progress().unwrap().tiered_through, + prior_successor.assignment.ticket().last_sequence() + ); + assert_eq!( + f.directory + .load(SessionId::from_bytes([1; 16]), NOW) + .await + .unwrap() + .unwrap() + .advertisement() + .bundle_head(), + prior_head + ); + drop(joined); + store.second_resume.notify_one(); + let error = tokio::time::timeout(Duration::from_secs(3), durability.shutdown()) + .await + .unwrap() + .unwrap_err(); + if matches!(outcome, Outcome::FailedUpload) { + assert!(format!("{error:?}").contains("first bundle PUT failed")); + } + if matches!(outcome, Outcome::MissingRoot) { + f.count.unblock_body_reads_for(&root_path); + } + drop(checkpoint); + drop(seed_selected); + drop(seed); + drop(filler); + drop(prior_successor); + drop(first); + drop(second); + assert_eq!( + pool.resource_ledger() + .snapshot() + .unwrap() + .used + .retained_bytes(), + 0 + ); + pool.shutdown().await.unwrap(); + assert!(overlap, "fault did not exercise both accepted uploads"); + return; + } + store.resume.notify_one(); + let (checkpoint_result, first_selected, second_selected) = if successor_checkpoint { + let (checkpoint_result, first_selected) = + tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!( + checkpoint.as_mut(), + first.selection.as_ref().unwrap().selected() + ) + }) + .await + .unwrap(); + checkpoint_result.unwrap(); + assert!(poll!(second_checkpoint.as_mut().unwrap().as_mut()).is_pending()); + assert!(!second.selection.as_ref().unwrap().is_ready()); + assert_eq!( + durability.progress().unwrap().tiered_through, + first.assignment.ticket().last_sequence() + ); + store.second_resume.notify_one(); + let (checkpoint_result, second_selected) = + tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!( + second_checkpoint.as_mut().unwrap(), + second.selection.as_ref().unwrap().selected() + ) + }) + .await + .unwrap(); + (checkpoint_result, first_selected, second_selected) + } else { + tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!( + checkpoint.as_mut(), + first.selection.as_ref().unwrap().selected(), + second.selection.as_ref().unwrap().selected() + ) + }) + .await + .unwrap() + }; + checkpoint_result.unwrap(); + let selected = [first_selected.unwrap(), second_selected.unwrap()]; + assert_eq!( + authority.combined.load(Ordering::Acquire), + if successor_checkpoint { 2 } else { 1 } + ); + assert_eq!(selected[0].proof.base().unwrap(), root); + if let Some((root, _)) = &second_base { + assert_eq!(selected[1].proof.base().unwrap(), *root); + } + + for (index, (cell, selected)) in cells.iter_mut().zip(&selected).enumerate() { + let overlay = selected + .proof + .recovery_overlay(&f.layout, Limits::default()) + .await + .unwrap(); + let restored = cell + .replica + .prepare_recovered_overlay(&overlay, 1) + .await + .unwrap(); + let destination = f + .scratch + .path() + .join(format!("checkpoint-pipeline-{index}.sqlite")); + cell.replica + .open_root(&restored.root()) + .await + .unwrap() + .restore(&destination) + .await + .unwrap(); + let database = rusqlite::Connection::open(destination).unwrap(); + let outcomes: Vec<(String, String)> = database + .prepare("SELECT request, result FROM outcomes ORDER BY request") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap(); + let mut expected = vec![ + ("request-2".into(), "result-2".into()), + ("request-3".into(), "result-3".into()), + ]; + if index == 1 { + expected.push(("request-4".into(), "result-4".into())); + } + expected.push(("seed".into(), "original".into())); + assert_eq!(outcomes, expected); + cell.control = cell + .authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap(); + let root = f + .publisher(cell) + .materialize_bundle(&selected.proof) + .await + .unwrap(); + durability + .checkpoint_materialized(cell.authority.clone(), root, selected.clone()) + .await + .unwrap(); + cell.control = cell + .authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap(); + durability + .close_bundle_cell(&cell.authority, &cell.control) + .await + .unwrap(); + } + durability.shutdown().await.unwrap(); + drop(selected); + drop(seed_selected); + drop(seed); + drop(filler); + drop(prior_successor); + drop(first); + drop(second); + drop(checkpoint); + drop(second_checkpoint); + drop(second_base); + assert_eq!( + pool.resource_ledger() + .snapshot() + .unwrap() + .used + .retained_bytes(), + 0 + ); + pool.shutdown().await.unwrap(); + assert!( + overlap, + "checkpoint-bearing predecessor serialized immutable successor work" + ); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/composed.rs b/crates/cellule-runtime/src/node/bundle/tests/composed.rs index 7438091b..566de604 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/composed.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/composed.rs @@ -11,14 +11,14 @@ use crate::node::log_shipper::NodeLogShipper; use futures_util::{future::BoxFuture, poll}; use std::sync::atomic::{AtomicUsize, Ordering}; -struct HeldSelection { - original: Arc, - calls: AtomicUsize, - combined: AtomicUsize, - largest_combined: AtomicUsize, - hold_after: usize, - entered: tokio::sync::Notify, - resume: tokio::sync::Notify, +pub(super) struct HeldSelection { + pub(super) original: Arc, + pub(super) calls: AtomicUsize, + pub(super) combined: AtomicUsize, + pub(super) largest_combined: AtomicUsize, + pub(super) hold_after: usize, + pub(super) entered: tokio::sync::Notify, + pub(super) resume: tokio::sync::Notify, } impl NodeBundleAuthority for HeldSelection { fn bind<'a>( diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs b/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs index 7a45f480..03f9a2f5 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs @@ -42,6 +42,23 @@ async fn dense_multi_cell_encoding_preserves_the_original_native_order_and_bytes frames.extend(capture); assignments.push(assignment); } + let keys = cells + .iter() + .map(|cell| { + ( + *f.layout.application_id(), + *cell.control.value().cell.as_bytes(), + ) + }) + .collect(); + let mut original_catalog = crate::node::bundle::store::load_catalog_cells( + &f.layout, + f.node.advertisement().session(), + f.node.advertisement().bundle_head().unwrap(), + &keys, + ) + .await + .unwrap(); let proposal = f .directory .prepare_node_bundle(&f.node, &frames, &assignments, now) @@ -52,12 +69,28 @@ async fn dense_multi_cell_encoding_preserves_the_original_native_order_and_bytes // roots include generated identities, so a stored digest is not a // stable cross-run wire fixture. Both encoders must agree byte for byte // over the same 64-Cell cohort and its unrelated catalog siblings. - let mut original_catalog = proposal.catalog.clone(); + // Staging now discards preparation snapshots and unrelated live rows + // after encoding. Compare against the complete original input catalog, + // preserving every sibling and its predecessor shard/history references. + original_catalog.selected_through = proposal.catalog.selected_through; + original_catalog.predecessor = proposal.catalog.predecessor; + for binding in &proposal.catalog.bindings { + *original_catalog + .binding_mut(binding.control.bundle_binding.unwrap().digest) + .unwrap() = binding.clone(); + } let (original_body, original_digest) = catalog_index::encode_original(&mut original_catalog, &frames).unwrap(); assert_eq!(proposal.body, original_body); assert_eq!(proposal.head.digest, original_digest); - assert_eq!(proposal.catalog, original_catalog); + for binding in &proposal.catalog.bindings { + assert_eq!( + binding, + original_catalog + .binding_mut(binding.control.bundle_binding.unwrap().digest) + .unwrap() + ); + } let mut next_offset = None; for frame in &frames { let locators: Vec<_> = proposal diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs index bcd4bea6..6b33bf47 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs @@ -67,6 +67,17 @@ async fn successor_stages_before_upload_but_requires_ordered_selection_and_full_ first.head.epoch, first.head.digest.as_bytes(), ); + assert!( + first.catalog.index.is_none(), + "encoded proposal retained duplicate preparation histories" + ); + assert_eq!( + first.catalog.bindings.len(), + 64, + "staged metadata retains unrelated inventory rows" + ); + assert_eq!(first.catalog.bindings.capacity(), 64); + assert!(first.retained_work_bytes().unwrap() <= 2 * MAX_BUNDLE_BYTES as usize); let mut second_frames = Vec::new(); let mut second_assignments = Vec::new(); for cell in &mut cells { diff --git a/crates/cellule-runtime/src/node/bundle/tests/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/mod.rs index 1e469f1b..60f8664f 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/mod.rs @@ -17,6 +17,7 @@ const NOW: i64 = 1_000_000; const EPOCH: u64 = 2; mod actor; mod checkpoint_credit; +mod checkpoint_pipeline; mod composed; mod coverage; mod faults; diff --git a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs index 9f15dbc4..dfca904b 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs @@ -12,17 +12,17 @@ use futures_util::stream::BoxStream; use object_store::ObjectStore; use std::sync::atomic::{AtomicBool, Ordering}; #[derive(Debug, Default)] -struct ManagedStore { +pub(super) struct ManagedStore { inner: InMemory, - armed: AtomicBool, - puts: std::sync::atomic::AtomicUsize, - hold_second: AtomicBool, - second_entered: tokio::sync::Notify, - second_done: tokio::sync::Notify, - second_resume: tokio::sync::Notify, - fail_first: AtomicBool, - entered: tokio::sync::Notify, - resume: tokio::sync::Notify, + pub(super) armed: AtomicBool, + pub(super) puts: std::sync::atomic::AtomicUsize, + pub(super) hold_second: AtomicBool, + pub(super) second_entered: tokio::sync::Notify, + pub(super) second_done: tokio::sync::Notify, + pub(super) second_resume: tokio::sync::Notify, + pub(super) fail_first: AtomicBool, + pub(super) entered: tokio::sync::Notify, + pub(super) resume: tokio::sync::Notify, } impl std::fmt::Display for ManagedStore { @@ -335,7 +335,7 @@ async fn optional_overlap_preserves_foreground_headroom_with_live_selected_debt( .resource_ledger() .try_reserve_in( RetentionStage::Selection, - ResourceCost::zero().with_retained_bytes(8 << 20), + ResourceCost::zero().with_retained_bytes(14 << 20), ) .unwrap(); let first = case.first().await; @@ -361,6 +361,31 @@ async fn optional_overlap_preserves_foreground_headroom_with_live_selected_debt( case.finish([first, second]).await; } +#[tokio::test] +async fn compact_pipeline_overlaps_with_selected_debt_without_consuming_foreground_headroom() { + use crate::fleet::resource::{ResourceCost, RetentionStage}; + let mut case = Case::new(64 << 20, false, false).await; + let selected_debt = case + .pool + .resource_ledger() + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(8 << 20), + ) + .unwrap(); + let first = case.first().await; + let second = case.second().await; + tokio::time::timeout(Duration::from_secs(2), case.store.second_done.notified()) + .await + .unwrap(); + let snapshot = case.pool.resource_ledger().snapshot().unwrap(); + assert!(snapshot.used.retained_bytes() < snapshot.limit.retained_bytes() * 3 / 4); + assert_eq!(case.durability.progress().unwrap().tiered_through, 0); + case.store.resume.notify_one(); + drop(selected_debt); + case.finish([first, second]).await; +} + #[tokio::test] async fn fenced_join_waits_for_both_accepted_uploads_after_caller_cancellation() { failed_join(false).await; diff --git a/crates/cellule-runtime/src/node/durability/publication/mod.rs b/crates/cellule-runtime/src/node/durability/publication/mod.rs index ff6b3c89..513172e5 100644 --- a/crates/cellule-runtime/src/node/durability/publication/mod.rs +++ b/crates/cellule-runtime/src/node/durability/publication/mod.rs @@ -150,10 +150,14 @@ impl Publisher { // Cell departure is allowed only after this original callback joins, // not after enqueue. Otherwise closing/detaching its pin could overtake // a still-running catalog checkpoint from the accepted root task. - tokio::select! { - result = completion => result.map_err(|_| Error::RuntimeClosed)?.map_err(Error::Shared), - () = self.lease.wait_fenced() => self.terminal_error(), + // Admission transferred the obligation to the original producer. + // Fencing cannot abandon that callback or replace its source error + // while another accepted upload still delays the worker's termination. + match completion.await { + Ok(result) => result.map_err(Error::Shared)?, + Err(_) => return self.terminal_error(), } + self.lease.check() } fn terminal_error(&self) -> Result<()> { diff --git a/crates/cellule-runtime/src/node/durability/publication/pipeline.rs b/crates/cellule-runtime/src/node/durability/publication/pipeline.rs index 91ce4766..02f0a48b 100644 --- a/crates/cellule-runtime/src/node/durability/publication/pipeline.rs +++ b/crates/cellule-runtime/src/node/durability/publication/pipeline.rs @@ -26,12 +26,14 @@ fn reserve( original: &NodeDurability, authority: &dyn NodeBundlePublicationAuthority, captures: usize, + staged: &crate::node::bundle::StagedNodeBundle, ) -> Result { let first = authority.receipt_memory_bound(captures)?; let second = authority.receipt_memory_bound(MAX_CAPTURES)?; + let working = staged.retained_work_bytes()?; let bytes = first .checked_add(second) - .and_then(|bytes| bytes.checked_add((5 * crate::node::bundle::MAX_BUNDLE_BYTES) as usize)) + .and_then(|bytes| bytes.checked_add(working)) .ok_or(Error::Capacity("bundle pipeline memory"))?; let reserved = original .selection_resources @@ -77,7 +79,6 @@ pub(super) async fn publish( ) -> Result> { let telemetry = original.shipper.telemetry(); let first_capture_count = first.captures.len(); - let may_overlap = first.ready_checkpoints.values().is_empty(); let failure = OnceLock::new(); let round = match telemetry .publication_phase(Phase::CatalogWait, authority.begin_round()) @@ -111,6 +112,13 @@ pub(super) async fn publish( }; let (finished, first_finished) = oneshot::channel(); let (receipt, first_receipt) = oneshot::channel(); + // Preserve the original five-buffer working bound by serializing heavy + // preparation and verification. Uploads still overlap. Extra admission + // owns the first compact proposal while the successor uses that scratch. + // During first verification, its three scratch buffers and the at-most + // two-buffer successor fit the original five; the first proposal remains + // covered by this exact additional reservation throughout both cohorts. + let scratch = tokio::sync::Mutex::new(()); let (first_result, successor) = tokio::join!( async { let result = async { @@ -124,10 +132,10 @@ pub(super) async fn publish( .map(|selected| &selected.proof) .collect::>(); telemetry - .publication_phase( - Phase::Selection, - round.select(&uploaded, &borrowed, lease), - ) + .publication_phase(Phase::Selection, async { + let _scratch = scratch.lock().await; + round.select(&uploaded, &borrowed, lease).await + }) .await } .await; @@ -163,11 +171,6 @@ pub(super) async fn publish( }, async { let result = async { - if !may_overlap { - telemetry.node_publication_fallback(Fallback::Checkpoints); - let _ = receipt.send(None); - return Ok(None); - } let next = if let Some(next) = carry.take().or_else(|| feed.try_recv()) { Some(next) } else { @@ -183,29 +186,33 @@ pub(super) async fn publish( let _ = receipt.send(None); return Ok(None); }; - let mut memory = match reserve(original, authority.as_ref(), first_capture_count) { - Ok(memory) => memory, - Err(Error::Capacity("bundle pipeline headroom")) => { - telemetry.node_publication_fallback(Fallback::Headroom); - *carry = Some(next); - let _ = receipt.send(None); - return Ok(None); - } - Err(Error::Capacity(_)) => { - telemetry.node_publication_fallback(Fallback::Capacity); - *carry = Some(next); - let _ = receipt.send(None); - return Ok(None); - } - Err(error) => return Err(error), - }; + let mut memory = + match reserve(original, authority.as_ref(), first_capture_count, &staged) { + Ok(memory) => memory, + Err(Error::Capacity("bundle pipeline headroom")) => { + telemetry.node_publication_fallback(Fallback::Headroom); + *carry = Some(next); + let _ = receipt.send(None); + return Ok(None); + } + Err(Error::Capacity(_)) => { + telemetry.node_publication_fallback(Fallback::Capacity); + *carry = Some(next); + let _ = receipt.send(None); + return Ok(None); + } + Err(error) => return Err(error), + }; // This is a nonblocking admission under ordering. It either // prepaid both receipt lifetimes or returns to the serial path. let first_memory = memory.reserved.split_retained(memory.first)?; let second_memory = memory.reserved.split_retained(memory.second)?; let _ = receipt.send(Some(first_memory)); - // No standalone checkpoint is dispatched while this round owns - // ordering; the bounded gather includes only available row slots. + // The staged predecessor already carries its verified new + // bases. A successor inherits them and may compose further + // checkpoints in its available rows; neither callback gains + // credit until that cohort's ordered authoritative selection. + // No standalone checkpoint is dispatched under round ordering. let second = assemble( authority, feed, @@ -213,22 +220,24 @@ pub(super) async fn publish( checkpoints, next, None, - false, + true, lease, telemetry, ) .await?; let upload = async { let next = telemetry - .publication_phase( - Phase::Preparation, - round.stage( - Some(&staged), - &second.captures, - second.ready_checkpoints.values(), - Some(preparation), - ), - ) + .publication_phase(Phase::Preparation, async { + let _scratch = scratch.lock().await; + round + .stage( + Some(&staged), + &second.captures, + second.ready_checkpoints.values(), + Some(preparation), + ) + .await + }) .await?; lease.check()?; telemetry diff --git a/docs/managed-bundle-pipeline.md b/docs/managed-bundle-pipeline.md index 09264286..b00448e9 100644 --- a/docs/managed-bundle-pipeline.md +++ b/docs/managed-bundle-pipeline.md @@ -25,7 +25,7 @@ The manual `select` method uses these same three phases. | Event | Managed behavior | | --- | --- | | First cohort starts | Keep the existing 20-MiB working reservation and 2-MiB shard cache | -| Another capture arrives during publication | Try to admit another 20 MiB plus conservative receipt bounds for both cohorts on the original ledger, preserving publication progress and foreground burst headroom | +| Another capture arrives during publication | Admit the first proposal's encoded body and retained native metadata plus conservative receipt bounds for both cohorts; share the original preparation/verification scratch while preserving foreground burst headroom | | Extra credit unavailable | Keep that capture for the next serial round; do not wait while holding catalog ordering | | Successor PUT finishes first | Retain uploaded bytes; grant no coverage or response credit | | First selection succeeds | Confirm its original receipts and release its native capture credit independently of the second PUT | @@ -60,11 +60,21 @@ When root I/O is already alive and even the minimum working reservation cannot fit, dispatch skips the repeated dirty-fleet walk and sort. This is only a scheduling hint; the original atomic admission still decides each job. -A first cohort containing root checkpoint notifications uses serial publication. -The lookahead cohort contains native captures only. Ready root notifications -join the next first cohort or the existing idle checkpoint path. This preserves -the original root-credit release path and bounds callback retention during a -held upload. +Both cohorts may combine ready root checkpoints and native captures within the +original 64-row bound. A successor inherits its predecessor's encoded checkpoint +bases. Each callback receives credit only after its own fresh verification and +ordered authority CAS; an uploaded successor cannot credit either cohort early. +Standalone checkpoints never run while that round holds catalog ordering. + +After encoding and self-verification, a staged proposal retains its immutable +body and only the native bindings needed for selection. Loaded shard/history +snapshots and unrelated bindings are released; the complete wire bytes remain +unchanged. Staged metadata is bounded to 4 MiB and its capacities are charged +with the body under the optional overlap reservation. Heavy successor preparation +and predecessor verification share one scoped scratch lane. Uploads may overlap, +while the existing five-buffer working reservation, 64-MiB budget and foreground +headroom threshold remain unchanged. Retained receipts keep their separate +conservative admission. When extra receipt credit is unavailable, the round returns catalog ordering before the existing receipt-admission wait. That wait continues to service @@ -75,6 +85,9 @@ Managed failure, fencing and shutdown do not cancel already-dispatched uploads or checkpoint I/O. Cancelling a shutdown caller leaves the original worker alive; a later shutdown joins that worker and observes its terminal result. Manual callers continue to own their admission and accepted I/O lifetimes. +An accepted checkpoint waiter also joins its original callback. Fencing cannot +replace that callback's source error while a sibling upload delays the worker's +terminal status; a successful callback still checks the original lease. The first follower proof also owns a node authority activation obligation. One epoch-owned driver coalesces activation waiters; cancellation or an Object @@ -100,6 +113,14 @@ maximum-history and full-cohort fixtures check the conservative receipt bound. The real example authority also exercises two staged cohorts, heartbeat renewal, blocked binding and rejection of out-of-order selection. +Checkpoint regressions hold a checkpoint-bearing first upload and observe the +successor upload under the unchanged 64-MiB budget. A second case composes +checkpoints in both cohorts, releases the first callback while the second upload +is held, and cold-restores both exact histories. Upload failure, fencing, +cancelled shutdown and a missing checkpoint base preserve source errors, refuse +early credit and join every accepted upload. A selected-debt fixture verifies +overlap without consuming foreground burst headroom. + A longer-stall regression holds the first PUT across 31 seconds and renews the writer during that wait. Its initial implementation rejects successor staging because the advertisement saved at round entry has expired. The round now pins From da8887d87f52ab4d3411b28ae81ed108e803f61e Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 15:36:11 -0700 Subject: [PATCH 11/31] fix(host): retain same ensemble across monotone evacuation observations --- crates/cellule-host/docs/lifecycle.md | 6 + .../scenario/follower_maintenance/mod.rs | 24 ++- .../scenario/follower_maintenance/setup.rs | 6 +- .../scenario/follower_tests/authority_race.rs | 154 ++++++++++++++++++ .../scenario/follower_tests/evacuation.rs | 2 +- .../follower_tests/fixture/managed.rs | 10 +- .../minion/scenario/follower_tests/mod.rs | 1 + crates/cellule-host/minion/scenario/mod.rs | 10 +- .../src/durability/enrollment/maintenance.rs | 33 +++- docs/publication-progress-measurements.md | 26 +++ 10 files changed, 258 insertions(+), 14 deletions(-) create mode 100644 crates/cellule-host/minion/scenario/follower_tests/authority_race.rs diff --git a/crates/cellule-host/docs/lifecycle.md b/crates/cellule-host/docs/lifecycle.md index e2f1aed8..599b9f55 100644 --- a/crates/cellule-host/docs/lifecycle.md +++ b/crates/cellule-host/docs/lifecycle.md @@ -215,6 +215,12 @@ expired operation or incomplete enrollment produces no settlement evidence. Complete foreign/native inventory, Pending producers, failed-owner recovery and terminal action handoff remain required before physical-node shutdown. +Directory rechecks accept heartbeat, follower activation and covered-prefix +advancement for the same signed boot and Open ensemble. Generation, issue time, +lease expiry, progress, activation and coverage cannot regress; epoch and members +remain exact. The capture retains its final observation. Closure, replacement or +authority loss still rejects the interval, including changes after a held read. + ### Persist and revalidate follower replacement evidence `FollowerReplacementPolicy` gives the application's nonzero member minimum a diff --git a/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs b/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs index 1d5e411f..9701caef 100644 --- a/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs @@ -49,7 +49,9 @@ pub(super) async fn run( // These role-enabled futures contain native peer activation and complete // fleet capture. Their synchronous boxed constructors keep construction // temporaries out of the parent poll frame as well as its retained state. - let inputs = setup::initialize(root, &journal, nodes, boots, roles).await?; + let inputs = setup::initialize(root, &journal, nodes, boots, roles) + .await + .map_err(|source| failure("follower-maintenance-initialize", source))?; let version = journal.load_snapshot(scope()).await?.registry(); let page = journal.enrollments_page(version, None, 128).await?; let original = page @@ -236,7 +238,9 @@ pub(super) async fn run( "follower rotation lacks original tail coverage or the newer epoch", )); } - let renewed = service::resume(root, nodes, boots, &inputs).await?; + let renewed = service::resume(root, nodes, boots, &inputs) + .await + .map_err(|source| failure("follower-maintenance-resume", source))?; let current = journal.load_snapshot(scope()).await?; let policy = FollowerReplacementPolicy::new(scope(), 1, 2)?; journal @@ -249,7 +253,8 @@ pub(super) async fn run( .ok_or_else(|| invalid("follower maintenance operation disappeared"))?; let capture = nodes[0] .follower_evacuation(&original, operation, 2, deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-capture", source))?; if !Arc::ptr_eq(capture.rotation(), &completion) || capture.retired_members().len() != 2 || capture @@ -278,7 +283,8 @@ pub(super) async fn run( deadline(), clock, ) - .await?; + .await + .map_err(|source| failure("follower-maintenance-publication", source))?; let record = publication.record()?; let reader_capture = if let Some(readers) = &inputs.readers { // A follower proof alone cannot close a donor that also owns a reader. @@ -401,6 +407,16 @@ fn checked_report(report: &cellule_host::fleet::FleetReconcileReport) -> Journal Ok(()) } +// A finite scenario must keep the failing boundary and the original typed +// source together. Repeated Fenced errors without a phase cannot distinguish +// native rotation, resumed service and fresh policy verification. +pub(super) fn failure(name: &'static str, source: impl Into) -> JournalError { + Box::new(cellule_runtime::Error::Facility { + name, + source: source.into(), + }) +} + // Preserve the supervisor's shared source error when this finite command exits. #[derive(Debug)] struct RotationFailure(Arc); diff --git a/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs b/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs index 822d1952..38956581 100644 --- a/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs +++ b/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs @@ -200,7 +200,8 @@ pub(super) fn initialize<'a>( .await?; boots[0] .refresh_capacity(0, journal.as_ref(), deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-leader-refresh", source))?; let handle = nodes[0] .runtime() .bootstrap( @@ -216,7 +217,8 @@ pub(super) fn initialize<'a>( Ok(()) }, ) - .await?; + .await + .map_err(|source| failure("follower-maintenance-bootstrap", source))?; let now = clock()?; let identity = MutationIdentity { request_id: RequestId::from_bytes([1; 16]), diff --git a/crates/cellule-host/minion/scenario/follower_tests/authority_race.rs b/crates/cellule-host/minion/scenario/follower_tests/authority_race.rs new file mode 100644 index 00000000..6632b346 --- /dev/null +++ b/crates/cellule-host/minion/scenario/follower_tests/authority_race.rs @@ -0,0 +1,154 @@ +//! Capture must keep its exact ensemble across a legitimate boot heartbeat. +use super::*; +use futures_util::stream::BoxStream; +use object_store::{ObjectStore, path::Path}; + +#[derive(Debug, Default)] +struct HeldNodeRead { + inner: InMemory, + path: Mutex>, + entered: tokio::sync::Notify, + resume: tokio::sync::Notify, +} +impl std::fmt::Display for HeldNodeRead { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HeldNodeRead") + } +} +#[async_trait::async_trait] +impl ObjectStore for HeldNodeRead { + async fn put_opts( + &self, + path: &Path, + payload: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + async fn get_opts( + &self, + path: &Path, + options: object_store::GetOptions, + ) -> object_store::Result { + let result = self.inner.get_opts(path, options).await; + let held = { + let mut target = self.path.lock().unwrap(); + if target.as_ref() == Some(path) { + target.take(); + true + } else { + false + } + }; + if held { + // The provider has captured the old exact record. Publish the real + // heartbeat before delivering it, then let capture reobserve it. + self.entered.notify_one(); + self.resume.notified().await; + } + result + } + async fn put_multipart_opts( + &self, + path: &Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn managed_follower_evacuation_preserves_replacement_across_captured_heartbeat() { + captured_change(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn managed_follower_evacuation_rejects_closed_replacement_after_captured_read() { + captured_change(true).await; +} + +async fn captured_change(close: bool) { + let store = Arc::new(HeldNodeRead::default()); + let fixture = ManagedFixture::with_store(4, 2, store.clone()).await; + let (original, operation) = evacuation::maintenance(&fixture).await; + evacuation::spare(&fixture).await; + evacuation::rotated(&fixture).await; + *store.path.lock().unwrap() = Some(fixture.native.layout.node_path(session(0).as_bytes())); + let (result, ()) = tokio::join!( + evacuation::evidence(&fixture, &original, &operation), + async { + tokio::time::timeout(Duration::from_secs(3), store.entered.notified()) + .await + .unwrap(); + if close { + fixture + .native + .node + .runtime() + .node_durability() + .unwrap() + .1 + .shutdown() + .await + .unwrap(); + } else { + fixture.boots[0] + .refresh_capacity( + 0, + fixture.native.journal.as_ref(), + Instant::now() + Duration::from_secs(3), + ) + .await + .unwrap(); + } + store.resume.notify_one(); + } + ); + let message = result.as_ref().err().map(|error| format!("{error:?}")); + let valid = if close { + matches!(result, Err(Error::Fenced)) + } else { + result.as_ref().is_ok_and(|capture| { + capture + .authority() + .log() + .is_some_and(|log| log.epoch() == 2 && log.members() == [node_id(2), node_id(3)]) + }) + }; + drop(result); + fixture.finish().await; + assert!( + valid, + "unexpected captured authority result: close={close} {message:?}" + ); +} diff --git a/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs b/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs index 0c6bb65c..ef1679b9 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs @@ -107,7 +107,7 @@ pub(super) async fn rotated(fixture: &ManagedFixture) -> cellule_host::NodeLogRo request } -async fn evidence( +pub(super) async fn evidence( fixture: &ManagedFixture, original: &EnrollmentRecord, operation: &MaintenanceOperation, diff --git a/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs b/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs index 4319d8d1..db97d897 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs @@ -17,12 +17,20 @@ impl ManagedFixture { } pub async fn with_members(count: usize, max_epochs: usize) -> Self { + Self::with_store(count, max_epochs, Arc::new(InMemory::new())).await + } + + pub async fn with_store( + count: usize, + max_epochs: usize, + store: Arc, + ) -> Self { assert!((3..=4).contains(&count)); assert!((1..=3).contains(&max_epochs)); let root = tempfile::tempdir().unwrap(); let app = application::compile().unwrap(); let layout = CellStorageLayout::new( - Store::new(Arc::new(InMemory::new())), + Store::new(store), ObjectPath::from("managed-followers"), [3; 16], ); diff --git a/crates/cellule-host/minion/scenario/follower_tests/mod.rs b/crates/cellule-host/minion/scenario/follower_tests/mod.rs index 84aad55c..2580cb7d 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/mod.rs @@ -29,6 +29,7 @@ use std::sync::{ mod fixture; use fixture::*; mod aggregate; +mod authority_race; mod coverage; mod evacuation; mod inventory; diff --git a/crates/cellule-host/minion/scenario/mod.rs b/crates/cellule-host/minion/scenario/mod.rs index 0195cff2..b9db2abb 100644 --- a/crates/cellule-host/minion/scenario/mod.rs +++ b/crates/cellule-host/minion/scenario/mod.rs @@ -225,14 +225,20 @@ async fn execute(scenario: Scenario) -> JournalResult { if let Err(error) = node.shutdown().await && cleanup_error.is_none() { - cleanup_error = Some(Box::new(error) as JournalError); + cleanup_error = Some(Box::new(cellule_runtime::Error::Facility { + name: "fleet-example-node-shutdown", + source: Box::new(error), + }) as JournalError); } } for boot in &boots { if let Err(error) = boot.withdraw(&journal).await && cleanup_error.is_none() { - cleanup_error = Some(error); + cleanup_error = Some(Box::new(cellule_runtime::Error::Facility { + name: "fleet-example-boot-withdrawal", + source: error, + }) as JournalError); } } if result.is_ok() && cleanup_error.is_none() { diff --git a/crates/cellule-host/src/durability/enrollment/maintenance.rs b/crates/cellule-host/src/durability/enrollment/maintenance.rs index f1a3dfa3..2f07cd30 100644 --- a/crates/cellule-host/src/durability/enrollment/maintenance.rs +++ b/crates/cellule-host/src/durability/enrollment/maintenance.rs @@ -370,7 +370,7 @@ impl FleetFollowerEnrollment { .inspect_log_enrollment(&replacement.attempt, now_ms()?) .await? .ok_or(Error::Fenced)?; - let authority = enrollment.enrollment().advertisement().clone(); + let mut authority = enrollment.enrollment().advertisement().clone(); let source = roster.boot(self.node, self.session)?; if source.intent().mode() != NodeMode::Active || replacement.members.iter().any(|member| { @@ -412,16 +412,22 @@ impl FleetFollowerEnrollment { .inspect_log_enrollment(&replacement.attempt, now_ms()?) .await? .ok_or(Error::Fenced)?; - if current.enrollment().advertisement() != &authority - || authority + let observed = current.enrollment().advertisement(); + // Heartbeat, activation and covered-prefix publication can advance + // this same ensemble during suspended journal/directory reads. + // Reobserve their monotone progress; never accept a new boot, closed + // epoch or changed membership as the original replacement witness. + if !same_ensemble(&authority, observed)? + || observed .log() .is_none_or(|log| log.phase() != NodeLogPhase::Open) - || authority + || observed .operational_sample() .is_none_or(|sample| sample.mode != NodeMode::Active) { return Err(Error::Fenced); } + authority.clone_from(observed); let (application, durability) = self.runtime.node_durability().ok_or(Error::Fenced)?; if application != self.scope.application || !node.is_ready() @@ -482,6 +488,25 @@ fn same_boot( && current.issued_at_ms() >= original.issued_at_ms()) } +fn same_ensemble( + original: &NodeAdvertisement, + current: &NodeAdvertisement, +) -> cellule_runtime::Result { + Ok(same_boot(original, current)? + && current.expires_at_ms() >= original.expires_at_ms() + && current.progress() >= original.progress() + && current.log().is_some_and(|log| { + original.log().is_some_and(|old| { + old.phase() == NodeLogPhase::Open + && log.phase() == NodeLogPhase::Open + && log.epoch() == old.epoch() + && log.members() == old.members() + && (!old.active() || log.active()) + && log.tiered_through() >= old.tiered_through() + }) + })) +} + /// Immutable identity shared by native capture and durable revalidation. pub(crate) fn boot_identity(node: &NodeAdvertisement) -> cellule_runtime::Result { let mut hash = blake3::Hasher::new(); diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index e041ad73..d368514f 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -234,6 +234,32 @@ both original failures. The original CI logs remain and the job is retried without weaker assertions or longer deadlines; a passing repeat does not identify or fix the intermittent failure's cause. +### Checkpoint-overlap candidate + +The next candidate composes checkpoints in either of its two ordered cohorts, +releases duplicate preparation snapshots after encoding, and shares heavy +preparation/verification scratch. It preserves the existing 64-MiB budget, +foreground headroom, fresh dependency checks and authority CAS. An accepted +checkpoint joins its original callback to preserve upload source errors even +when a sibling upload delays worker termination. + +The focused bundle suite passes 150 cases; the subsequent six-case checkpoint +suite also covers a checkpoint in each cohort and independently ordered credit. +Exact encoding comparisons retain complete sibling catalogs and require +byte/digest equality. These are correctness and bounded-lifetime regressions; +the candidate's fresh application capacity ramp has not yet completed. The +500-write/s and mixed/read-only tables above still describe source `03d2eba`, +not this candidate. See the [managed protocol](managed-bundle-pipeline.md). + +The maintenance investigation also reproduces a distinct capture race: a real +same-boot heartbeat between two directory observations caused `Fenced` through +full-advertisement equality. Capture now retains the exact Open epoch, membership +and signed boot identity while reobserving monotone generation, activation, +coverage and lease progress. Its original heartbeat regression fails before the +change and passes after it; a held-read closure case still rejects. The original +CI failures and 50 passing local repeats remain retained. This concrete fix is +not yet confirmation of the hosted failure's cause or a passing new CI run. + ## Retained failed attempts | Attempt | Outcome | From 952362be3847c65885171e9e8974a77ec0901117 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 15:49:54 -0700 Subject: [PATCH 12/31] test: retain exact retries after device-backed owner loss --- .../routed/recovery_faults/inherited/setup.rs | 7 ++++-- .../lifecycle/ownership/sparse_process.rs | 25 +++++++++++++++++++ docs/publication-progress-measurements.md | 16 ++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs b/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs index c37de3e9..08bcd6ba 100644 --- a/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs +++ b/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs @@ -85,6 +85,9 @@ impl Inherited { ); capacity.follower_free_bytes = capacity.free_disk_bytes.min(64 << 20); capacity.log_protocol = cellule_runtime::node::NODE_LOG_PROTOCOL_VERSION; + // Derive expiry from the same instant: a second clock read can + // cross a millisecond boundary and exceed the maximum lifetime. + let issued_at = clock().unwrap(); let next = NodeAdvertisement::sign( ad.node(), ad.session(), @@ -95,8 +98,8 @@ impl Inherited { ad.release(), &ed25519_dalek::SigningKey::from_bytes(&[index as u8 + 1; 32]), ad.progress(), - clock().unwrap(), - clock().unwrap() + 30_000, + issued_at, + issued_at + 30_000, ad.module_digests().to_vec(), ad.peer_versions().to_vec(), ad.failure_domain().clone(), diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs index 861aa92f..2ea2646b 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs @@ -258,6 +258,31 @@ async fn rustfs_process_killed_sparse_owner_restores_published_root_and_continue ) .await .unwrap(); + // The killed owner acknowledged this exact mutation. Recovery must retain + // both its state and request outcome: an original retry must never execute + // the mutation again merely because the local writer files disappeared. + let retried = successor + .execute( + mutation_identity_window(67, 10, 10_000), + Digest::from_bytes([68; 32]), + 21, + 1_024, + 1_024, + |_| { + Err(cellule_runtime::Error::Control( + "acknowledged retry executed after owner kill", + )) + }, + ) + .await + .unwrap(); + assert_eq!( + retried, + StoredOutcome::Success { + result: b"published".to_vec(), + commit_sequence: 1, + } + ); let outcome = successor .execute( mutation_identity_window(71, 10, 10_000), diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index d368514f..93d3d29e 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -260,6 +260,22 @@ change and passes after it; a held-read closure case still rejects. The original CI failures and 50 passing local repeats remain retained. This concrete fix is not yet confirmation of the hosted failure's cause or a passing new CI run. +An isolated controller repeat passed 383 of 384 cases. The remaining failure +was fixture construction: separate clock reads could sign a 30,001-ms +advertisement against the 30,000-ms maximum. Deriving both timestamps from one +reading preserves the lifetime contract; all three inherited-recovery cases +then pass with their original assertions. + +The device-backed sparse-owner process test also passes against fresh ext4 +SQLite and RustFS volumes. After the child acknowledges one mutation, the test +kills its process, deletes its local writer files, restores the authoritative +root, and retries the original request with a handler that fails if invoked. +The retry returns the original result and sequence; the next mutation verifies +the recovered state. This qualifies that published-root process-loss path, +not unpublished follower tails, network partitions or physical power loss. +The initial missing-import build failure and corrected passing run are both +retained under the v9 artifact directory. + ## Retained failed attempts | Attempt | Outcome | From 0b1d7f3a691d13334229f1cc58e7593a3aa3297f Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 16:19:42 -0700 Subject: [PATCH 13/31] test: recover native follower tails across process loss and bucket partition --- .../tests/runtime/lifecycle.rs | 4 +- .../tests/runtime/lifecycle/ownership.rs | 2 + .../lifecycle/ownership/follower_process.rs | 682 ++++++++++++++++++ .../lifecycle/ownership/partition_proxy.rs | 92 +++ .../lifecycle/ownership/sparse_process.rs | 2 +- docs/publication-progress-measurements.md | 77 +- 6 files changed, 849 insertions(+), 10 deletions(-) create mode 100644 crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs create mode 100644 crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs diff --git a/crates/cellule-runtime/tests/runtime/lifecycle.rs b/crates/cellule-runtime/tests/runtime/lifecycle.rs index 3c65cd21..3e6999a2 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle.rs @@ -61,7 +61,7 @@ pub mod residency; #[derive(Debug)] pub(super) struct PausingStore { - inner: Arc, + inner: Arc, armed: AtomicBool, update_armed: AtomicBool, failing: AtomicBool, @@ -87,7 +87,7 @@ pub(super) struct PausingStore { } impl PausingStore { - pub(super) fn new(inner: Arc) -> Self { + pub(super) fn new(inner: Arc) -> Self { Self { inner, armed: AtomicBool::new(false), diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs index 7acc881a..a3d59675 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs @@ -2,7 +2,9 @@ use super::*; +mod follower_process; mod lease; +mod partition_proxy; mod prefix; mod recovery; mod sparse_process; diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs new file mode 100644 index 00000000..1488445e --- /dev/null +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs @@ -0,0 +1,682 @@ +//! Device-backed process loss of a two-follower ACK before root publication. +//! Native followers share the killed process; this is storage/recovery +//! qualification, not authentication or remote-peer transport qualification. +use super::*; +use cellule_runtime::node::log_transport::LocalFollowerTransport; +use cellule_runtime::node::{NodeAdvertisement, NodeCapacity, NodeDirectory, NodeFailureDomain}; +use std::path::{Path as FsPath, PathBuf}; +use std::process::{Child, Command, Stdio}; + +const CHILD: &str = "runtime::lifecycle::ownership::follower_process::rustfs_follower_tail_child"; +const CHILD_ROOT: &str = "CELLULE_FOLLOWER_TAIL_CHILD_ROOT"; +const PARTITION: &str = "CELLULE_FOLLOWER_TAIL_PARTITION"; +const LEADER: SessionId = SessionId::from_bytes([151; 16]); +const SUCCESSOR: SessionId = SessionId::from_bytes([154; 16]); +const MEMBERS: [NodeId; 2] = [NodeId::from_bytes([152; 16]), NodeId::from_bytes([153; 16])]; + +struct KillOnDrop(Child); +impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn fixture( + store: Store, + partition: bool, +) -> (CellTarget, CellStorageLayout, CellReplica, CellAuthority) { + let target = CellTarget::new( + TenantId::from_bytes([61; 16]), + ApplicationId::from_bytes([62; 16]), + NamespaceId::from_bytes([63; 16]), + b"follower-process-tail", + ) + .unwrap(); + let prefix = std::env::var("CELLULE_TEST_PREFIX").unwrap(); + let layout = CellStorageLayout::new( + store, + Path::from(format!( + "{prefix}/follower-process-tail/{}", + if partition { "partition" } else { "kill" } + )), + [62; 16], + ); + let replica = CellReplica::new( + layout.clone(), + *target.cell_id().as_bytes(), + [64; 16], + Limits::default(), + ) + .unwrap(); + let authority = CellAuthority::new(layout.clone()); + (target, layout, replica, authority) +} + +fn directory(layout: &CellStorageLayout) -> NodeDirectory { + NodeDirectory::new( + layout.clone(), + Digest::from_bytes([90; 32]), + Digest::from_bytes([91; 32]), + Digest::from_bytes([92; 32]), + ) +} + +async fn enroll(directory: &NodeDirectory) { + let mut leader = None; + for session in [ + LEADER, + SessionId::from_bytes([152; 16]), + SessionId::from_bytes([153; 16]), + SUCCESSOR, + ] { + let expires = if session == LEADER { 10_001 } else { 20_000 }; + let ad = NodeAdvertisement::sign( + NodeId::from_bytes(*session.as_bytes()), + session, + "https://process-fixture.internal:8081".into(), + Digest::from_bytes([90; 32]), + Digest::from_bytes([94; 32]), + Digest::from_bytes([91; 32]), + Digest::from_bytes([92; 32]), + &ed25519_dalek::SigningKey::from_bytes( + session.as_bytes().repeat(2).as_slice().try_into().unwrap(), + ), + 1, + 1, + expires, + vec![Digest::from_bytes([95; 32])], + vec![1], + NodeFailureDomain::default(), + NodeCapacity { + free_memory_bytes: 1 << 20, + free_disk_bytes: 1 << 30, + follower_free_bytes: if MEMBERS.contains(&NodeId::from_bytes(*session.as_bytes())) { + 1 << 30 + } else { + 0 + }, + follower_retained_bytes: 0, + job_credits: 1, + log_protocol: cellule_runtime::node::NODE_LOG_PROTOCOL_VERSION, + }, + ) + .unwrap(); + let observed = directory.create(ad, 1).await.unwrap(); + if session == LEADER { + leader = Some(observed); + } + } + let enrolled = directory + .recruit_log(&leader.unwrap(), 1, 2, 4, 2) + .await + .unwrap(); + assert_eq!(enrolled.advertisement().log().unwrap().members(), MEMBERS); + directory.activate_log(&enrolled, 3).await.unwrap(); +} + +struct NativeAuthority(NodeDirectory); +impl NodeLogAuthority for NativeAuthority { + fn activate<'a>( + &'a self, + epoch: u64, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result<()>> { + Box::pin(async move { + let current = self + .0 + .load_if_live(LEADER, 4) + .await? + .ok_or(cellule_runtime::Error::Fenced)?; + let log = current + .advertisement() + .log() + .ok_or(cellule_runtime::Error::Fenced)?; + assert_eq!(log.epoch(), epoch); + assert!(log.active()); + assert_eq!(log.members(), MEMBERS); + Ok(()) + }) + } + fn advance_coverage<'a>( + &'a self, + _epoch: u64, + _through: u64, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result<()>> { + // This cut must retain an unpublished tail. Any credit here fails the test. + Box::pin(async { + Err(cellule_runtime::Error::Node( + "unexpected bucket credit before killed owner", + )) + }) + } + fn close<'a>( + &'a self, + _retirement: &'a cellule_runtime::node::log::NodeLogRetirementObservation, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result<()>> { + Box::pin(async { + Err(cellule_runtime::Error::Node( + "killed owner cannot close its log", + )) + }) + } +} + +struct NativeTransport(Vec<(NodeId, LocalFollowerTransport)>); +impl NativeTransport { + fn open(root: &FsPath) -> Self { + Self( + MEMBERS + .into_iter() + .enumerate() + .map(|(index, member)| { + let store = cellule_runtime::FollowerStore::open( + root.join(format!("follower-{index}")), + Limits::default(), + DiskBudget::new(1 << 30), + ) + .unwrap(); + (member, LocalFollowerTransport::new(member, store)) + }) + .collect(), + ) + } + fn peer(&self, member: NodeId) -> &LocalFollowerTransport { + &self.0.iter().find(|(id, _)| *id == member).unwrap().1 + } +} +impl NodeLogTransport for NativeTransport { + fn append<'a>( + &'a self, + member: NodeId, + request: AppendRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result> { + self.peer(member).append(member, request) + } + fn seal<'a>( + &'a self, + member: NodeId, + request: SealRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result> { + self.peer(member).seal(member, request) + } + fn retire<'a>( + &'a self, + member: NodeId, + request: RetireRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result> { + self.peer(member).retire(member, request) + } + fn tail<'a>( + &'a self, + member: NodeId, + request: TailRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result>> { + self.peer(member).tail(member, request) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rustfs_follower_tail_child() { + let Ok(root) = std::env::var(CHILD_ROOT) else { + return; + }; + let root = PathBuf::from(root); + let partition = std::env::var_os(PARTITION).is_some(); + let remote = sparse_process::rustfs_store(); + let pausing = Arc::new(PausingStore::new(remote.inner().clone())); + let (target, layout, replica, authority) = fixture(Store::new(pausing.clone()), partition); + let catalog = cellule_runtime::cell::catalog::CellCatalog::new(layout.clone(), target.tenant()); + let proof = catalog.lookup(target.cell_id()).await.unwrap().unwrap(); + let observed = authority.load(target.cell_id()).await.unwrap().unwrap(); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + SqlWorkerPool::new(1, 10).unwrap(), + 64 << 20, + LEADER, + ReplicaHost::default(), + ) + .unwrap(); + let lease = NodeLeaseGuard::new(0, 60_000).unwrap(); + runtime.install_node_lease(lease.clone()).unwrap(); + let handle = runtime + .acquire_idle_restored( + proof, + replica, + authority.clone(), + observed, + root.join("owner/active.sqlite"), + Owner { + session: LEADER, + endpoint: "https://process-fixture.internal:8081".into(), + }, + ) + .await + .unwrap(); + if partition { + let first = handle + .execute( + mutation_identity_window(155, 10, 10_000), + Digest::from_bytes([156; 32]), + 20, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success( + b"published-before-partition".to_vec(), + )) + }, + ) + .await + .unwrap(); + assert_eq!(first.commit_sequence(), 1); + assert_eq!( + authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap() + .value() + .ltx_root() + .unwrap() + .commit_sequence, + 1 + ); + } + let transport: Arc = Arc::new(NativeTransport::open(&root)); + let gate = + DurabilityGate::new(LEADER, NodeId::from_bytes(*LEADER.as_bytes()), 1, MEMBERS).unwrap(); + let shipper = NodeLogShipper::new(gate.clone(), transport.clone(), Limits::default()).unwrap(); + runtime + .install_node_durability( + target.application(), + Arc::new(NodeDurability::new( + gate, + shipper, + Arc::new(NativeAuthority(directory(&layout))), + transport, + lease, + )), + ) + .unwrap(); + let responses = Arc::new(super::super::durability::RecordingResponses::default()); + runtime.install_telemetry(responses.clone()).unwrap(); + runtime.active_catalog_entries().await.unwrap(); + if partition { + std::fs::write( + root.join("ready-to-partition"), + b"published-before-partition:1", + ) + .unwrap(); + while !root.join("partition-started").exists() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let uncertain = tokio::time::timeout( + std::time::Duration::from_secs(5), + handle.execute( + mutation_identity_window(157, 10, 10_000), + Digest::from_bytes([158; 32]), + 21, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(b"uncertain-partition".to_vec())) + }, + ), + ) + .await; + assert!( + !matches!(uncertain, Ok(Ok(_))), + "partition exposed unproved success: {uncertain:?}" + ); + assert!(responses.0.lock().unwrap().is_empty()); + // Timeout is uncertain: it does not certify that SQLite never committed. + std::fs::write(root.join("partition-result"), format!("{uncertain:?}")).unwrap(); + std::fs::write(root.join("acknowledged"), b"published-before-partition:1").unwrap(); + std::future::pending::<()>().await; + } + // Hold the first data PUT, before either bytes or root credit are published. + pausing.arm(); + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(10), + handle.execute( + mutation_identity_window(155, 10, 10_000), + Digest::from_bytes([156; 32]), + 20, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(b"fleet-tail".to_vec())) + }, + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + outcome, + StoredOutcome::Success { + result: b"fleet-tail".to_vec(), + commit_sequence: 1 + } + ); + assert_eq!( + responses.0.lock().unwrap().as_slice(), + &[cellule_runtime::fleet::telemetry::CommandResponseSource::Fleet] + ); + pausing.wait_until_blocked().await; + assert_eq!( + authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap() + .value() + .ltx_root() + .unwrap() + .commit_sequence, + 0 + ); + std::fs::write(root.join("acknowledged"), b"fleet-tail:1").unwrap(); + std::future::pending::<()>().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires isolated RustFS, explicit credentials and CELLULE_TEST_DEVICE_PATH on a device filesystem"] +async fn rustfs_killed_follower_tail_owner_recovers_every_ack_and_exact_retry() { + run_cut(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires isolated RustFS, explicit credentials and CELLULE_TEST_DEVICE_PATH on a device filesystem"] +async fn rustfs_partitioned_owner_exposes_no_unproved_success_and_recovers_earlier_ack() { + run_cut(true).await; +} + +async fn run_cut(partition: bool) { + let device = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); + let scratch = tempfile::tempdir_in(device).unwrap(); + let root = scratch.path(); + std::fs::create_dir(root.join("owner")).unwrap(); + let (target, layout, replica, authority) = fixture(sparse_process::rustfs_store(), partition); + let catalog = cellule_runtime::cell::catalog::CellCatalog::new(layout.clone(), target.tenant()); + let proof = catalog + .provision( + CatalogEntry::new( + &target, + CatalogRole::Application, + Digest::from_bytes([65; 32]), + 1, + ) + .unwrap(), + ) + .await + .unwrap(); + let initial = authority + .create_initial( + &proof, + IncarnationId::from_bytes([64; 16]), + Owner { + session: LEADER, + endpoint: "https://process-fixture.internal:8081".into(), + }, + ) + .await + .unwrap(); + let bootstrap_runtime = + CellRuntime::new(SqlWorkerPool::new(1, 10).unwrap(), 64 << 20, LEADER).unwrap(); + let bootstrap = bootstrap_runtime + .bootstrap( + proof.clone(), + replica.clone(), + authority.clone(), + initial, + root.join("bootstrap.sqlite"), + |tx| { + tx.execute_batch( + "CREATE TABLE counter(value INTEGER NOT NULL); INSERT INTO counter VALUES(0)", + )?; + Ok(()) + }, + ) + .await + .unwrap(); + bootstrap.drain().await.unwrap(); + bootstrap_runtime.shutdown().await.unwrap(); + let directory = directory(&layout); + // The exact ensemble is authoritative before the child can acknowledge. + enroll(&directory).await; + let proxy = if partition { + Some(super::partition_proxy::PartitionProxy::start().await) + } else { + None + }; + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", CHILD, "--nocapture"]) + .env(CHILD_ROOT, root) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + if let Some(proxy) = &proxy { + command + .env(PARTITION, "1") + .env("CELLULE_TEST_ENDPOINT", &proxy.endpoint); + } + let mut child = KillOnDrop(command.spawn().unwrap()); + tokio::time::timeout(std::time::Duration::from_secs(20), async { + loop { + assert!( + child.0.try_wait().unwrap().is_none(), + "child exited before its ACK cut" + ); + if root.join("acknowledged").exists() { + break; + } + if partition + && root.join("ready-to-partition").exists() + && !root.join("partition-started").exists() + { + assert_eq!( + std::fs::read(root.join("ready-to-partition")).unwrap(), + b"published-before-partition:1" + ); + proxy.as_ref().unwrap().partition(); + std::fs::write(root.join("partition-started"), b"network-cut").unwrap(); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!( + std::fs::read(root.join("acknowledged")).unwrap(), + if partition { + b"published-before-partition:1".as_slice() + } else { + b"fleet-tail:1".as_slice() + } + ); + child.0.kill().unwrap(); + assert!(!child.0.wait().unwrap().success()); + if let Some(proxy) = proxy { + assert!( + proxy.rejected() > 0, + "network partition was never exercised" + ); + eprintln!( + "partition={} rejected_connections={} response={}", + proxy.endpoint, + proxy.rejected(), + std::fs::read_to_string(root.join("partition-result")).unwrap() + ); + proxy.shutdown().await; + } + std::fs::remove_dir_all(root.join("owner")).unwrap(); + assert_eq!( + authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap() + .value() + .ltx_root() + .unwrap() + .commit_sequence, + u64::from(partition) + ); + let transport: Arc = Arc::new(NativeTransport::open(root)); + let fenced = directory + .claim_expired(LEADER, SUCCESSOR, 10_001) + .await + .unwrap(); + for member in MEMBERS { + assert_eq!( + transport + .seal( + member, + SealRequest { + leader_session: LEADER, + log_epoch: 1 + } + ) + .await + .unwrap(), + FollowerReceipt { + base_sequence: 1, + durable_through: 1 + } + ); + let frames = transport + .tail( + member, + TailRequest { + leader_session: LEADER, + log_epoch: 1, + first_sequence: 1, + }, + ) + .await + .unwrap(); + assert_eq!(frames.len(), 1); + let frame = cellule_ltx::inspect_node_frame(frames[0].clone(), Limits::default()).unwrap(); + assert_eq!(frame.scope().leader_session, *LEADER.as_bytes()); + assert_eq!(frame.scope().log_epoch, 1); + assert_eq!(frame.scope().node_sequence, 1); + assert_eq!(frame.scope().commit_sequence, if partition { 2 } else { 1 }); + assert_eq!(frame.scope().cell, *target.cell_id().as_bytes()); + } + let recovery = cellule_runtime::node::log_recovery::NodeLogRecovery::from_fenced( + transport, + &fenced, + Limits::default(), + ) + .unwrap(); + let manifests = cellule_runtime::recovery::manifest::RecoveryManifestStore::new( + layout.clone(), + Limits::default(), + ); + let coordinator = + cellule_runtime::node::log_recovery::RecoveryCoordinator::new(recovery, manifests.clone()); + let inventory = + cellule_runtime::node::log_recovery::recoverable_cells(&catalog, &authority, LEADER, 10) + .await + .unwrap(); + let completed = coordinator + .recover_and_seal(&directory, fenced, inventory, 10_002) + .await + .unwrap(); + assert_eq!(completed.controls.len(), 1); + let successor = + CellRuntime::new(SqlWorkerPool::new(1, 10).unwrap(), 64 << 20, SUCCESSOR).unwrap(); + let restored = successor + .takeover_restored( + proof, + replica, + authority.clone(), + completed.controls.into_iter().next().unwrap(), + completed.takeover, + manifests, + root.join("successor.sqlite"), + Owner { + session: SUCCESSOR, + endpoint: "https://successor.internal:8081".into(), + }, + ) + .await + .unwrap(); + let retry = restored + .execute( + mutation_identity_window(155, 10, 10_000), + Digest::from_bytes([156; 32]), + 30, + 1_024, + 1_024, + |_| { + Err(cellule_runtime::Error::Control( + "acknowledged follower-tail retry executed twice", + )) + }, + ) + .await + .unwrap(); + assert_eq!( + retry, + StoredOutcome::Success { + result: if partition { + b"published-before-partition".to_vec() + } else { + b"fleet-tail".to_vec() + }, + commit_sequence: 1 + } + ); + let bytes = restored + .query(64, 64, |connection| { + let value: i64 = + connection.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }) + .await + .unwrap(); + assert_eq!( + i64::from_be_bytes(bytes.try_into().unwrap()), + if partition { 2 } else { 1 } + ); + if partition { + let resolved = restored + .execute( + mutation_identity_window(157, 10, 10_000), + Digest::from_bytes([158; 32]), + 31, + 1_024, + 1_024, + |_| { + Err(cellule_runtime::Error::Control( + "recovered uncertain partition retry executed twice", + )) + }, + ) + .await + .unwrap(); + assert_eq!( + resolved, + StoredOutcome::Success { + result: b"uncertain-partition".to_vec(), + commit_sequence: 2 + } + ); + let bytes = restored + .query(64, 64, |connection| { + let value: i64 = + connection.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }) + .await + .unwrap(); + assert_eq!(i64::from_be_bytes(bytes.try_into().unwrap()), 2); + } + restored.drain().await.unwrap(); + successor.shutdown().await.unwrap(); + scratch.close().unwrap(); +} diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs new file mode 100644 index 00000000..b08df5c9 --- /dev/null +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs @@ -0,0 +1,92 @@ +//! Test-owned TCP partition of one child's real RustFS transport. +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::watch; +use tokio::task::{JoinHandle, JoinSet}; + +pub(super) struct PartitionProxy { + pub(super) endpoint: String, + mode: watch::Sender, + rejected: Arc, + joined: Option>, +} +impl PartitionProxy { + pub(super) async fn start() -> Self { + let endpoint = std::env::var("CELLULE_TEST_ENDPOINT").unwrap(); + // This isolated qualification uses plain HTTP to local RustFS. The + // proxy forwards the original signed bytes, including the Host header. + let destination = endpoint + .strip_prefix("http://") + .unwrap() + .trim_end_matches('/') + .to_owned(); + assert!(!destination.contains('/')); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let (mode, mut changed) = watch::channel(0_u8); + let rejected = Arc::new(AtomicUsize::new(0)); + let counted = rejected.clone(); + let joined = tokio::spawn(async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + biased; + _ = changed.changed() => { + if *changed.borrow() == 2 { break; } + } + Some(result) = connections.join_next(), if !connections.is_empty() => { result.unwrap(); } + incoming = listener.accept() => { + let (mut downstream, _) = incoming.unwrap(); + if *changed.borrow() != 0 { + counted.fetch_add(1, Ordering::SeqCst); + continue; + } + let mut cut = changed.clone(); + let destination = destination.clone(); + let counted = counted.clone(); + connections.spawn(async move { + let mut upstream = TcpStream::connect(destination).await.unwrap(); + tokio::select! { + biased; + _ = cut.changed() => { counted.fetch_add(1, Ordering::SeqCst); } + _ = tokio::io::copy_bidirectional(&mut downstream, &mut upstream) => {} + } + }); + } + } + } + connections.abort_all(); + while let Some(result) = connections.join_next().await { + if let Err(error) = result { + assert!(error.is_cancelled()); + } + } + }); + Self { + endpoint, + mode, + rejected, + joined: Some(joined), + } + } + pub(super) fn partition(&self) { + self.mode.send(1).unwrap(); + } + pub(super) fn rejected(&self) -> usize { + self.rejected.load(Ordering::SeqCst) + } + pub(super) async fn shutdown(mut self) { + self.mode.send(2).unwrap(); + self.joined.take().unwrap().await.unwrap(); + } +} +impl Drop for PartitionProxy { + fn drop(&mut self) { + if let Some(joined) = &self.joined { + joined.abort(); + } + } +} diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs index 2ea2646b..b0744bf5 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs @@ -8,7 +8,7 @@ const CHILD_PATH: &str = "runtime::lifecycle::ownership::sparse_process::rustfs_ const ACTIVE_PATH_ENV: &str = "CELLULE_LTX_SPARSE_OWNER_ACTIVE"; const READY_PATH_ENV: &str = "CELLULE_LTX_SPARSE_OWNER_READY"; -fn rustfs_store() -> Store { +pub(super) fn rustfs_store() -> Store { let required = |name| std::env::var(name).unwrap_or_else(|_| panic!("missing {name}")); build_explicit_store( &required("CELLULE_TEST_BUCKET"), diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 93d3d29e..4de0f25b 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -210,11 +210,11 @@ under the v7 directory, with `mixed-publication-analysis.json`, ## Next capacity work -1. Compose ready root checkpoints and native captures in one verified canonical - catalog update, or otherwise restore safe overlap. The measured mixed path - falls back to serial publication in 1,019 of 1,022 rounds. Preserve root credit, - exact dependency verification and the authority CAS; retain the current guard - until the replacement protocol and fault tests prove the same ordering. +1. Qualify the checkpoint-overlap candidate below on an idle host. The earlier + mixed path fell back to serial publication in 1,019 of 1,022 rounds; the + candidate removes that fallback while preserving exact dependency verification, + ordered root credit and the authority CAS. Its first fresh 750-write/s run + failed delivery and debt gates under competing compiler load. 2. Reduce preparation/verification work and bytes per cohort. Use the new phase and stage measurements to check actual allocation bounds and repeated catalog work. Ramp fresh five-minute offers through 750, 1,000, 1,500 and 2,000 writes/s, @@ -257,8 +257,10 @@ full-advertisement equality. Capture now retains the exact Open epoch, membershi and signed boot identity while reobserving monotone generation, activation, coverage and lease progress. Its original heartbeat regression fails before the change and passes after it; a held-read closure case still rejects. The original -CI failures and 50 passing local repeats remain retained. This concrete fix is -not yet confirmation of the hosted failure's cause or a passing new CI run. +CI failures and 50 passing local repeats remain retained. Hosted workspace run +38181130233 subsequently passed, including the public controller models. This +concrete fix and passing run do not establish that the heartbeat race caused +the original hosted failures. An isolated controller repeat passed 383 of 384 cases. The remaining failure was fixture construction: separate clock reads could sign a 30,001-ms @@ -276,6 +278,67 @@ not unpublished follower tails, network partitions or physical power loss. The initial missing-import build failure and corrected passing run are both retained under the v9 artifact directory. +Two further ignored integration cases passed against fresh ext4 follower, +SQLite and RustFS volumes. The exact two-follower ensemble is authoritative +before the child can acknowledge. The first case holds the original data PUT, +verifies a follower-based success while the authoritative root remains at +sequence zero, kills the child and removes its local writer files. Recovery +reopens both native follower logs, verifies their exact retained frame, restores +the acknowledged state and returns the original retry without running its +mutation again. + +The second case forwards the child's real RustFS traffic through a test-owned +TCP proxy, publishes one mutation, then closes existing connections and rejects +new connections. The next mutation receives no success before its client +deadline. Its captured state is nevertheless present on both follower disks; +recovery retains that uncertain mutation as well as the earlier ACK. Both +original request IDs return their stored outcomes without reexecution. The +proxy recorded 49 rejected or closed connections in the passing run. + +The native followers share the killed process and use separate device-backed +directories. These cases qualify process loss, native follower persistence and +an owner-to-object-store TCP partition, not remote peer transport, follower +network partitions or power loss. Initial setup and incorrect uncertain-tail +expectation failures remain retained alongside the passing `device-crash-r5` +logs. Strict runtime test Clippy and the isolated shared-store integration suite +pass: 246 cases, with six environment-dependent cases ignored. + +### Fresh 750-write/s checkpoint-overlap run + +Source `da8887d`, five-minute offered window, 2,000 Cells, unchanged 64-MiB +budget and the same client/auditor binaries: + +| Measurement | Result | +| --- | --- | +| Offered writes | 225,000 | +| Successful writes within the window | 210,347; 701.16/s | +| Returned errors / dropped offers | 11,915 / 2,731 | +| Scheduled p50 / p95 / p99 | 9.7 / 50.1 / 318.4 ms | +| Warmup errors / drops | 0 / 42 | +| Every-ACK warm and cold readback and original retries | All 234,813; zero errors | +| Joined fleet drain / cold startup | 108.56 / 29.75 seconds | +| Final three-minute debt / age slopes | +866,204 bytes/s / +393 ms/s; failed | +| Pipeline checkpoint / headroom fallbacks | 0 / 3,579 | + +The case completed its audits but failed the capacity gate. Host samples inside +the measured window show competing compiler activity in 19 of 20 observations, +averaging 349% sampled CPU and peaking at 696%. It does not establish idle-host +capacity or isolate a causal throughput effect of the candidate. Retain this +failed attempt and repeat on an idle host before advancing the ramp. + +Within this window, materialization averaged 44.17 ms and checkpoint waiting +93.83 ms. Preparation, upload and selection averaged 20.62, 20.31 and 12.34 ms. +The publisher retains its original 22-MiB scratch/cache reservation; sampled +root I/O concurrency falls from two to one as retained selection metadata and +captures grow. These observations identify admission and root work to investigate; +they do not justify changing the budget or weakening the debt gate. + +Artifacts: `cellule-checkpoint-overlap-20261011-v9`, case +`cellule-fleet-parity-checkpoint-overlap-v9-750-r1`, with +`capacity-750-derived.json`, `capacity-750-host-window.json` and the case's +`publication-analysis.json`. The 1,000/1,500/2,000 ramp and fresh read/mixed +qualification remain outstanding. + ## Retained failed attempts | Attempt | Outcome | From d054266574c42cdc3796e25c3334dcdd206d4a1c Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 16:45:57 -0700 Subject: [PATCH 14/31] Avoid duplicate bundle validation and add constrained fleet profile --- crates/cellule-ltx/docs/publication.md | 8 +++ crates/cellule-ltx/src/bundle.rs | 42 +++++++++++--- crates/cellule-ltx/tests/cell/bundle.rs | 70 +++++++++++++++++++++++ docs/publication-progress-measurements.md | 38 ++++++++++++ scripts/perf/README.md | 18 ++++++ scripts/perf/run.py | 49 +++++++++++----- 6 files changed, 204 insertions(+), 21 deletions(-) diff --git a/crates/cellule-ltx/docs/publication.md b/crates/cellule-ltx/docs/publication.md index 32668a53..d1d9ad8f 100644 --- a/crates/cellule-ltx/docs/publication.md +++ b/crates/cellule-ltx/docs/publication.md @@ -37,6 +37,14 @@ retry pins and rechecks the selected capture bytes, so path replacement cannot change an in-flight proposal. The host owns request admission, deadlines, and reconciliation after ambiguous results. +The in-memory CRB1 bundle encoder verifies each complete LTX segment before +copying its owned bytes into the envelope. It reparses the complete footer, +checks contiguous extents and duplicate identities, and requires every parsed +row to match the original verified metadata. This avoids a second LTX scan of +the same bytes while preserving the canonical encoding. External memory and +file decoders independently verify every segment; root preparation and fresh +origin verification retain their own checks. + `RootPreparation` identifies a native verified derivation while uploads may still be running. Its construction is private and it grants no uploaded-root, restore, serving, authority or acknowledgement rights. A `RootPreparationMetadata` future diff --git a/crates/cellule-ltx/src/bundle.rs b/crates/cellule-ltx/src/bundle.rs index e0d5821c..58069156 100644 --- a/crates/cellule-ltx/src/bundle.rs +++ b/crates/cellule-ltx/src/bundle.rs @@ -236,7 +236,24 @@ impl Bundle { bytes.extend(footer); bytes.extend(len.to_le_bytes()); bytes.extend(b"CRB1"); - Self::decode(bytes, limits) + let bytes = Bytes::from(bytes); + let decoded = memory_rows(&bytes, limits)?; + // Every complete segment was verified before its bytes moved into + // this private envelope. Reparse and validate the entire footer, then + // require its exact original rows; repeating the LTX scan adds no + // proof about these unchanged owned bytes. External decoders still + // independently verify every segment. + if decoded.len() != rows.len() + || decoded.iter().zip(&rows).any(|(decoded, original)| { + decoded.repository != original.repository + || decoded.epoch != original.epoch + || decoded.info != original.info + || decoded.offset != original.offset + }) + { + return Err(LtxError::LTXCorrupted); + } + Ok(Self::from_validated_memory(bytes, decoded)) } /// Verifies the complete envelope, every extent, and every segment digest. @@ -247,11 +264,7 @@ impl Bundle { /// Verifies a shared envelope without copying its complete body. pub fn decode_bytes(bytes: Bytes, limits: Limits) -> Result { let limits = limits.validate()?; - if bytes.len() as u64 > limits.max_plan_bytes { - return Err(LtxError::Limit(crate::LimitKind::BundleBytes)); - } - let (rows, payload_end) = decode_footer(&bytes, limits)?; - validate_row_layout(&rows, payload_end, limits)?; + let rows = memory_rows(&bytes, limits)?; for row in &rows { let start = usize::try_from(row.offset).map_err(|_| LtxError::LTXCorrupted)?; let end = start @@ -261,12 +274,16 @@ impl Bundle { .ok_or(LtxError::LTXCorrupted)?; crate::recovery::verify_segment(&bytes[start..end], &row.info, limits)?; } - Ok(Self { + Ok(Self::from_validated_memory(bytes, rows)) + } + + fn from_validated_memory(bytes: Bytes, rows: Vec) -> Self { + Self { digest: *blake3::hash(&bytes).as_bytes(), length: bytes.len() as u64, body: BundleBody::Memory(bytes), rows, - }) + } } /// Verifies an envelope from a caller-owned file without retaining its body in memory. @@ -437,6 +454,15 @@ impl Bundle { } } +fn memory_rows(bytes: &[u8], limits: Limits) -> Result> { + if bytes.len() as u64 > limits.max_plan_bytes { + return Err(LtxError::Limit(crate::LimitKind::BundleBytes)); + } + let (rows, payload_end) = decode_footer(bytes, limits)?; + validate_row_layout(&rows, payload_end, limits)?; + Ok(rows) +} + struct BundleUploadSource { body: BundleBody, length: u64, diff --git a/crates/cellule-ltx/tests/cell/bundle.rs b/crates/cellule-ltx/tests/cell/bundle.rs index 8cdf044c..1153ffaa 100644 --- a/crates/cellule-ltx/tests/cell/bundle.rs +++ b/crates/cellule-ltx/tests/cell/bundle.rs @@ -98,6 +98,76 @@ fn encode_rejects_a_duplicate_segment_identity() { assert!(Bundle::encode(vec![entry(), entry()], Limits::default()).is_err()); } +#[test] +fn encode_matches_the_complete_canonical_envelope_and_independent_decoder() { + let directory = tempfile::tempdir().unwrap(); + let (info, body) = segment(directory.path(), 4); + let entries = (1..=4) + .map(|id| BundleEntry::for_cell([id; 32], [id; 16], info.clone(), body.clone())) + .collect::>(); + let mut payload = Vec::new(); + let rows = entries + .iter() + .map(|entry| { + let row = BundleRow { + repository: entry.repository.clone(), + epoch: entry.epoch.clone(), + info: entry.info.clone(), + offset: payload.len() as u64, + }; + payload.extend_from_slice(&entry.bytes); + row + }) + .collect::>(); + let expected = envelope(&rows, payload); + let encoded = Bundle::encode(entries, Limits::default()).unwrap(); + assert_eq!(encoded.read_all().unwrap().as_ref(), expected); + assert_eq!(encoded.digest(), *blake3::hash(&expected).as_bytes()); + let decoded = Bundle::decode(expected, Limits::default()).unwrap(); + assert_eq!(decoded.digest(), encoded.digest()); + for (index, row) in rows.iter().enumerate() { + assert_eq!(decoded.rows()[index].info, row.info); + assert_eq!(decoded.rows()[index].offset, row.offset); + assert_eq!(decoded.read_segment(index).unwrap().as_ref(), body); + } +} + +#[test] +fn encode_still_verifies_ltx_after_its_outer_digest_is_rebound() { + let directory = tempfile::tempdir().unwrap(); + let (mut info, mut body) = segment(directory.path(), 5); + *body.last_mut().unwrap() ^= 1; + info.blake3 = *blake3::hash(&body).as_bytes(); + assert!( + Bundle::encode( + vec![BundleEntry::for_cell([5; 32], [5; 16], info, body)], + Limits::default(), + ) + .is_err() + ); +} + +#[test] +fn encode_includes_the_footer_in_the_complete_plan_bound() { + let directory = tempfile::tempdir().unwrap(); + let (info, body) = segment(directory.path(), 6); + let limits = Limits { + max_capture_bytes: info.size_bytes, + max_file_bytes: info.size_bytes, + max_plan_bytes: info.size_bytes, + ..Limits::default() + }; + assert!(matches!( + Bundle::encode( + vec![BundleEntry::for_cell([6; 32], [6; 16], info, body)], + limits, + ), + Err(cellule_ltx::LtxError::Limit( + cellule_ltx::LimitKind::BundleBytes + )) + )); +} + #[test] fn builder_rejects_more_than_the_segment_limit() { let directory = tempfile::tempdir().unwrap(); diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 4de0f25b..dec7022d 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -339,6 +339,44 @@ Artifacts: `cellule-checkpoint-overlap-20261011-v9`, case `publication-analysis.json`. The 1,000/1,500/2,000 ramp and fresh read/mixed qualification remain outstanding. +### Removing a duplicate LTX scan + +A diagnostic captured an actual SQLite mutation and encoded 64 distinct Cell +rows from the same pinned 203-byte LTX segment. The in-memory CRB1 constructor +verified each segment before copying it, then verified it again through the +external decoder. Temporary counters in an isolated snapshot measured 128 full +segment validations per encoding. The candidate performs 64, retaining complete +footer/layout checks and exact comparisons with the original verified rows. +Its independent decoder still performs all 64 validations. The complete +40,144-byte envelope and digest match exactly before and after. + +Eight debug-build samples of 100 encodings had median durations of 1,041.49 ms +before and 636.36 ms after. These shared-host component timings do not establish +application throughput or a causal fleet speedup. Instrumentation was removed +before verification: the full LTX suite passed 323 tests/doctests with seven +ignored, the runtime suite passed 1,316 with thirteen ignored, local-only LTX +passed 66, and strict workspace Clippy passed. Existing origin verification, +formats and resource bounds remain intact. Artifacts are in the v9 +`bundle-encode-diagnostic` directory, including the pinned input, raw samples, +counter comparison and contract results. + +### Requested three-node resource profile + +The `three-node-1cpu` profile uses one owner and two required followers, each +limited to 1 vCPU and 2 GiB memory with swap disabled. Their guest CPU sets are +0, 1 and 2; RustFS occupies 3–4 and the client 5–8. A live Docker probe confirmed +the configured limits and effective cgroup quota, memory and zero-swap limits +for all five containers. This establishes resource enforcement, not application +capacity. The original queued 8-CPU-owner repeat was stopped before startup in +response to this resource request; its deferral and observations remain retained. + +Fresh application runs will retain the same 2,000 Cells, 64-MiB budget, five-minute +windows and every-ACK warm/cold readback and retry checks. Results belong to this +separate resource profile. Artifacts are under +`cellule-three-small-nodes-20261011-v10`; the initial resource evidence is +`limits-probe.json`. These containers share one Linux VM and their temporary +database storage remains tmpfs; physical-media tests are a separate route. + ## Retained failed attempts | Attempt | Outcome | diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 4d4158fa..f90b4ff1 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -61,6 +61,24 @@ separate profile: do not compare it as an algorithm change against historical shared-VM runs. Guest CPU placement does not isolate host services or qualify physical-device durability; report those limits separately. +`--resource-profile three-node-1cpu` runs one owner and two required followers, +each with 1 CPU, 2 GiB memory and no swap. They occupy guest CPUs 0, 1 and 2. +RustFS uses CPUs 3–4 with a 2-CPU/2-GiB limit; the client uses CPUs 5–8 with +a 4-CPU/2-GiB limit. Each application node uses one SQLite worker and one +Tokio worker. This profile requires at least nine guest CPUs and 12 GiB usable +memory and saves the same Docker and effective cgroup checks. It keeps the +64-MiB retained-work budget and all delivery, debt and ACK audit requirements. +These are three capped containers in one Linux VM; host failures remain shared. +Record this profile separately from either 8-CPU owner profile. Node state is +still tmpfs, so device-backed crash tests remain a separate verification route. + +```sh +python3 scripts/perf/run.py cellule-fleet --context dedicated-linux \ + --artifacts /absolute/external/candidate --resource-profile three-node-1cpu \ + --tag three-small-nodes --cells 2000 --seconds 300 --warmup 30 \ + --repetitions 1 --write-rates 750 1000 1500 2000 +``` + The example authority's renewal and publication regressions use a generated TLS identity and an in-memory directory. From an isolated verification checkout, run them explicitly with the TLS directory produced by diff --git a/scripts/perf/run.py b/scripts/perf/run.py index 5d970251..4a83cffe 100644 --- a/scripts/perf/run.py +++ b/scripts/perf/run.py @@ -18,10 +18,20 @@ def isolated_owner(): return getattr(ARGS, 'resource_profile', 'shared-vm') == 'isolated-owner' -def resource_limits(role): +def small_nodes(): + return getattr(ARGS, 'resource_profile', 'shared-vm') == 'three-node-1cpu' + +def resource_limits(role, node_index=None): if role not in ('owner', 'follower', 'provider', 'client'): raise ValueError('unknown benchmark resource role') - if isolated_owner(): + if small_nodes(): + cpus, gib = {'owner': (1, 2), 'follower': (1, 2), + 'provider': (2, 2), 'client': (4, 2)}[role] + if role == 'follower' and node_index not in (1, 2): + raise ValueError('small-node follower requires its placement index') + cpuset = {'owner': '0', 'follower': str(node_index), + 'provider': '3-4', 'client': '5-8'}[role] + elif isolated_owner(): cpus, gib = {'owner': (8, 16), 'follower': (1, 2), 'provider': (2, 2), 'client': (4, 2)}[role] cpuset = '0-7' if role == 'owner' else '8-11' @@ -31,10 +41,10 @@ def resource_limits(role): cpuset = None return cpus, gib, cpuset -def resource_args(role): - cpus, gib, cpuset = resource_limits(role) +def resource_args(role, node_index=None): + cpus, gib, cpuset = resource_limits(role, node_index) args = ['--cpus', str(cpus), '--memory', f'{gib}g'] - if isolated_owner() or role != 'client': + if isolated_owner() or small_nodes() or role != 'client': args += ['--memory-swap', f'{gib}g'] if cpuset: args += ['--cpuset-cpus', cpuset] @@ -43,11 +53,18 @@ def resource_args(role): def validate_resource_host(info): if isolated_owner() and (info['NCPU'] < 12 or info['MemTotal'] < 23 * 1024**3): raise ValueError('isolated owner requires at least 12 guest CPUs and 23 GiB usable memory') + if small_nodes() and (info['NCPU'] < 9 or info['MemTotal'] < 12 * 1024**3): + raise ValueError('three small nodes and separate support roles require at least 9 guest CPUs and 12 GiB usable memory') def verify_resource_role(name, role): - if not isolated_owner(): + if not isolated_owner() and not small_nodes(): return None - cpus, gib, cpuset = resource_limits(role) + node_index = None + if role == 'follower': + if not name.endswith(('-peer-1', '-peer-2')): + raise ValueError('follower name lacks its placement index') + node_index = int(name[-1]) + cpus, gib, cpuset = resource_limits(role, node_index) config = json.loads(docker('inspect', name).stdout)[0]['HostConfig'] expected = {'NanoCpus': cpus * 10**9, 'Memory': gib * 1024**3, 'MemorySwap': gib * 1024**3, 'CpusetCpus': cpuset} @@ -230,9 +247,12 @@ def sampler(directory, stop): def start_node(system, durability, index, prefix, name): port = 8080 + index * 10 - args = ['run', '-d', '--name', name, '--network', 'host', *resource_args('follower' if index else 'owner'), '--ulimit', 'nofile=65536:65536', '--tmpfs', '/scratch:rw,size=4g', *CREDS] + workers = 1 if small_nodes() else 8 + args = ['run', '-d', '--name', name, '--network', 'host', *resource_args('follower' if index else 'owner', index), '--ulimit', 'nofile=65536:65536', '--tmpfs', '/scratch:rw,size=4g', *CREDS] + if small_nodes(): + args += ['-e', 'TOKIO_WORKER_THREADS=1'] if system == 'cellule': - args += ['-e', f'PARITY_RETAINED_BYTES={ARGS.retained_bytes}', '-e', f'PARITY_DISK_BYTES={ARGS.disk_bytes}', '-v', f'{BASE}:/work:ro', '-e', 'TMPDIR=/scratch', '-e', f'CELLULE_AXUM_CELLS={ARGS.cells}', '-e', 'CELLULE_AXUM_WORKERS=8', '-e', f'CELLULE_AXUM_BIND=127.0.0.1:{port}', '-e', 'CELLULE_TEST_ENDPOINT=http://127.0.0.1:9000', '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', f'CELLULE_TEST_PREFIX={prefix}'] + args += ['-e', f'PARITY_RETAINED_BYTES={ARGS.retained_bytes}', '-e', f'PARITY_DISK_BYTES={ARGS.disk_bytes}', '-v', f'{BASE}:/work:ro', '-e', 'TMPDIR=/scratch', '-e', f'CELLULE_AXUM_CELLS={ARGS.cells}', '-e', f'CELLULE_AXUM_WORKERS={workers}', '-e', f'CELLULE_AXUM_BIND=127.0.0.1:{port}', '-e', 'CELLULE_TEST_ENDPOINT=http://127.0.0.1:9000', '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', f'CELLULE_TEST_PREFIX={prefix}'] if durability == 'fleet' or index == 0: args += ['-e', 'CELLULE_AXUM_FLEET_DIR=/scratch/fleet'] if durability == 'bucket' and index == 0: @@ -241,7 +261,7 @@ def start_node(system, durability, index, prefix, name): args += ['-e', f'CELLULE_AXUM_FOLLOWER={index}'] args += [RUST, 'sh', '-c', f"mkdir -m 700 /scratch/fleet && cp /work/tls/ca.crt /work/tls/node-{index}.crt /work/tls/node-{index}.key /scratch/fleet/ && exec /work/{MANIFEST['binaries']['sql']['path']}"] else: - args += ['-e', f'CELLD_PLACEMENT_WEIGHT={(1000000000 if index == 0 else 1)}', '-e', 'CELLD_WATCH=/scratch', '-e', f'CELLD_DURABILITY={durability}', '-e', 'CELLD_SHUTDOWN_TOTAL_MS=600000', '-e', 'CELLD_TOKIO_THREADS=8', '-e', 'CELLD_REBALANCE_INTERVAL_MS=0', '-e', f"CELLD_NODE={name.replace('-cold', '-owner')}", '-e', 'RUST_LOG=warn,celld::node_log=info', CELLD, '--bucket', f's3://comparison/{prefix}', '--endpoint', 'http://127.0.0.1:9000', '--region', 'us-east-1', '--listen', f'127.0.0.1:{port}', '--internal-listen', f'127.0.0.1:{port + 1}', '--advertise', f'127.0.0.1:{port + 1}'] + args += ['-e', f'CELLD_PLACEMENT_WEIGHT={(1000000000 if index == 0 else 1)}', '-e', 'CELLD_WATCH=/scratch', '-e', f'CELLD_DURABILITY={durability}', '-e', 'CELLD_SHUTDOWN_TOTAL_MS=600000', '-e', f'CELLD_TOKIO_THREADS={workers}', '-e', 'CELLD_REBALANCE_INTERVAL_MS=0', '-e', f"CELLD_NODE={name.replace('-cold', '-owner')}", '-e', 'RUST_LOG=warn,celld::node_log=info', CELLD, '--bucket', f's3://comparison/{prefix}', '--endpoint', 'http://127.0.0.1:9000', '--region', 'us-east-1', '--listen', f'127.0.0.1:{port}', '--internal-listen', f'127.0.0.1:{port + 1}', '--advertise', f'127.0.0.1:{port + 1}'] docker(*args) verify_resource_role(name, 'follower' if index else 'owner') return wait_ready(name, 'Follower service:' if index else 'Orders service:') if system == 'cellule' else wait_ready(name, port=port) @@ -332,7 +352,10 @@ def run_case(system, durability): directory.mkdir(exist_ok=False) (directory / 'runner.py').write_bytes(RUNNER_BYTES) prefix = label + '-' + uuid.uuid4().hex[:12] - put(directory / 'case.json', {'system': system, 'durability': durability, 'prefix': prefix, 'framework_commit': MANIFEST['framework_revision'] if system == 'cellule' else 'f2bf648663a610eefde71f3547ad61e9b896b1f0', 'resident_cells': ARGS.cells, 'concurrency': int(ARGS.concurrency), 'queue_capacity': int(ARGS.queue_capacity), 'candidate_binary': MANIFEST['binaries']['sql']['path'], 'celld_application': 'celld-app', 'retained_budget_bytes': int(ARGS.retained_bytes), 'managed_disk_budget_bytes': int(ARGS.disk_bytes), 'value_bytes': 96, 'owner_cpus': 8, 'owner_memory_bytes': 16 * 1024 ** 3, 'followers': 2 if durability == 'fleet' else 0, 'profile': 'isolated-owner-sql-ledger-96-v1' if isolated_owner() else 'shared-vm-sql-ledger-96', 'provider_storage': 'fresh Linux Docker volume', 'provider_filesystem_required': True, 'provider_lifecycle_required': True, 'telemetry': ARGS.telemetry, 'warmup_seconds': ARGS.warmup, 'seconds': ARGS.seconds, 'runner_sha256': RUNNER_SHA256, 'docker_host': DOCKER_HOST, 'diagnostic': ARGS.seconds < 300 or ARGS.warmup < 30 or ARGS.telemetry == 'off'}) + owner_cpus, owner_gib, _ = resource_limits('owner') + profile = ('three-node-1cpu-sql-ledger-96-v1' if small_nodes() else + 'isolated-owner-sql-ledger-96-v1' if isolated_owner() else 'shared-vm-sql-ledger-96') + put(directory / 'case.json', {'system': system, 'durability': durability, 'prefix': prefix, 'framework_commit': MANIFEST['framework_revision'] if system == 'cellule' else 'f2bf648663a610eefde71f3547ad61e9b896b1f0', 'resident_cells': ARGS.cells, 'concurrency': int(ARGS.concurrency), 'queue_capacity': int(ARGS.queue_capacity), 'candidate_binary': MANIFEST['binaries']['sql']['path'], 'celld_application': 'celld-app', 'retained_budget_bytes': int(ARGS.retained_bytes), 'managed_disk_budget_bytes': int(ARGS.disk_bytes), 'value_bytes': 96, 'owner_cpus': owner_cpus, 'owner_memory_bytes': owner_gib * 1024 ** 3, 'followers': 2 if durability == 'fleet' else 0, 'profile': profile, 'provider_storage': 'fresh Linux Docker volume', 'provider_filesystem_required': True, 'provider_lifecycle_required': True, 'telemetry': ARGS.telemetry, 'warmup_seconds': ARGS.warmup, 'seconds': ARGS.seconds, 'runner_sha256': RUNNER_SHA256, 'docker_host': DOCKER_HOST, 'diagnostic': ARGS.seconds < 300 or ARGS.warmup < 30 or ARGS.telemetry == 'off'}) names = [] stop = threading.Event() sampling = None @@ -542,8 +565,8 @@ def provision(): import argparse parser = argparse.ArgumentParser(description='Paired, pinned SQL durability verification; raw evidence stays outside the repository.') parser.add_argument('--context', required=True) - parser.add_argument('--resource-profile', choices=['shared-vm', 'isolated-owner'], default='shared-vm', - help='isolated-owner reserves guest CPUs 0-7 for the owner and 8-11 for support roles') + parser.add_argument('--resource-profile', choices=['shared-vm', 'isolated-owner', 'three-node-1cpu'], default='shared-vm', + help='isolated-owner reserves CPUs 0-7; three-node-1cpu pins three 1-CPU/2-GiB nodes separately from support roles') parser.add_argument('--artifacts', type=Path, required=True) parser.add_argument('--tag', default=uuid.uuid4().hex[:10]) parser.add_argument('--seconds', type=int, default=300) From ba0915c9e03590f8284d1f81b96c470cfa33c2c2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 17:34:38 -0700 Subject: [PATCH 15/31] test(perf): verify three separate small node VMs --- docs/publication-progress-measurements.md | 57 +++++++++++++-- scripts/perf/README.md | 27 ++++++++ scripts/perf/run.py | 84 +++++++++++++++++++---- 3 files changed, 150 insertions(+), 18 deletions(-) diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index dec7022d..d66aef72 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -370,12 +370,57 @@ for all five containers. This establishes resource enforcement, not application capacity. The original queued 8-CPU-owner repeat was stopped before startup in response to this resource request; its deferral and observations remain retained. -Fresh application runs will retain the same 2,000 Cells, 64-MiB budget, five-minute -windows and every-ACK warm/cold readback and retry checks. Results belong to this -separate resource profile. Artifacts are under -`cellule-three-small-nodes-20261011-v10`; the initial resource evidence is -`limits-probe.json`. These containers share one Linux VM and their temporary -database storage remains tmpfs; physical-media tests are a separate route. +The preliminary 2,000-Cell diagnostic used 30 seconds warmup and 60-second +measurement windows. At 100 offered writes/s plus 1,000 reads/s, it delivered +99.20 writes/s and 992.18 reads/s, with zero returned errors but 47 write drops +and 468 read drops; one read also dropped during warmup. Read-only offered +1,000/s delivered 999.98/s with zero errors or drops. All 10,954 acknowledged +mutations passed warm readback and exact retry, but the owner exceeded the +original 120-second drain deadline and the cold audit did not run. The case +failed; these short, busy-host measurements do not qualify capacity. +Artifacts are in `cellule-three-small-nodes-20261011-v10`, including +`setup-verification-result.json`, raw windows and failed lifecycle evidence. + +The user clarified that the required topology is **three separate VMs**. The +`three-vm-1cpu` profile routes each node to its own Docker host and retains +fresh node volumes on those VMs' disks. Hypervisor configurations allocate +one vCPU and 2 GiB each; distinct guest boot IDs and Docker host identities, +usable memory, disabled guest/container swap and OOM counters are recorded. +RustFS and the client run in a separate 6-vCPU, 8-GiB support VM. Loopback TCP +routes use SSH tunnels while retaining end-to-end pinned peer mTLS; their +transport overhead and shared Mac host belong to the measurement scope. +The 2,000-Cell, 64-MiB, five-minute capacity run still requires an idle host, +every-ACK warm/cold audits, stable debt and the original drain deadline. +New topology artifacts are in `cellule-three-vms-20261011-v11`. + +A fresh 16-Cell smoke run completed on the separate VMs. Each window used +30 seconds warmup and 60 seconds measurement, with the unchanged 96-byte value +and 64-MiB budget: + +| Window | Delivered writes/s | Delivered reads/s | Returned errors / drops | Scheduled p99, write / read | +| --- | ---: | ---: | --- | --- | +| 10 offered writes/s + 100 reads/s | 10.00 | 100.00 | 0 / 0 | 2,058.6 / 1,885.5 ms | +| Read-only, 100 offered reads/s | 0 | 100.00 | 0 / 0 | — / 44.7 ms | + +All 917 ACKs passed warm and cold readback and exact retry. Fleet drain took +8.28 seconds; fresh bucket-only startup took 1.10 seconds. No guest or container +OOM occurred. Mixed response-source deltas were 134 follower proofs and 466 +bundle proofs; each follower recorded four failed append attempts and its log +reported `Fenced`. These observations do not identify the earlier CI failure. +Host samples showed competing work, with mixed-window CPU idle values of +38.81%, 38.34%, 0% and 61.90%. The latency target failed, and a 16-Cell smoke +cannot qualify 2,000-Cell capacity. Clean shutdown and cold bucket restore also +do not substitute for remote-VM crash recovery of an unpublished follower tail. +Exact resource identities, metrics, hashes and raw evidence are retained in +`three-vm-smoke-results.json` and `cellule-fleet-parity-three-vms-smoke-r1`. +The fresh 2,000-Cell, five-minute 750-write/s case waits for the original idle-host +start check; higher ramp points remain pending. + +The four ext4-backed process/partition cases were also repeated with fresh +volumes after a prior cleanup observation was interrupted. All four passed in +51.36 seconds; the repeat recorded both provider and worker stopped without +OOM. This is device-backed process evidence, separate from remote-node VM +failure or power-loss qualification. Both attempts remain retained. ## Retained failed attempts diff --git a/scripts/perf/README.md b/scripts/perf/README.md index f90b4ff1..b8666121 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -79,6 +79,33 @@ python3 scripts/perf/run.py cellule-fleet --context dedicated-linux \ --repetitions 1 --write-rates 750 1000 1500 2000 ``` +`--resource-profile three-vm-1cpu --node-contexts OWNER FOLLOWER1 FOLLOWER2` +instead places each node on a separate Docker host. Each node VM must have +one guest CPU and a 2-GiB configured memory limit; the runner checks one CPU, +usable RAM between 1.75 and 2 GiB, and three distinct Docker host identities. +Record hypervisor configuration separately to verify the total VM allocation. +Node containers use CPU 0, a 2-GiB ceiling and no swap; disable guest swap as +well. This profile retains a fresh disk-backed node volume, rather than tmpfs, +and records its filesystem. Node volumes and provider volumes remain retained. + +The separate support context needs at least six CPUs and 7 GiB usable RAM: +RustFS uses CPUs 0–1 and the client 2–5, with the same 2-GiB container limits. +All nodes must reach the provider at loopback port 9000. The owner and client +must reach follower ports 8090 and 8100, and the client must reach owner port +8080. This fixture keeps loopback-only application endpoints; prepare and record +those cross-VM routes before running. TCP tunnels retain end-to-end pinned mTLS +but add transport overhead that belongs in the reported configuration. Capture +guest memory, swap, boot identity and OOM evidence alongside host contention. +Existing delivery, debt, audit and 120-second drain requirements still apply. + +```sh +python3 scripts/perf/run.py cellule-fleet --context separate-support \ + --artifacts /absolute/external/candidate --resource-profile three-vm-1cpu \ + --node-contexts node-owner node-follower-1 node-follower-2 \ + --tag three-vms --cells 2000 --seconds 300 --warmup 30 \ + --repetitions 1 --write-rates 750 1000 1500 2000 +``` + The example authority's renewal and publication regressions use a generated TLS identity and an in-memory directory. From an isolated verification checkout, run them explicitly with the TLS directory produced by diff --git a/scripts/perf/run.py b/scripts/perf/run.py index 4a83cffe..77eac231 100644 --- a/scripts/perf/run.py +++ b/scripts/perf/run.py @@ -11,6 +11,7 @@ ARGS = None MANIFEST = None DOCKER_HOST = None +NODE_CONTEXTS = {} RUNNER_BYTES = Path(__file__).read_bytes() RUNNER_SHA256 = hashlib.sha256(RUNNER_BYTES).hexdigest() CREDS = ['-e', 'AWS_ACCESS_KEY_ID=benchmark_access', '-e', 'AWS_SECRET_ACCESS_KEY=benchmark_secret_private', '-e', 'AWS_DEFAULT_REGION=us-east-1'] @@ -19,12 +20,19 @@ def isolated_owner(): return getattr(ARGS, 'resource_profile', 'shared-vm') == 'isolated-owner' def small_nodes(): - return getattr(ARGS, 'resource_profile', 'shared-vm') == 'three-node-1cpu' + return getattr(ARGS, 'resource_profile', 'shared-vm') in ('three-node-1cpu', 'three-vm-1cpu') + +def separate_vms(): + return getattr(ARGS, 'resource_profile', 'shared-vm') == 'three-vm-1cpu' def resource_limits(role, node_index=None): if role not in ('owner', 'follower', 'provider', 'client'): raise ValueError('unknown benchmark resource role') - if small_nodes(): + if separate_vms(): + cpus, gib = {'owner': (1, 2), 'follower': (1, 2), + 'provider': (2, 2), 'client': (4, 2)}[role] + cpuset = {'owner': '0', 'follower': '0', 'provider': '0-1', 'client': '2-5'}[role] + elif small_nodes(): cpus, gib = {'owner': (1, 2), 'follower': (1, 2), 'provider': (2, 2), 'client': (4, 2)}[role] if role == 'follower' and node_index not in (1, 2): @@ -53,7 +61,9 @@ def resource_args(role, node_index=None): def validate_resource_host(info): if isolated_owner() and (info['NCPU'] < 12 or info['MemTotal'] < 23 * 1024**3): raise ValueError('isolated owner requires at least 12 guest CPUs and 23 GiB usable memory') - if small_nodes() and (info['NCPU'] < 9 or info['MemTotal'] < 12 * 1024**3): + if separate_vms() and (info['NCPU'] < 6 or info['MemTotal'] < 7 * 1024**3): + raise ValueError('separate support VM requires at least 6 guest CPUs and 7 GiB usable memory') + if small_nodes() and not separate_vms() and (info['NCPU'] < 9 or info['MemTotal'] < 12 * 1024**3): raise ValueError('three small nodes and separate support roles require at least 9 guest CPUs and 12 GiB usable memory') def verify_resource_role(name, role): @@ -106,8 +116,16 @@ def provider_filesystem(): result = parse_provider_filesystem(docker('exec', STORE, 'sh', '-c', 'df -Pk /data && df -Pi /data').stdout) return dict(result, timestamp=time.time()) -def docker(*args, check=True, timeout=700): - p = subprocess.run(DOCKER + list(map(str, args)), capture_output=True, text=True, timeout=timeout) +def docker(*args, check=True, timeout=700, context=None): + if context is None: + name = None + if args[0] == 'run' and '--name' in args: + name = args[args.index('--name') + 1] + elif args[0] in ('exec', 'inspect', 'logs', 'kill', 'wait', 'rm', 'stop', 'restart'): + name = next((value for value in args[1:] if value in NODE_CONTEXTS), None) + context = NODE_CONTEXTS.get(name, CTX) + command = ['docker', '--context', context] if context is not None else DOCKER + p = subprocess.run(command + list(map(str, args)), capture_output=True, text=True, timeout=timeout) if check and p.returncode: raise RuntimeError(f'docker {args[0]} failed: {p.stdout} {p.stderr}') return p @@ -214,8 +232,12 @@ def snapshot(directory, label, names): data[name] = {'inspect': json.loads(docker('inspect', name).stdout)[0]} role = 'provider' if name == STORE else 'client' if name == CONTROL else 'follower' if name.endswith(('peer-1', 'peer-2')) else 'owner' data[name]['resource_placement'] = verify_resource_role(name, role) - p = docker('exec', name, 'sh', '-c', 'cat /proc/1/limits; cat /proc/1/status; ls /proc/1/fd | wc -l; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/cpu.stat', check=False) + p = docker('exec', name, 'sh', '-c', 'cat /proc/1/limits; cat /proc/1/status; ls /proc/1/fd | wc -l; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/memory.peak; cat /sys/fs/cgroup/cpu.stat', check=False) data[name]['process'] = p.stdout + p.stderr + if name in NODE_CONTEXTS: + guest = docker('exec', name, 'sh', '-c', 'cat /proc/sys/kernel/random/boot_id; cat /proc/meminfo; cat /proc/swaps; awk \'$1 == "oom_kill" {print}\' /proc/vmstat') + data[name]['guest'] = guest.stdout + data[name]['docker_context'] = NODE_CONTEXTS[name] state = data[name]['inspect']['State'] if state['OOMKilled'] or not state['Running']: failures.append(f'{name}: {state}') @@ -232,7 +254,10 @@ def sampler(directory, stop): with (directory / 'docker-stats.jsonl').open('w') as f: while not stop.is_set(): try: - p = docker('stats', '--no-stream', '--format', '{{json .}}', check=False, timeout=30) + stats = [] + for context in dict.fromkeys([CTX, *NODE_CONTEXTS.values()]): + p = docker('stats', '--no-stream', '--format', '{{json .}}', check=True, timeout=30, context=context) + stats.extend(dict(json.loads(line), DockerContext=context) for line in p.stdout.splitlines() if line) except Exception as error: f.write(json.dumps({'timestamp': time.time(), 'error': str(error)}) + '\n') f.flush() @@ -241,14 +266,23 @@ def sampler(directory, stop): filesystem = provider_filesystem() except Exception as error: filesystem = {'error': str(error), 'measurement_usable': False} - f.write(json.dumps({'timestamp': time.time(), 'provider_filesystem': filesystem, 'stats': [json.loads(x) for x in p.stdout.splitlines() if x]}) + '\n') + f.write(json.dumps({'timestamp': time.time(), 'provider_filesystem': filesystem, 'stats': stats}) + '\n') f.flush() stop.wait(3) def start_node(system, durability, index, prefix, name): port = 8080 + index * 10 workers = 1 if small_nodes() else 8 - args = ['run', '-d', '--name', name, '--network', 'host', *resource_args('follower' if index else 'owner', index), '--ulimit', 'nofile=65536:65536', '--tmpfs', '/scratch:rw,size=4g', *CREDS] + if separate_vms(): + NODE_CONTEXTS[name] = ARGS.node_contexts[index] + volume = name + '-state' + if docker('volume', 'inspect', volume, check=False, context=NODE_CONTEXTS[name]).returncode == 0: + raise RuntimeError('node state volume is not fresh: ' + volume) + docker('volume', 'create', '--label', 'cellule.perf.artifacts=' + str(BASE), volume, context=NODE_CONTEXTS[name]) + storage = ['-v', volume + ':/scratch'] + else: + storage = ['--tmpfs', '/scratch:rw,size=4g'] + args = ['run', '-d', '--name', name, '--network', 'host', *resource_args('follower' if index else 'owner', index), '--ulimit', 'nofile=65536:65536', *storage, *CREDS] if small_nodes(): args += ['-e', 'TOKIO_WORKER_THREADS=1'] if system == 'cellule': @@ -264,6 +298,10 @@ def start_node(system, durability, index, prefix, name): args += ['-e', f'CELLD_PLACEMENT_WEIGHT={(1000000000 if index == 0 else 1)}', '-e', 'CELLD_WATCH=/scratch', '-e', f'CELLD_DURABILITY={durability}', '-e', 'CELLD_SHUTDOWN_TOTAL_MS=600000', '-e', f'CELLD_TOKIO_THREADS={workers}', '-e', 'CELLD_REBALANCE_INTERVAL_MS=0', '-e', f"CELLD_NODE={name.replace('-cold', '-owner')}", '-e', 'RUST_LOG=warn,celld::node_log=info', CELLD, '--bucket', f's3://comparison/{prefix}', '--endpoint', 'http://127.0.0.1:9000', '--region', 'us-east-1', '--listen', f'127.0.0.1:{port}', '--internal-listen', f'127.0.0.1:{port + 1}', '--advertise', f'127.0.0.1:{port + 1}'] docker(*args) verify_resource_role(name, 'follower' if index else 'owner') + if separate_vms(): + evidence = {'context': NODE_CONTEXTS[name], 'volume': volume, 'retained': True, + 'filesystem': docker('exec', name, 'sh', '-c', 'df -T /scratch; cat /proc/mounts').stdout} + put(BASE / (name + '-storage.json'), evidence) return wait_ready(name, 'Follower service:' if index else 'Orders service:') if system == 'cellule' else wait_ready(name, port=port) def stop_node(name, directory): @@ -353,10 +391,26 @@ def run_case(system, durability): (directory / 'runner.py').write_bytes(RUNNER_BYTES) prefix = label + '-' + uuid.uuid4().hex[:12] owner_cpus, owner_gib, _ = resource_limits('owner') - profile = ('three-node-1cpu-sql-ledger-96-v1' if small_nodes() else + profile = ('three-vm-1cpu-sql-ledger-96-v1' if separate_vms() else + 'three-node-1cpu-sql-ledger-96-v1' if small_nodes() else 'isolated-owner-sql-ledger-96-v1' if isolated_owner() else 'shared-vm-sql-ledger-96') put(directory / 'case.json', {'system': system, 'durability': durability, 'prefix': prefix, 'framework_commit': MANIFEST['framework_revision'] if system == 'cellule' else 'f2bf648663a610eefde71f3547ad61e9b896b1f0', 'resident_cells': ARGS.cells, 'concurrency': int(ARGS.concurrency), 'queue_capacity': int(ARGS.queue_capacity), 'candidate_binary': MANIFEST['binaries']['sql']['path'], 'celld_application': 'celld-app', 'retained_budget_bytes': int(ARGS.retained_bytes), 'managed_disk_budget_bytes': int(ARGS.disk_bytes), 'value_bytes': 96, 'owner_cpus': owner_cpus, 'owner_memory_bytes': owner_gib * 1024 ** 3, 'followers': 2 if durability == 'fleet' else 0, 'profile': profile, 'provider_storage': 'fresh Linux Docker volume', 'provider_filesystem_required': True, 'provider_lifecycle_required': True, 'telemetry': ARGS.telemetry, 'warmup_seconds': ARGS.warmup, 'seconds': ARGS.seconds, 'runner_sha256': RUNNER_SHA256, 'docker_host': DOCKER_HOST, 'diagnostic': ARGS.seconds < 300 or ARGS.warmup < 30 or ARGS.telemetry == 'off'}) + if separate_vms(): + case = json.loads((directory / 'case.json').read_text()) + case['node_contexts'] = ARGS.node_contexts + case['node_storage'] = 'fresh retained Docker volume on each node VM disk' + put(directory / 'case.json', case) names = [] + if separate_vms(): + hosts = [] + for context in ARGS.node_contexts: + info = json.loads(docker('info', '--format', '{{json .}}', context=context).stdout) + if info['NCPU'] != 1 or not 1.75 * 1024**3 <= info['MemTotal'] <= 2 * 1024**3: + raise RuntimeError('node VM differs from 1-vCPU/2-GiB profile: ' + context) + hosts.append(dict(context=context, **{key: info[key] for key in ('ID', 'NCPU', 'MemTotal', 'Architecture', 'ServerVersion')})) + if len({host['ID'] for host in hosts}) != 3: + raise RuntimeError('three-node profile requires three distinct Docker hosts') + put(directory / 'node-hosts.json', hosts) stop = threading.Event() sampling = None summary = {'schema_version': 1, 'case': label, 'writes': [], 'reads': [], 'completed': False, 'failure': None, 'build_manifest_sha256': hashlib.sha256((BASE / 'build.json').read_bytes()).hexdigest()} @@ -565,8 +619,10 @@ def provision(): import argparse parser = argparse.ArgumentParser(description='Paired, pinned SQL durability verification; raw evidence stays outside the repository.') parser.add_argument('--context', required=True) - parser.add_argument('--resource-profile', choices=['shared-vm', 'isolated-owner', 'three-node-1cpu'], default='shared-vm', - help='isolated-owner reserves CPUs 0-7; three-node-1cpu pins three 1-CPU/2-GiB nodes separately from support roles') + parser.add_argument('--resource-profile', choices=['shared-vm', 'isolated-owner', 'three-node-1cpu', 'three-vm-1cpu'], default='shared-vm', + help='isolated-owner reserves CPUs 0-7; three-node-1cpu caps nodes in one VM; three-vm-1cpu uses three separate 1-CPU/2-GiB Docker hosts') + parser.add_argument('--node-contexts', nargs=3, metavar=('OWNER', 'FOLLOWER1', 'FOLLOWER2'), + help='three separate node Docker hosts; loopback peer/provider routes must already be connected') parser.add_argument('--artifacts', type=Path, required=True) parser.add_argument('--tag', default=uuid.uuid4().hex[:10]) parser.add_argument('--seconds', type=int, default=300) @@ -587,6 +643,10 @@ def provision(): help='after steady points offer 1.5x this qualified capacity for 60s, then 0.5x for 30s') parser.add_argument('cases', nargs='*', choices=['cellule-fleet', 'celld-fleet', 'cellule-bucket', 'celld-bucket']) ARGS = parser.parse_args() + if separate_vms() != bool(ARGS.node_contexts): + parser.error('three-vm-1cpu requires exactly three node contexts, and other profiles do not use them') + if ARGS.node_contexts and (len(set(ARGS.node_contexts)) != 3 or ARGS.context in ARGS.node_contexts): + parser.error('owner, followers and support require distinct Docker contexts') BASE = ARGS.artifacts.resolve() CTX = ARGS.context if BASE == Path(__file__).resolve().parents[2] or Path(__file__).resolve().parents[2] in BASE.parents: From d821b6eff460499c656882a666fb0152ef923689 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 18:31:32 -0700 Subject: [PATCH 16/31] fix: reserve clock headroom in signed follower requests --- crates/cellule-axum/examples/fleet/tests.rs | 32 ++++++++++++++++++ .../cellule-axum/examples/fleet/transport.rs | 7 +--- crates/cellule-axum/examples/fleet/wire.rs | 12 +++++++ docs/publication-progress-measurements.md | 33 +++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/crates/cellule-axum/examples/fleet/tests.rs b/crates/cellule-axum/examples/fleet/tests.rs index 59be10cc..7f7d97f2 100644 --- a/crates/cellule-axum/examples/fleet/tests.rs +++ b/crates/cellule-axum/examples/fleet/tests.rs @@ -126,3 +126,35 @@ fn grant_issuance_and_append_tokens_have_separate_signed_shapes() { request.grant.pop(); assert!(wire::validate(&request, 1000).is_err()); } + +#[test] +fn sender_deadline_passes_receiver_horizon_with_clock_headroom_and_preserves_grant_expiry() { + let mut request = wire::Request { + sender: vec![1; 16], + leader: vec![1; 16], + member: vec![2; 16], + epoch: 1, + operation: 5, + first_sequence: 1, + deadline_ms: wire::request_deadline(1000, 0).unwrap(), + ..Default::default() + }; + // The real three-VM request failed this first receiver check with a 1-ms + // lag. Exercise the same producer/consumer seam, including its full buffer. + assert!(wire::validate(&request, 999).is_ok()); + assert!(wire::validate(&request, 0).is_ok()); + assert!(wire::validate(&request, -1).is_err()); + request.operation = 1; + request.frames = vec![vec![3; 128]]; + request.grant = vec![7; 32]; + request.deadline_ms = wire::request_deadline(1000, 2000).unwrap(); + assert!(wire::validate(&request, 1000).is_ok()); + assert!(matches!( + wire::validate(&request, 2000), + Err(Error::PeerAuthorization("expired capacity log request")) + )); + assert!(matches!( + wire::request_deadline(i64::MAX, 0), + Err(Error::Deadline) + )); +} diff --git a/crates/cellule-axum/examples/fleet/transport.rs b/crates/cellule-axum/examples/fleet/transport.rs index 74a419a6..37cd4a13 100644 --- a/crates/cellule-axum/examples/fleet/transport.rs +++ b/crates/cellule-axum/examples/fleet/transport.rs @@ -102,12 +102,7 @@ impl Transport { }; request.sender = self.sender.as_bytes().to_vec(); request.member = member.as_bytes().to_vec(); - let deadline = clock()?.checked_add(10_000).ok_or(Error::Deadline)?; - request.deadline_ms = if request.deadline_ms == 0 { - deadline - } else { - request.deadline_ms.min(deadline) - }; + request.deadline_ms = wire::request_deadline(clock()?, request.deadline_ms)?; let phase = std::time::Instant::now(); let body = request.encode_to_vec(); let digest = blake3::hash(&body); diff --git a/crates/cellule-axum/examples/fleet/wire.rs b/crates/cellule-axum/examples/fleet/wire.rs index c580018d..622180d7 100644 --- a/crates/cellule-axum/examples/fleet/wire.rs +++ b/crates/cellule-axum/examples/fleet/wire.rs @@ -82,6 +82,18 @@ fn signing_bytes(body: &[u8], domain: &[u8]) -> Vec { bytes } +pub(super) fn request_deadline(now: i64, previous: i64) -> Result { + // Leave clock headroom below the receiver's ten-second horizon. Using the + // whole horizon rejected a real grant request when its clock lagged by 1 ms. + // A grant's own expiry remains an earlier, independent limit. + let deadline = now.checked_add(9_000).ok_or(Error::Deadline)?; + Ok(if previous == 0 { + deadline + } else { + previous.min(deadline) + }) +} + pub(super) fn validate(request: &Request, now: i64) -> Result<()> { if request.sender.len() != 16 || request.member.len() != 16 diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index d66aef72..cafb9f8d 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -416,6 +416,39 @@ Exact resource identities, metrics, hashes and raw evidence are retained in The fresh 2,000-Cell, five-minute 750-write/s case waits for the original idle-host start check; higher ramp points remain pending. +A separate 120-second diagnostic on the same three VMs traced all 41 rejected +append attempts to grant expiry **after native fsync**. The lane remained open +and the token digest matched; the original post-write check refused the proof. +Failed attempts spent 22.2–872.0 ms in fsync. Error-only instrumentation retained +the original errors and authority checks. The window delivered 9.88 writes/s +and 98.81 reads/s, with zero returned errors but 14 write and 143 read drops; +scheduled write/read p99 was 3,363.3/3,011.9 ms. All 1,503 ACKs passed warm/cold +readback and exact retry; drain took 18.10 seconds. This busy-host diagnostic +does not qualify capacity. Evidence is in `cellule-grant-fenced-20261011-v12`. + +A one-second early-renewal prototype then failed for a separate reason: a signed +grant request exceeded the follower's unchanged ten-second horizon by 1 ms. +The owner entered rotation and its Cells stopped serving. All 1,200 measured +write and 12,000 read attempts failed; the warm audit could not read the 140 +earlier ACKs, and the cold audit did not run. Owner drain exceeded 120 seconds +and cleanup stopped it; followers stopped normally, with no container OOM. +The early-renewal prototype was not promoted and its effect remains unproved. +`cellule-grant-renewal-20261011-v13` retains the failed attempt and exact deadline +samples. These append/grant observations do not establish the cause of the +earlier maintenance CI failures, whose fixture uses local follower transport. + +The client now leaves one second below the receiver's existing ten-second +horizon; earlier grant deadlines still take precedence. Five signed-transport +tests passed, including the reproduced 1-ms lag, the headroom boundary, expired +grant deadlines and overflow. A fresh 120-second run with this deadline change +and the original grant reuse completed: 9.46 writes/s and 94.58 reads/s, zero +returned errors, 65 write drops and 650 read drops. Scheduled write/read p99 was +6,866.7/6,842.5 ms. All 1,452 ACKs passed warm/cold readback and exact retry; +drain took 10.27 seconds. There were 39 post-fsync grant-expiry rejections, with +failed fsyncs spanning 15.3–1,469.7 ms. The deadline fix preserves serving across +the observed clock difference; this busy-host diagnostic still fails delivery +and latency qualification. Evidence is in `cellule-request-deadline-20261011-v14`. + The four ext4-backed process/partition cases were also repeated with fresh volumes after a prior cleanup observation was interrupted. All four passed in 51.36 seconds; the repeat recorded both provider and worker stopped without From b1818da5fda84ac5cd14dd5f01eeeb7296602ebe Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 18:55:35 -0700 Subject: [PATCH 17/31] docs: record fresh verification on three separate small VMs --- docs/publication-progress-measurements.md | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index cafb9f8d..b9110c33 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -449,6 +449,36 @@ failed fsyncs spanning 15.3–1,469.7 ms. The deadline fix preserves serving acr the observed clock difference; this busy-host diagnostic still fails delivery and latency qualification. Evidence is in `cellule-request-deadline-20261011-v14`. +The committed revision `d821b6e` was then rebuilt without diagnostic patches and +verified on the three separate VMs. Both windows used 16 Cells, 96-byte values, +the unchanged 64-MiB budget, 30 seconds warmup and 120 seconds measurement: + +| Window / request | Offered/s | Delivered/s | Scheduled p50 / p95 / p99 | Errors / drops | +| --- | ---: | ---: | --- | --- | +| Mixed writes | 10 | 9.04 | 34.3 / 2,650.2 / 10,284.6 ms | 0 / 115 | +| Mixed reads | 100 | 90.51 | 4.9 / 2,313.0 / 10,217.1 ms | 0 / 1,139 | +| Read-only | 100 | 100.00 | 4.7 / 13.1 / 53.7 ms | 0 / 0 | + +The mixed scheduled histogram overflowed for 13 write and 127 read samples, +so its table percentiles use nearest-rank quantiles from all attempted measured +request journals. Dropped offers have no latency sample; the original driver's +overflow and failed qualification are retained. All 1,402 ACKs passed warm/cold +readback and exact retry. Drain took 15.78 seconds; cold startup took 2.18 seconds. +All node processes stopped normally; guest swap and OOM counters stayed zero. +Mixed response-source deltas were 86 follower proofs and 999 bundle proofs; +the followers recorded 13 and 12 failed appends. Their failures grant no Fleet +credit. Start, minute and end samples in both windows showed active Fleet in +epoch 1 without fencing or rotation. + +Host CPU idle ranged from 3.16–52.17% in mixed samples and 0.17–66.64% in +read-only samples. This verifies functional behavior at the stated workload; +it does not qualify capacity. `cellule-three-vms-current-20261011-v15` retains +the source/binary identity, raw windows, `three-vm-current-results.json` and +`three-vm-current-results.md`. The waiting high-rate run now uses this committed +build, the original 2,000-Cell/five-minute/64-MiB profile, unchanged timed driver +and previously verified setup-only pacing. Earlier waiting queues were joined +before any high-rate case was admitted. + The four ext4-backed process/partition cases were also repeated with fresh volumes after a prior cleanup observation was interrupted. All four passed in 51.36 seconds; the repeat recorded both provider and worker stopped without From 9b7585b4633271d75e3621087e0717b4a3c24c9a Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 19:30:10 -0700 Subject: [PATCH 18/31] Identify failing follower maintenance phases while preserving source errors --- .../scenario/follower_maintenance/mod.rs | 107 +++++++++++++----- docs/fleet-operations-plan.md | 5 +- 2 files changed, 83 insertions(+), 29 deletions(-) diff --git a/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs b/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs index 9701caef..7eebdd09 100644 --- a/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs @@ -123,8 +123,11 @@ pub(super) async fn run( // The driver owns cordon and evacuation transitions. No fixture writes a // ready-to-close row or an invented native role proof. loop { - let report = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&report)?; + let phase = "follower-maintenance-enter-evacuation"; + let report = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &report)?; for blocker in report.blockers { if !blockers.contains(&blocker) { blockers.push(blocker); @@ -145,14 +148,18 @@ pub(super) async fn run( } boots[1] .refresh_capacity(1, journal.as_ref(), deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-donor-refresh", source))?; if nodes[1].state() == NodeState::Stopped || !nodes[1].is_management_ready() { return Err(invalid("follower source stopped before replacement")); } // The missing replacement remains an explicit policy blocker and cannot // retire an uncovered original follower or authorize native Finalize. - let blocked = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&blocked)?; + let phase = "follower-maintenance-await-follower-policy"; + let blocked = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &blocked)?; if blocked .snapshot .head() @@ -173,7 +180,10 @@ pub(super) async fn run( } } if let Some(readers) = &inputs.readers { - readers.check_original(&journal, true).await?; + readers + .check_original(&journal, true) + .await + .map_err(|source| failure("follower-maintenance-original-reader", source))?; } let store = nodes[1] .try_owned_component::(cellule_host::FOLLOWER_STORE_COMPONENT)? @@ -197,11 +207,13 @@ pub(super) async fn run( // blocks finalization despite the donor having zero local writers. boots[3] .refresh_capacity(3, journal.as_ref(), deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-spare-refresh", source))?; let selected = boots[0] .directory .select_log_members(session(0), 1, clock()?, 4) - .await?; + .await + .map_err(|source| failure("follower-maintenance-select-members", source))?; if selected != [node_id(2), node_id(3)] { return Err(invalid( "follower replacement selection includes the donor or lacks redundancy", @@ -210,8 +222,15 @@ pub(super) async fn run( // This canonical actor barrier covers every acknowledged frame before the // existing supervisor fences each old member and closes the original epoch. inputs.coverage_hold.release(); - inputs.acknowledged.source.drain().await?; - let request = nodes[0].request_node_log_rotation(1)?; + inputs + .acknowledged + .source + .drain() + .await + .map_err(|source| failure("follower-maintenance-original-source-drain", source))?; + let request = nodes[0] + .request_node_log_rotation(1) + .map_err(|source| failure("follower-maintenance-request-rotation", source))?; let completion = tokio::time::timeout_at(deadline(), async { loop { let observed = request.observe()?; @@ -230,7 +249,9 @@ pub(super) async fn run( tokio::time::sleep(Duration::from_millis(10)).await; } }) - .await??; + .await + .map_err(|source| failure("follower-maintenance-rotation-wait", source))? + .map_err(|source| failure("follower-maintenance-rotation", source))?; if completion.retirement().barrier().covered_through() == 0 || completion.replacement_epoch() != 2 { @@ -288,29 +309,48 @@ pub(super) async fn run( let record = publication.record()?; let reader_capture = if let Some(readers) = &inputs.readers { // A follower proof alone cannot close a donor that also owns a reader. - let blocked = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&blocked)?; + let phase = "follower-maintenance-await-reader-policy"; + let blocked = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &blocked)?; readers .require_blocked(&blocked, &journal, &nodes[1]) - .await?; + .await + .map_err(|source| failure(phase, source))?; for blocker in blocked.blockers { if !blockers.contains(&blocker) { blockers.push(blocker); } } - readers.prepare_replacement(&journal, boots).await?; - let blocked = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&blocked)?; + readers + .prepare_replacement(&journal, boots) + .await + .map_err(|source| failure("follower-maintenance-prepare-reader", source))?; + let phase = "follower-maintenance-await-reader-ready"; + let blocked = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &blocked)?; readers .require_blocked(&blocked, &journal, &nodes[1]) - .await?; - Some(readers.complete_replacement(&journal, boots).await?) + .await + .map_err(|source| failure(phase, source))?; + Some( + readers + .complete_replacement(&journal, boots) + .await + .map_err(|source| failure("follower-maintenance-complete-reader", source))?, + ) } else { None }; let completed = loop { - let report = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&report)?; + let phase = "follower-maintenance-complete"; + let report = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &report)?; for blocker in report.blockers { if !blockers.contains(&blocker) { blockers.push(blocker); @@ -360,9 +400,14 @@ pub(super) async fn run( "published follower policy lost original retirement history", )); } - service::readback(root, &inputs, &renewed).await?; + service::readback(root, &inputs, &renewed) + .await + .map_err(|source| failure("follower-maintenance-writer-readback", source))?; if let Some((readers, capture)) = inputs.readers.as_ref().zip(reader_capture.as_ref()) { - readers.readback(&journal, capture).await?; + readers + .readback(&journal, capture) + .await + .map_err(|source| failure("follower-maintenance-reader-readback", source))?; } let roster = FleetRoster::collect(journal.as_ref(), &completed, deadline()).await?; let final_counts = observation::complete_counts(&fleet, &roster, deadline()) @@ -397,12 +442,18 @@ pub(super) async fn run( routed_activation_replays: 0, }) } -fn checked_report(report: &cellule_host::fleet::FleetReconcileReport) -> JournalResult<()> { - if let Some(failure) = report.failures.first() { - return Err(Box::new(failure.error.clone())); +fn checked_report( + phase: &'static str, + report: &cellule_host::fleet::FleetReconcileReport, +) -> JournalResult<()> { + if let Some(failed) = report.failures.first() { + return Err(failure( + phase, + Box::new(failed.error.clone()) as JournalError, + )); } - if let Some(failure) = &report.maintenance_failure { - return Err(Box::new(failure.clone())); + if let Some(failed) = &report.maintenance_failure { + return Err(failure(phase, Box::new(failed.clone()) as JournalError)); } Ok(()) } diff --git a/docs/fleet-operations-plan.md b/docs/fleet-operations-plan.md index 9eb52b0a..629d3936 100644 --- a/docs/fleet-operations-plan.md +++ b/docs/fleet-operations-plan.md @@ -1853,7 +1853,10 @@ combined reader/follower `maintenance-roles` with the evidence limits recorded in execution evidence. Reader-only, live-follower-only and combined reader/follower maintenance each have focused end-to-end minion scenarios, as does failed-owner follower -maintenance. A role-complete maintenance CLI and the full fault/provider +maintenance. Live-follower errors identify the failing reconciliation, rotation +or readback phase and preserve the original source error. This context supports +diagnosing intermittent failures; it does not establish their earlier cause. +A role-complete maintenance CLI and the full fault/provider qualification remain required: ```sh From a0860db4b2f18ca6ea4a62ec537620be1358cd26 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 19:30:10 -0700 Subject: [PATCH 19/31] Renew cached append grants before their authority expires --- .../cellule-axum/examples/fleet/transport.rs | 8 +++++- docs/publication-progress-measurements.md | 25 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/cellule-axum/examples/fleet/transport.rs b/crates/cellule-axum/examples/fleet/transport.rs index 37cd4a13..320cef2b 100644 --- a/crates/cellule-axum/examples/fleet/transport.rs +++ b/crates/cellule-axum/examples/fleet/transport.rs @@ -212,7 +212,13 @@ impl NodeLogTransport for Transport { .collect(); for attempt in 0..2 { let valid = cached.as_ref().is_some_and(|grant| { - std::time::Instant::now() < grant.expires + // Leave time for transport and native fsync before the + // follower rechecks authority. Its signed expiry remains + // unchanged and can still reject a slower append. + grant + .expires + .saturating_duration_since(std::time::Instant::now()) + > Duration::from_secs(1) && grant .authorization .authorize( diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index b9110c33..07eeb101 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -432,7 +432,7 @@ The owner entered rotation and its Cells stopped serving. All 1,200 measured write and 12,000 read attempts failed; the warm audit could not read the 140 earlier ACKs, and the cold audit did not run. Owner drain exceeded 120 seconds and cleanup stopped it; followers stopped normally, with no container OOM. -The early-renewal prototype was not promoted and its effect remains unproved. +That failed early-renewal prototype was not promoted. `cellule-grant-renewal-20261011-v13` retains the failed attempt and exact deadline samples. These append/grant observations do not establish the cause of the earlier maintenance CI failures, whose fixture uses local follower transport. @@ -479,6 +479,29 @@ build, the original 2,000-Cell/five-minute/64-MiB profile, unchanged timed drive and previously verified setup-only pacing. Earlier waiting queues were joined before any high-rate case was admitted. +After the deadline fix, a fresh uninstrumented early-renewal probe used the +same three VMs, 16 Cells and 120-second mixed window. The client renews cached +grants when one second or less remains. Signed five-second expiry, native +pre/post-fsync authority checks and the bounded exact-frame retry stay unchanged. +Both followers recorded zero failed appends and neither peer log contained +`Fenced`, compared with 13 and 12 measured failures in the preceding separate +run. The renewal change is now applied to the SQL example transport; it leaves +time for transport and fsync but cannot guarantee an append finishes before +expiry. + +The probe delivered 9.51 writes/s and 95.18 reads/s, with zero returned errors +and 58 write / 578 read drops. Scheduled write/read p99 was 4,897.4/4,747.7 ms. +One write completed after the window and is excluded from delivered throughput. +All 1,459 ACKs passed warm/cold readback and exact retry; joined drain took +11.01 seconds and cold startup took 2.24 seconds. Node processes stopped +normally, guest OOM/swap counters remained zero, and boot IDs stayed stable. +Every mixed-window host sample reported 0% CPU idle. These separate busy-host +runs do not establish a throughput gain; dropped offers and seconds-long tails +still fail capacity and latency qualification. The explicit patch, source and +binary hashes, raw windows and `three-vm-headroom-results.json` are retained in +`cellule-grant-headroom-20261011-v16`. A fresh committed build and the unchanged +idle-host ramp remain required. + The four ext4-backed process/partition cases were also repeated with fresh volumes after a prior cleanup observation was interrupted. All four passed in 51.36 seconds; the repeat recorded both provider and worker stopped without From 08c43e5e6dc5f8be82d094326a56e531e635f3f0 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 20:01:31 -0700 Subject: [PATCH 20/31] Record current VM metrics and require a paired regression decision --- docs/publication-progress-measurements.md | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 07eeb101..98bfe647 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -502,6 +502,35 @@ binary hashes, raw windows and `three-vm-headroom-results.json` are retained in `cellule-grant-headroom-20261011-v16`. A fresh committed build and the unchanged idle-host ramp remain required. +The committed early-renewal revision `a0860db` was rebuilt without overlays and +completed both fresh 120-second windows on the same three VMs: + +| Window / request | Offered/s | Delivered/s | Scheduled p50 / p95 / p99 | Errors / drops | +| --- | ---: | ---: | --- | --- | +| Mixed writes | 10 | 9.98 | 47.8 / 1,070.8 / 1,870.8 ms | 0 / 2 | +| Mixed reads | 100 | 99.77 | 6.2 / 796.4 / 1,666.0 ms | 0 / 28 | +| Read-only | 100 | 100.00 | 5.1 / 12.5 / 34.6 ms | 0 / 0 | + +All 1,515 ACKs passed warm/cold readback and exact retries. Joined fleet drain +took 10.81 seconds; cold startup took 3.28 seconds. Both followers recorded +zero measured append failures. The host remained busy: mixed CPU idle samples +ranged from 0–60.23%, and read-only samples from 0–50.30%. Dropped offers still +fail capacity qualification. These rates describe the stated offered workloads; +they are not maximum write/read capacity. Source/binary identity, guest resource +observations and raw evidence are in `cellule-grant-current-20261011-v18`. + +Following the user's regression concern, the high-rate candidate queue is held +pending a baseline comparison. `origin/main` at `c79834d` is being rebuilt with +only the proven request-deadline correction common to both builds. Baseline +grant reuse remains unchanged. The comparison alternates baseline/candidate +order on identical VM limits, workloads and budgets, retaining every ACK audit +and host resource sample. It includes the original 2,000-Cell, five-minute, +750-write/s profile. Performance changes that fail to improve the controlled +throughput/tail-latency comparison must be reverted; correctness fixes and +recovery evidence remain required. No paired result or rollback decision exists +yet. The fresh idle-host comparison is retained in +`cellule-origin-comparison-20261011-v19`. + The four ext4-backed process/partition cases were also repeated with fresh volumes after a prior cleanup observation was interrupted. All four passed in 51.36 seconds; the repeat recorded both provider and worker stopped without From 9f22376b04ff9270c3c9fbfa1060b44572d7e2b2 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 21:59:15 -0700 Subject: [PATCH 21/31] docs: record fresh PR 76 write comparison failures --- docs/publication-progress-measurements.md | 59 +++++++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 98bfe647..8c372af3 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -1,9 +1,60 @@ # Publication-progress measurements -October 11, 2026. These fresh public-application diagnostics use Node Bundle v1, -two required followers, and the existing 64-MiB retained-work budget. They establish -a smaller 500-write/s acceptance milestone; the separate 2K-write/20K-read -qualification remains open. +These public-application diagnostics use Node Bundle v1, two required followers, +and the existing 64-MiB retained-work budget. The earlier 500-write/s milestone +uses an 8-vCPU owner. Fresh verification on three separate 1-vCPU/2-GiB VMs +has not passed latency and backlog qualification. The separate +2K-write/20K-read qualification remains open. + +## Fresh PR #76 write verification + +October 12, 2026 UTC. These new runs use three separate Linux VMs, each with +1 vCPU and 2 GiB, on the shared workstation. Each has fresh native disk-backed +node storage. The provider and client run separately. Each workload has 16 Cells, +96-byte values, a 30-second warmup, a 300-second write-only window at 10 offered +writes/s, and the unchanged 64-MiB budget and 128/128 client limits. + +| Fresh run | Writes completed inside window / offered | Errors / drops | Scheduled p50 / p95 / p99 | Warm / cold ACK checks | +| --- | ---: | ---: | --- | --- | +| Main `c79834d`, common deadline correction only | 3,000 / 3,000 (10.000/s) | 0 / 0 | 25.6 / 476.0 / 1,341.3 ms | All 3,317 pass / all 3,317 pass | +| PR #76 with two failure-only tracing overlays | 2,705 / 3,000 (9.017/s) | 295 / 0 | 42.9 / 1,360.3 / 5,007.9 ms | 3,022 checked, 126 errors / not reached | +| PR #76, uninstrumented repeat | 2,998 / 3,000 (9.993/s) | 0 / 0 | 33.1 / 691.3 / 2,314.5 ms | All 3,317 pass / all 3,317 pass | + +The table retains canonical all-attempt percentiles; the tracing row includes +failed attempts. The uninstrumented repeat completes its remaining two writes +during in-flight drain. Its production code is identical to PR head `08c43e5`; +the binary was built from `a0860db`, with no measurement overlay. Main retains +its original append-grant reuse and includes only the request-deadline correction +common to both sides. Workload and auditor binaries are byte-identical. + +Both completed cases pass every ACK's warm/cold readback and original-request +retry. Joined drain takes 11.50 seconds on main and 11.59 seconds on the PR; +cold startup takes 1.16 and 2.51 seconds. Neither passes the existing 50-ms +scheduled write-p99 gate. The PR repeat also fails the unchanged backlog gate: +unpublished bytes have a positive 612.63 B/s least-squares slope over the last +three one-minute segments. Main's corresponding byte and age slopes are +non-positive. These are individual diagnostics, not maximum throughput results. + +All three windows have 17 usable host/guest samples and zero collection failures. +Host CPU idle ranges are 6.7–66.78% on main, 0–28.57% during tracing and +0–64.88% on the PR repeat. Unequal competing load prevents a causal performance +comparison. No gain over main has been established; the alternating idle-host +comparison and conditional rollback remain pending. The original 2,000-Cell +750→1,000→1,500→2,000 write/s ramp and its evidence requirements remain intact. + +The tracing run reproduces a distinct five-second SQL timeout, fencing two +Cells while the fleet remains active. It records zero failed log-append rounds +or follower append failures; the earlier pre-load append failure does not recur. +The warm audit fails, cold restore is not reached and owner shutdown exceeds its +unchanged deadline. A passing uninstrumented repeat does not erase that failure. +The earlier 2,000-Cell readiness timeout and all-failed 16-Cell sweep remain +retained, including the latter's missing in-window host/guest evidence. + +Raw artifacts are under `cellule-origin-comparison-20261011-v19`, +`cellule-append-trace-20261012-v23` and `cellule-grant-current-20261011-v18`. +The compact `fresh-write-comparison-v25.json` and `.md` reports reconcile request +journals, original histograms, source/binary manifests, ACK audits, publication +frontiers, provider lifecycle and resource traces. Earlier measurements follow. ## Workload and resources From 23f035dca8b983a4f40b0fe252a4d5e6b8cfd943 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sun, 11 Oct 2026 22:48:44 -0700 Subject: [PATCH 22/31] test: isolate checkpoint upload admission barriers --- .../src/node/bundle/tests/checkpoint_pipeline.rs | 12 ++++++++++++ docs/publication-progress-measurements.md | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs index 3f9974f4..f66b1fec 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs @@ -117,6 +117,13 @@ async fn checkpoint_case(retained: usize, outcome: Outcome) { tokio::time::timeout(Duration::from_secs(3), store.second_done.notified()) .await .unwrap(); + // Consume the prior successor's entry permit before resetting its PUT + // counter. Otherwise it can certify the next cohort before that upload + // enters, and fencing legitimately prevents admission instead of testing + // that shutdown joins an accepted PUT. + tokio::time::timeout(Duration::from_secs(3), store.second_entered.notified()) + .await + .unwrap(); store.resume.notify_one(); tokio::time::timeout(Duration::from_secs(3), authority.entered.notified()) .await @@ -180,6 +187,11 @@ async fn checkpoint_case(retained: usize, outcome: Outcome) { let overlap = tokio::time::timeout(Duration::from_secs(2), successor_entered.notified()) .await .is_ok(); + assert_eq!( + store.puts.load(Ordering::Acquire), + 2, + "current successor PUT has not entered" + ); assert!(!first.selection.as_ref().unwrap().is_ready()); assert!(!second.selection.as_ref().unwrap().is_ready()); assert!( diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 8c372af3..ce867c2d 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -302,6 +302,18 @@ the candidate's fresh application capacity ramp has not yet completed. The 500-write/s and mixed/read-only tables above still describe source `03d2eba`, not this candidate. See the [managed protocol](managed-bundle-pipeline.md). +Hosted workspace run [38206015346](https://github.com/crabbuild/cellule/actions/runs/38206015346) +at `9f22376` failed the checkpoint shutdown-cancellation test. An assertion-only +isolated reproduction showed its successor-entry notification returning with +only one current PUT entered: the notification belonged to the preceding +cohort. The test now consumes that prior notification before resetting its +counter and requires two current PUTs before injecting fencing. All six +checkpoint cases pass with the original deadlines, shutdown assertions, +resource-release checks and 64-MiB budget. Production code is unchanged; +this test-barrier correction does not establish a performance gain or identify +the original maintenance failure's cause. Artifacts are retained under +`cellule-checkpoint-barrier-20261012-v27`. + The maintenance investigation also reproduces a distinct capture race: a real same-boot heartbeat between two directory observations caused `Fenced` through full-advertisement equality. Capture now retains the exact Open epoch, membership From 89d45995091b8b16ef808298ef81485b98b3fd1f Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 00:26:23 -0700 Subject: [PATCH 23/31] test: recover unpublished ACKs across separate VMs --- crates/cellule-axum/examples/fleet/mod.rs | 2 + .../examples/fleet/remote_fault_tests.rs | 378 ++++++++++++++++++ docs/publication-progress-measurements.md | 37 ++ docs/remote-follower-tail-verification.md | 66 +++ scripts/verify-remote-follower-tail.py | 240 +++++++++++ 5 files changed, 723 insertions(+) create mode 100644 crates/cellule-axum/examples/fleet/remote_fault_tests.rs create mode 100644 docs/remote-follower-tail-verification.md create mode 100755 scripts/verify-remote-follower-tail.py diff --git a/crates/cellule-axum/examples/fleet/mod.rs b/crates/cellule-axum/examples/fleet/mod.rs index 938bed80..88fd2e76 100644 --- a/crates/cellule-axum/examples/fleet/mod.rs +++ b/crates/cellule-axum/examples/fleet/mod.rs @@ -13,6 +13,8 @@ use std::{ }; mod authority; +#[cfg(test)] +mod remote_fault_tests; mod server; #[cfg(test)] mod tests; diff --git a/crates/cellule-axum/examples/fleet/remote_fault_tests.rs b/crates/cellule-axum/examples/fleet/remote_fault_tests.rs new file mode 100644 index 00000000..f7f8e1d0 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/remote_fault_tests.rs @@ -0,0 +1,378 @@ +//! Real signed follower RPCs, unpublished ACK, killed owner and canonical recovery. +//! The external driver owns three distinct VM/container roles and their native disks. +use super::*; +use cellule_ltx::{CellReplica, Host, Limits}; +use cellule_runtime::{ + CatalogRole, CellTarget, MutationIdentity, NamespaceId, SqlWorkerPool, TenantId, + cell::{ + catalog::{CatalogEntry, CellCatalog}, + executor::{HandlerOutcome, StoredOutcome}, + }, + control::{Owner as CellOwner, authority::CellAuthority}, + identity::{IncarnationId, RequestId}, + node::{ + log_recovery::{NodeLogRecovery, RecoveryCoordinator, recoverable_cells}, + log_transport::NodeLogTransport, + }, + recovery::manifest::RecoveryManifestStore, +}; +use cellule_store::Store; +use futures_util::stream::BoxStream; +use object_store::{ObjectStore, path::Path}; +use std::sync::atomic::{AtomicBool, Ordering}; + +const APPLICATION: ApplicationId = ApplicationId::from_bytes([62; 16]); +const CODE: Digest = Digest::from_bytes([65; 32]); +const INCARNATION: IncarnationId = IncarnationId::from_bytes([64; 16]); +const REQUEST: RequestId = RequestId::from_bytes([155; 16]); +const PAYLOAD: Digest = Digest::from_bytes([156; 32]); + +#[derive(Debug)] +struct HeldData { + inner: Arc, + armed: AtomicBool, + entered: tokio::sync::Notify, +} +impl std::fmt::Display for HeldData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HeldData") + } +} +#[async_trait::async_trait] +impl ObjectStore for HeldData { + async fn put_opts( + &self, + path: &Path, + payload: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + if path.as_ref().ends_with(".nodebundle") && self.armed.load(Ordering::Acquire) { + self.entered.notify_one(); + // A killed owner cannot select this immutable upload. Authority and + // enrollment operations still use the real strongly consistent store. + std::future::pending::<()>().await; + } + self.inner.put_opts(path, payload, options).await + } + async fn put_multipart_opts( + &self, + path: &Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts( + &self, + path: &Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.inner.get_opts(path, options).await + } + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +fn target() -> CellTarget { + CellTarget::new( + TenantId::from_bytes([61; 16]), + APPLICATION, + NamespaceId::from_bytes([63; 16]), + b"remote-follower-tail", + ) + .unwrap() +} +fn marker(name: &str, value: serde_json::Value) { + let root = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); + let path = root.join(name); + let mut file = std::fs::File::create(path).unwrap(); + use std::io::Write as _; + file.write_all(serde_json::to_string_pretty(&value).unwrap().as_bytes()) + .unwrap(); + file.sync_all().unwrap(); + println!("Remote fault marker: {name}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[ignore = "requires isolated RustFS, pinned TLS and an external driver owning three separate 1-vCPU/2-GiB VMs"] +async fn remote_crash_fixture_child() { + // Refuse an incomplete invocation rather than quietly substituting memory. + let role = std::env::var("CELLULE_REMOTE_FAULT_ROLE").unwrap(); + std::env::var("CELLULE_TEST_ENDPOINT").unwrap(); + let device = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); + let config = Config::from_env(false).unwrap().unwrap(); + let metrics = Arc::new(QueryMetrics::default()); + let (remote, prefix) = super::super::example_storage().await.unwrap(); + let held = Arc::new(HeldData { + inner: remote.inner().clone(), + armed: AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + }); + let store = if role == "owner" { + Store::new(held.clone()) + } else { + remote + }; + let layout = cellule_ltx::CellStorageLayout::new(store, prefix, *APPLICATION.as_bytes()); + if role == "follower" { + assert!(config.index == 1 || config.index == 2); + let port = 8080 + u16::from(config.index) * 10; + config + .serve_follower( + layout, + CODE, + format!("127.0.0.1:{port}").parse().unwrap(), + metrics, + ) + .await + .unwrap(); + return; + } + assert_eq!(config.index, 0); + let session = SessionId::from_bytes(*uuid::Uuid::now_v7().as_bytes()); + let pool = SqlWorkerPool::new(1, 8) + .unwrap() + .with_native_memory_limit(512 << 20) + .unwrap(); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + pool, + 64 << 20, + session, + Host::default(), + ) + .unwrap(); + runtime.install_telemetry(metrics.clone()).unwrap(); + let owner = config + .start_owner( + layout.clone(), + CODE, + session, + APPLICATION, + &runtime, + metrics.clone(), + ) + .await + .unwrap(); + let directory = owner.directory(); + let target = target(); + let catalog = CellCatalog::new(layout.clone(), target.tenant()); + let authority = CellAuthority::new(layout.clone()); + let replica = CellReplica::new( + layout.clone(), + *target.cell_id().as_bytes(), + *INCARNATION.as_bytes(), + Limits::default(), + ) + .unwrap(); + let current_owner = CellOwner { + session, + endpoint: owner.endpoint.clone(), + }; + if role == "owner" { + let proof = catalog + .provision(CatalogEntry::new(&target, CatalogRole::Application, CODE, 1).unwrap()) + .await + .unwrap(); + let initial = authority + .create_initial(&proof, INCARNATION, current_owner) + .await + .unwrap(); + let handle = runtime.bootstrap(proof, replica, authority.clone(), initial, + device.join("owner.sqlite"), |tx| { + tx.execute_batch("CREATE TABLE counter(value INTEGER NOT NULL); INSERT INTO counter VALUES(0)")?; + Ok(()) + }).await.unwrap(); + held.armed.store(true, Ordering::Release); + let now = clock().unwrap(); + let identity = MutationIdentity { + request_id: REQUEST, + issued_at_ms: now, + expires_at_ms: now.checked_add(120_000).unwrap(), + }; + let outcome = tokio::time::timeout( + Duration::from_secs(20), + handle.execute(identity, PAYLOAD, now, 1_024, 1_024, |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(1i64.to_be_bytes().to_vec())) + }), + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + outcome, + StoredOutcome::Success { + result: 1i64.to_be_bytes().to_vec(), + commit_sequence: 1 + } + ); + tokio::time::timeout(Duration::from_secs(5), held.entered.notified()) + .await + .unwrap(); + let original = authority.load(target.cell_id()).await.unwrap().unwrap(); + assert_eq!(original.value().ltx_root().unwrap().commit_sequence, 0); + let advertised = directory + .load(session, clock().unwrap()) + .await + .unwrap() + .unwrap(); + assert!(advertised.advertisement().log().unwrap().active()); + assert_eq!( + advertised + .advertisement() + .bundle_head() + .map(|head| head.selected_through()) + .unwrap_or(0), + 0 + ); + marker( + "acknowledged.json", + serde_json::json!({ + "session": uuid::Uuid::from_bytes(*session.as_bytes()).to_string(), + "issued_at_ms": identity.issued_at_ms, "expires_at_ms": identity.expires_at_ms, + "commit_sequence": 1, "bucket_commit_sequence": 0, + "native_epoch": advertised.advertisement().log().unwrap().epoch(), + "required_followers": advertised.advertisement().log().unwrap().members().len(), + "owner_expires_at_ms": advertised.advertisement().expires_at_ms(), + "held_bundle_upload": true + }), + ); + std::future::pending::<()>().await; + return; + } + assert_eq!(role, "recover"); + let before: serde_json::Value = + serde_json::from_str(&std::env::var("CELLULE_REMOTE_FAULT_ACK").unwrap()).unwrap(); + let predecessor = SessionId::from_bytes( + *uuid::Uuid::parse_str(before["session"].as_str().unwrap()) + .unwrap() + .as_bytes(), + ); + tokio::time::timeout(Duration::from_secs(60), async { + while directory + .is_live(predecessor, clock().unwrap()) + .await + .unwrap() + { + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .unwrap(); + // Expired advertisements cannot be loaded as live authority. The fencing + // CAS returns the exact original membership that recovery must account for. + let fenced = directory + .claim_expired(predecessor, session, clock().unwrap()) + .await + .unwrap(); + let members = fenced.log().unwrap().members().to_vec(); + assert_eq!(members.len(), 2); + let mut peers = Vec::new(); + for member in members { + if let Some(peer) = directory + .resolve_node(member, clock().unwrap()) + .await + .unwrap() + { + peers.push(peer); + } + } + let transport: Arc = Arc::new( + transport::Transport::new( + directory.clone(), + session, + config.tls().unwrap(), + peers, + metrics, + ) + .unwrap(), + ); + let recovery = NodeLogRecovery::from_fenced(transport, &fenced, Limits::default()).unwrap(); + let manifests = RecoveryManifestStore::new(layout, Limits::default()); + let inventory = recoverable_cells(&catalog, &authority, predecessor, 1) + .await + .unwrap(); + assert_eq!(inventory.len(), 1); + let completed = RecoveryCoordinator::new(recovery, manifests.clone()) + .recover_and_seal(&directory, fenced, inventory, clock().unwrap()) + .await + .unwrap(); + assert_eq!(completed.controls.len(), 1); + let proof = catalog.lookup(target.cell_id()).await.unwrap().unwrap(); + let control = completed.controls.into_iter().next().unwrap(); + let handle = runtime + .takeover_restored( + proof, + replica, + authority, + control, + completed.takeover, + manifests, + device.join("recovered.sqlite"), + current_owner, + ) + .await + .unwrap(); + let identity = MutationIdentity { + request_id: REQUEST, + issued_at_ms: before["issued_at_ms"].as_i64().unwrap(), + expires_at_ms: before["expires_at_ms"].as_i64().unwrap(), + }; + let retry = handle + .execute(identity, PAYLOAD, clock().unwrap(), 1_024, 1_024, |_| { + Err(Error::Control("recovered request executed twice")) + }) + .await + .unwrap(); + assert_eq!( + retry, + StoredOutcome::Success { + result: 1i64.to_be_bytes().to_vec(), + commit_sequence: 1 + } + ); + assert_eq!( + handle + .query(64, 64, |tx| { + let value: i64 = tx.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }) + .await + .unwrap(), + 1i64.to_be_bytes() + ); + handle.drain().await.unwrap(); + runtime.shutdown().await.unwrap(); + owner.stop().await.unwrap(); + marker( + "recovered.json", + serde_json::json!({ + "commit_sequence": 1, "value": 1, "exact_retry": true, + "reexecution": false, "runtime_joined": true, "owner_joined": true, + "follower_transport": "original signed mTLS capacity transport" + }), + ); +} diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index ce867c2d..9c6446fd 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -314,6 +314,43 @@ this test-barrier correction does not establish a performance gain or identify the original maintenance failure's cause. Artifacts are retained under `cellule-checkpoint-barrier-20261012-v27`. +Fresh hosted Rust run [38209589260](https://github.com/crabbuild/cellule/actions/runs/38209589260) +passed the full workspace, both original maintenance cases, MSRV and strict +lint at `23f035d`. The separate routing performance run retained a failure and +a source-identical repeat: + +| Forwarded writes, concurrency 1 | Main writes/s | PR writes/s | Main p95 / p99 | PR p95 / p99 | +| --- | ---: | ---: | ---: | ---: | +| [Attempt 1](https://github.com/crabbuild/cellule/actions/runs/38209589255/attempts/1), failed | 80.33 | 69.60 | 22.37 / 92.93 ms | 37.53 / 140.66 ms | +| [Attempt 2](https://github.com/crabbuild/cellule/actions/runs/38209589255/attempts/2), passed | 73.38 | 73.25 | 21.04 / 26.66 ms | 20.84 / 26.67 ms | + +Each attempt retains all four paired repeats against main `c79834d`, with the +same measurement harness and identical baseline/candidate binary hashes across +attempts. Comparisons were independently recomputed. No slow pair, threshold or +deadline was removed. Attempt 1 remains a performance failure; its regression +did not reproduce, and attempt 2 does not prove a fleet improvement. These are +single-Cell root-publication routing tests on hosted x86_64 Linux/RustFS, +without shared follower-log durability. They do not replace the three-VM +capacity ramp or its every-ACK audit. + +Two fresh [remote unpublished-tail cases](remote-follower-tail-verification.md) +also passed on three separate 1-vCPU/2-GiB VMs with native ext4. Each acknowledged +one write on both remote followers while bucket coverage remained zero, killed +the owner, waited for actual lease expiry, and recovered through the canonical +remote sealing/tail protocol into a fresh successor volume. Exact retry returned +the original result without executing its handler. Successor and followers +joined; all owned containers were terminal without OOM, and follower VM boots +and OOM counters were unchanged. Strict SQL-example test lint passed. The first +failed fixture invocation and all source/binary/resource evidence remain in +`cellule-remote-tail-20261012-v28`; successor enrollment then preceded the old +lease's expiry. These cases qualify one unpublished ACK each after process loss, +not physical power loss, peer partition, or throughput. + +The missing support VM and four raw TCP tunnels were restored. The matched +750-write/s comparison remains queued behind the unchanged idle-host gate and +exclusive fixture admission. The original full ramp, read/mixed tests and +conditional rollback remain outstanding. + The maintenance investigation also reproduces a distinct capture race: a real same-boot heartbeat between two directory observations caused `Fenced` through full-advertisement equality. Capture now retains the exact Open epoch, membership diff --git a/docs/remote-follower-tail-verification.md b/docs/remote-follower-tail-verification.md new file mode 100644 index 00000000..f0b5281d --- /dev/null +++ b/docs/remote-follower-tail-verification.md @@ -0,0 +1,66 @@ +# Remote unpublished-tail verification + +This fixture checks a follower-acknowledged write after owner process loss, +using the SQL example's original signed mTLS transport and the public recovery +coordinator. It requires three separate Linux VMs and a support Docker host. +It is an environment-dependent fault test, not a throughput benchmark. + +| Requirement | Verification | +| --- | --- | +| Three separate node VMs | Distinct Docker host IDs and kernel boot IDs | +| Each node has 1 vCPU and 2 GiB | Host inventory and container cgroup limits | +| Native durable storage | Fresh ext4 Docker volumes for SQLite and follower logs | +| Remote durability before publication | Both followers acknowledge; Cell root and selected bundle coverage remain zero | +| Owner process loss | SIGKILL after the ACK; successor uses a different fresh volume | +| Recovery authority | Actual predecessor lease expiry, fencing CAS, remote sealing and complete tail recovery | +| Exact retry | Original result and commit sequence; retry handler fails if executed | +| Joined resources | Successor and followers exit successfully; all owned containers become terminal without OOM | + +The support host runs the pinned RustFS provider. Configure loopback routes +between these hosts before running: every node must reach the provider at +`127.0.0.1:9000`, and the owner must reach the two followers at +`127.0.0.1:8090` and `127.0.0.1:8100`. Forward raw TCP; the application keeps +mTLS and signed peer authorization end to end. The support host needs two +available CPUs for its provider, and node VMs must have swap disabled. + +Build the SQL example's **test executable** for the node VMs' Linux architecture +in an isolated source snapshot. Use the pinned Rust image from +[the capacity driver](../scripts/perf/run.py), with `CARGO_TARGET_DIR` under the +mounted Workspace target directory. The build command is: + +```sh +cargo test --release --locked -p cellule-axum --example sql --no-run --message-format=json +``` + +Use the `compiler-artifact` executable for target `sql` with `profile.test=true`, +rather than the ordinary SQL server executable. The driver checks that it +contains the exact ignored `fleet::remote_fault_tests::remote_crash_fixture_child` +test. Supply that executable and the Docker contexts: + +```sh +python3 scripts/verify-remote-follower-tail.py \ + --artifacts "$HOME/Workspace/crabbuild-target/cellule-remote-tail" \ + --binary /absolute/path/to/linux-sql-test-executable \ + --context support-context \ + --node-contexts owner-context follower-1-context follower-2-context \ + --tag remote-tail-r1 +``` + +Use a new tag for each case. The driver creates private fixture certificates, +an isolated object prefix, and fresh provider/node volumes. It retains lifecycle +records, logs, native-storage/resource observations, and the recovered result +outside the source checkout. Completed containers and volumes remain available +for inspection. Run it separately from performance measurements on these hosts. + +Two fresh cases passed with one unpublished ACK per case on the three-VM +profile. The original failed fixture invocation is retained: successor +enrollment occurred before predecessor lease expiry, exceeding the example's +three-live-session bound. The corrected driver waits for actual expiry without +changing that bound or spoofing time. See +[the measurement record](publication-progress-measurements.md). + +This checks process loss, not physical power loss or remote-peer partitions. +The existing [device-backed fault tests](../crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs) +cover separate object-store TCP-partition cases. Neither fixture establishes +capacity, latency improvement, or the cause of the original intermittent +maintenance `Fenced` failures. diff --git a/scripts/verify-remote-follower-tail.py b/scripts/verify-remote-follower-tail.py new file mode 100755 index 00000000..eb382dad --- /dev/null +++ b/scripts/verify-remote-follower-tail.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import pathlib +import subprocess +import shutil +import time + +ROOT = pathlib.Path(__file__).resolve().parent.parent +parser = argparse.ArgumentParser(description='Verify a remote unpublished ACK after owner process loss on three separate 1-vCPU/2-GiB Docker hosts.') +parser.add_argument('--artifacts', type=pathlib.Path, required=True) +parser.add_argument('--binary', type=pathlib.Path, required=True, help='release SQL example test executable, not the ordinary SQL server') +parser.add_argument('--context', required=True, help='separate provider/client support Docker context') +parser.add_argument('--node-contexts', nargs=3, required=True, metavar=('OWNER', 'FOLLOWER1', 'FOLLOWER2')) +parser.add_argument('--tag', required=True, help='new isolated object prefix and container-name suffix') +args = parser.parse_args() +BASE = args.artifacts.resolve() +if BASE.is_relative_to(ROOT): + parser.error('evidence must be outside the source checkout') +BASE.mkdir(parents=True, exist_ok=True) +TAG = args.tag +if not TAG or any(c not in 'abcdefghijklmnopqrstuvwxyz0123456789-' for c in TAG): + parser.error('tag must contain lowercase letters, digits and hyphens') +CASE = BASE / TAG +CASE.mkdir(exist_ok=False) +SUPPORT = args.context +CONTEXTS = args.node_contexts +(BASE / 'bin').mkdir(exist_ok=True) +if args.binary.resolve() != BASE / 'bin/sql_fault': + shutil.copy2(args.binary, BASE / 'bin/sql_fault') +if not (BASE / 'tls').exists(): + subprocess.run(['python3', str(ROOT / 'scripts/generate-capacity-tls.py'), str(BASE / 'tls')], check=True) +for name in ['control.py', 'wait-store-ready.py']: + shutil.copy2(ROOT / 'scripts/perf' / name, BASE / name) +RUST = 'rust:1.98.1-bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e' +STORE = 'ghcr.io/rustfs/rustfs@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858' +SELECTOR = 'fleet::remote_fault_tests::remote_crash_fixture_child' +owned = [] +status = dict(phase='setup', started_at=time.time(), performance_qualification=False, + fault='owner process SIGKILL after remote fleet ACK before bundle publication', + transport='original pinned mTLS + signed grants + sealed remote tail', + scope='three independent 1-CPU/2-GiB VMs, native ext4; not physical power loss', + joins=[], ack_count=0, audit_count=0) + + +def save(): + temporary = CASE / 'lifecycle.json.tmp' + temporary.write_text(json.dumps(status, indent=2) + '\n') + temporary.replace(CASE / 'lifecycle.json') + + +def docker(context, *arguments, check=True, timeout=25): + result = subprocess.run(['docker', '--context', context, *arguments], text=True, + capture_output=True, timeout=timeout) + if check and result.returncode: + raise RuntimeError(f'Docker {context} {arguments[:2]} failed: {result.stderr}') + return result + + +def wait_until(predicate, seconds, label): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.5) + raise RuntimeError('Fixture guard expired: ' + label) + + +def native_snapshot(context, name): + code = """import pathlib,json,subprocess +p=pathlib.Path('/proc') +print(json.dumps(dict(boot_id=(p/'sys/kernel/random/boot_id').read_text().strip(), +memory=(p/'meminfo').read_text(),swaps=(p/'swaps').read_text(), +oom_kill=dict(x.split() for x in (p/'vmstat').read_text().splitlines())['oom_kill'], +filesystem=subprocess.check_output(['df','-PT','/scratch'],text=True), +memory_max=pathlib.Path('/sys/fs/cgroup/memory.max').read_text().strip(), +cpu_max=pathlib.Path('/sys/fs/cgroup/cpu.max').read_text().strip(), +memory_events=pathlib.Path('/sys/fs/cgroup/memory.events').read_text()))) +""" + data = json.loads(docker(context, 'exec', name, 'python3', '-c', code).stdout) + assert ' ext4 ' in data['filesystem'] and 'virtiofs' not in data['filesystem'] + assert data['memory_max'] == str(2 << 30) + assert data['cpu_max'].split() == ['100000', '100000'] + assert len(data['swaps'].splitlines()) == 1 + assert 'oom_kill 0' in data['memory_events'] + return data + + +def start_role(role, index): + context = CONTEXTS[index] + name = f'cellule-{TAG}-{role}-{index}' + volume = name + '-native' + if docker(context, 'volume', 'inspect', volume, check=False).returncode == 0: + raise RuntimeError('Native state volume is not fresh: ' + volume) + docker(context, 'volume', 'create', '--label', 'cellule.fault.artifacts=' + str(CASE), volume) + args = ['run', '-d', '--name', name, '--label', 'cellule.fault.artifacts=' + str(CASE), + '--network', 'host', '--cpus', '1', '--memory', '2g', '--memory-swap', '2g', + '--ulimit', 'nofile=65536:65536', '-v', volume + ':/scratch', '-v', str(BASE) + ':/work:ro', + '-e', 'AWS_ACCESS_KEY_ID=benchmark_access', '-e', 'AWS_SECRET_ACCESS_KEY=benchmark_secret_private', + '-e', 'AWS_DEFAULT_REGION=us-east-1', '-e', 'CELLULE_TEST_ENDPOINT=http://127.0.0.1:9000', + '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', 'CELLULE_TEST_PREFIX=' + TAG, + '-e', 'CELLULE_TEST_DEVICE_PATH=/scratch', '-e', 'TMPDIR=/scratch', + '-e', 'CELLULE_AXUM_FLEET_DIR=/scratch/fleet', '-e', 'CELLULE_REMOTE_FAULT_ROLE=' + role, + '-e', 'TOKIO_WORKER_THREADS=1', '-e', 'RUST_BACKTRACE=1'] + if index: + args += ['-e', 'CELLULE_AXUM_FOLLOWER=' + str(index)] + if role == 'recover': + args += ['-e', 'CELLULE_REMOTE_FAULT_ACK=' + json.dumps(status['ack'])] + args += [RUST, 'sh', '-c', f'mkdir -m 700 /scratch/fleet && cp /work/tls/ca.crt /work/tls/node-{index}.crt /work/tls/node-{index}.key /scratch/fleet/ && exec /work/bin/sql_fault {SELECTOR} --exact --ignored --nocapture'] + # Track ownership before the RPC: a failed Docker response can be ambiguous. + owned.append((context, name)) + docker(context, *args) + status.setdefault('roles', []).append(dict(role=role, index=index, context=context, name=name, volume=volume)) + save() + return context, name + + +def marker(context, name, file): + result = docker(context, 'exec', name, 'cat', '/scratch/' + file, check=False) + return json.loads(result.stdout) if result.returncode == 0 else None + + +def joined(context, name, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + info = json.loads(docker(context, 'inspect', name).stdout)[0] + if not info['State']['Running']: + status['joins'].append(dict(context=context, name=name, state=info['State'])) + assert not info['State']['OOMKilled'] + return info['State']['ExitCode'] + time.sleep(0.5) + raise RuntimeError('Process did not join: ' + name) + + +try: + save() + listed = docker(SUPPORT, 'run', '--rm', '-v', str(BASE) + ':/work:ro', RUST, '/work/bin/sql_fault', SELECTOR, '--exact', '--ignored', '--list').stdout + assert SELECTOR + ': test' in listed and '1 test, 0 benchmarks' in listed + status['binary_sha256'] = hashlib.sha256((BASE / 'bin/sql_fault').read_bytes()).hexdigest() + info = {context: json.loads(docker(context, 'info', '--format', '{{json .}}').stdout) + for context in CONTEXTS} + assert len({value['ID'] for value in info.values()}) == 3 + assert all(value['NCPU'] == 1 and 1.8 * (1 << 30) < value['MemTotal'] <= 2 << 30 + for value in info.values()) + (CASE / 'node-hosts.json').write_text(json.dumps(info, indent=2) + '\n') + store = 'cellule-' + TAG + '-store' + volume = store + '-native' + assert docker(SUPPORT, 'volume', 'inspect', volume, check=False).returncode != 0 + docker(SUPPORT, 'volume', 'create', '--label', 'cellule.fault.artifacts=' + str(CASE), volume) + owned.append((SUPPORT, store)) + docker(SUPPORT, 'run', '-d', '--name', store, '--label', 'cellule.fault.artifacts=' + str(CASE), + '--network', 'host', '--cpus', '2', '--cpuset-cpus', '0-1', '--memory', '2g', + '--memory-swap', '2g', '-v', volume + ':/data', '-e', 'RUSTFS_ACCESS_KEY=benchmark_access', + '-e', 'RUSTFS_SECRET_KEY=benchmark_secret_private', '-e', 'MIMALLOC_ALLOW_THP=0', STORE, '/data') + control = BASE / 'control.py' + wait_until(lambda: docker(SUPPORT, 'run', '--rm', '--network', 'host', '-v', + str(control) + ':/control.py:ro', RUST, 'python3', '/control.py', + 'bucket', check=False).returncode == 0, 60, 'real bucket creation') + ready = BASE / 'wait-store-ready.py' + result = docker(SUPPORT, 'run', '--rm', '--network', 'host', '-v', str(ready) + ':/ready.py:ro', + RUST, 'python3', '/ready.py', timeout=70) + (CASE / 'provider-ready.json').write_text(result.stdout) + followers = [start_role('follower', index) for index in (1, 2)] + for context, name in followers: + wait_until(lambda: 'Follower service:' in docker(context, 'logs', name).stdout, + 60, 'follower service ready') + owner = start_role('owner', 0) + status['phase'] = 'awaiting_unpublished_remote_ack' + save() + wait_until(lambda: marker(*owner, 'acknowledged.json') is not None, 45, 'remote ACK') + status['ack'] = marker(*owner, 'acknowledged.json') + assert status['ack']['commit_sequence'] == 1 and status['ack']['bucket_commit_sequence'] == 0 + assert status['ack']['required_followers'] == 2 and status['ack']['held_bundle_upload'] + status['ack_count'] = 1 + before = {name: native_snapshot(context, name) for context, name in [owner, *followers]} + assert len({value['boot_id'] for value in before.values()}) == 3 + (CASE / 'native-before-crash.json').write_text(json.dumps(before, indent=2) + '\n') + status['phase'] = 'killing_owner' + save() + docker(*owner[:1], 'kill', '--signal', 'KILL', owner[1]) + assert joined(*owner, 10) == 137 + # The embedding example bounds its live inventory to the three-node fleet. + # Enroll the replacement only after actual old-boot lease expiry, using + # wall time; never fake a future recovery timestamp or relax that bound. + status['phase'] = 'waiting_real_owner_lease_expiry' + save() + wait_until(lambda: time.time() * 1000 > status['ack']['owner_expires_at_ms'] + 1000, + 40, 'real predecessor lease expiry before successor enrollment') + successor = start_role('recover', 0) + successor_native = native_snapshot(*successor) + assert successor_native['boot_id'] == before[owner[1]]['boot_id'] + assert successor_native['oom_kill'] == before[owner[1]]['oom_kill'] + (CASE / 'native-successor.json').write_text(json.dumps(successor_native, indent=2) + '\n') + status['phase'] = 'recovering_remote_tail' + save() + assert joined(*successor, 100) == 0 + # Read the marker from the stopped container without reusing its database. + docker(successor[0], 'cp', successor[1] + ':/scratch/recovered.json', str(CASE / 'recovered.json')) + recovered = json.loads((CASE / 'recovered.json').read_text()) + assert recovered['commit_sequence'] == recovered['value'] == 1 + assert recovered['exact_retry'] and not recovered['reexecution'] + assert recovered['runtime_joined'] and recovered['owner_joined'] + status['audit_count'] = 1 + for context, name in followers: + after = native_snapshot(context, name) + assert after['boot_id'] == before[name]['boot_id'] + assert after['oom_kill'] == before[name]['oom_kill'] + status.setdefault('native_followers_after_recovery', {})[name] = after + docker(context, 'kill', '--signal', 'INT', name) + assert joined(context, name, 90) == 0 + status['phase'] = 'passed' +except Exception as error: + status.update(failure_phase=status['phase'], phase='failed', error=repr(error)) +finally: + # Even a failed scenario has to join every accepted process and retain logs. + cleanup_errors = [] + for context, name in owned: + try: + inspection = docker(context, 'inspect', name, check=False) + if inspection.returncode: + if 'No such object' in inspection.stderr or 'No such container' in inspection.stderr: + continue + raise RuntimeError(inspection.stderr) + info = json.loads(inspection.stdout)[0] + if info['State']['Running']: + docker(context, 'stop', '--time', '10', name, timeout=25) + result = docker(context, 'logs', name, check=False) + (CASE / (name + '.log')).write_text(result.stdout + result.stderr) + terminal = json.loads(docker(context, 'inspect', name).stdout)[0] + assert not terminal['State']['Running'] and not terminal['State']['OOMKilled'] + (CASE / (name + '-terminal.json')).write_text(json.dumps(terminal, indent=2) + '\n') + except Exception as error: + cleanup_errors.append(dict(context=context, name=name, error=repr(error))) + if cleanup_errors: + status.update(phase='failed', cleanup_errors=cleanup_errors) + status.update(finished_at=time.time(), all_owned_containers_terminal=not cleanup_errors) + save() +print(json.dumps(status, indent=2)) +raise SystemExit(0 if status['phase'] == 'passed' else 1) From 9e68a3e8ec1d528f396d2fd5d226a492fb77b889 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 09:00:21 -0700 Subject: [PATCH 24/31] fix: renew recovery claims during long verification --- .../docs/failover-and-followers.md | 6 + .../src/node/bundle/recovery/drain.rs | 61 ++++----- .../src/node/bundle/tests/faults.rs | 10 ++ .../src/node/bundle/tests/recovery/mod.rs | 43 +++++- .../cellule-runtime/src/node/directory/mod.rs | 5 +- .../src/node/directory/recovery.rs | 71 ++++++++-- .../src/node/log_recovery/claim.rs | 126 ++++++++++++++++++ .../src/node/log_recovery/mod.rs | 54 +++++--- .../src/node/log_shipper/replication/mod.rs | 14 ++ .../src/node/tests/candidates.rs | 6 + docs/publication-progress-measurements.md | 100 +++++++++++++- 11 files changed, 416 insertions(+), 80 deletions(-) create mode 100644 crates/cellule-runtime/src/node/log_recovery/claim.rs diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index a90234b7..ce023daa 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -1640,6 +1640,12 @@ uploading or attaching a manifest grants no transfer authority. The public lineage and Cell CAS, including quiet bindings. It stages bounded catalog cohorts and selects their complete terminal inventory with one recovery-claim node CAS before log seal. Claims are rechecked after I/O; exact lost replies reconcile. +The managed path renews the same claim generation while verification and +materialization run, using the existing 30-second lifetime. Renewal shares the +terminal node CAS lock and ends with the attempt, including cancellation. A dead +claimant, changed proof or expired claim still rejects. Terminal inventory +verification runs outside that lock; sealing checks its exact immutable head +against fresh authority before the final CAS. Partial-root retries reuse the original digest-verified manifest and match every remaining tail's scope, predecessor, complete physical/logical range and bundle digest. A materialized root ahead of the old catalog is only a verified base; diff --git a/crates/cellule-runtime/src/node/bundle/recovery/drain.rs b/crates/cellule-runtime/src/node/bundle/recovery/drain.rs index 4f0f4414..b429571d 100644 --- a/crates/cellule-runtime/src/node/bundle/recovery/drain.rs +++ b/crates/cellule-runtime/src/node/bundle/recovery/drain.rs @@ -1,6 +1,6 @@ //! Terminal catalog selection owned by a verified complete recovery attempt. use super::*; -use crate::node::log_recovery::RecoveryCell; +use crate::node::log_recovery::{RecoveryCell, RecoveryClaimLease}; use crate::recovery::manifest::RecoveryManifestStore; use std::collections::BTreeMap; @@ -30,6 +30,7 @@ impl BundleRecoveryDrain { cells: &[RecoveryCell], durable_through: u64, limits: cellule_ltx::Limits, + claim: Option<&RecoveryClaimLease<'_>>, ) -> Result> { let Some(original) = fenced.bundle_head() else { return Ok(None); @@ -42,7 +43,15 @@ impl BundleRecoveryDrain { let mut scopes = BTreeMap::new(); let mut closed = BTreeMap::new(); let mut unpinned = BTreeMap::new(); - for binding in fenced_bindings(layout, fenced).await? { + let bindings = if let Some(claim) = claim { + // Renewal changes expiry, while the fenced immutable head stays + // fixed. Check the lease's exact current proof before reading it. + claim.verify().await?; + index::binding_inventory(layout, fenced.session(), original).await? + } else { + fenced_bindings(layout, fenced).await? + }; + for binding in bindings { if binding.phase == BindingPhase::Closed { if binding.terminal != Some(( @@ -216,9 +225,8 @@ impl BundleRecoveryDrain { directory: &crate::node::NodeDirectory, manifests: &RecoveryManifestStore, fenced: &FencedNodeSession, - now_ms: i64, - started: std::time::Instant, - ) -> Result<(FencedNodeSession, Vec)> { + claim: &RecoveryClaimLease<'_>, + ) -> Result> { if fenced.bundle_head() != Some(self.original) || directory.layout.node_path(fenced.session().as_bytes()) != manifests.layout().node_path(fenced.session().as_bytes()) @@ -229,7 +237,7 @@ impl BundleRecoveryDrain { } let mut controls = Vec::with_capacity(self.scopes.len()); for scope in self.scopes.values() { - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + claim.verify().await?; let authority = &scope.cell.authority; let mut current = authority .load(scope.binding.control.cell) @@ -259,7 +267,7 @@ impl BundleRecoveryDrain { let successor = current .value() .publish_recovery(&prepared, current.value().next_due_ms)?; - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + claim.verify().await?; current = match authority .transition( ¤t, @@ -349,15 +357,14 @@ impl BundleRecoveryDrain { catalog.selected_through = self.durable_through; let prepared = directory.upload_catalog(Some(head), catalog, &[]).await?; staged = prepared.head; - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + claim.verify().await?; } // Intermediate cohort objects are immutable staging, never authority. // Select the fully closed inventory with one original-claim node CAS. store::ensure_session_drained(manifests.layout(), fenced.session(), Some(staged)).await?; - let (mut current, token) = - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + let (mut proof, mut current, token) = claim.lock_verified().await?; if staged == self.original { - return Ok((fenced.clone(), controls)); + return Ok(controls); } current.bundle = Some(staged); let path = manifests.layout().node_path(fenced.session().as_bytes()); @@ -370,11 +377,15 @@ impl BundleRecoveryDrain { { // Lost reply only reconciles this exact immutable head, original // claim generation, expiry and recovering log. - check_claim(directory, &fresh, logical_now(now_ms, started)?) - .await - .map_err(|_| Error::from(source))?; + let actual = directory.load_record_at(&path).await?; + if !matches!(actual, Some((NodeRecord::Tombstone(current), _)) if current.fenced().is_ok_and(|current| current == fresh)) + || claim.now()? >= fresh.claim_expires_at_ms() + { + return Err(source.into()); + } } - Ok((fresh, controls)) + *proof = fresh; + Ok(controls) } } @@ -387,26 +398,6 @@ fn same_scope(original: &Control, current: &Control) -> bool { && original.schema == current.schema } -async fn check_claim( - directory: &crate::node::NodeDirectory, - fenced: &FencedNodeSession, - now_ms: i64, -) -> Result<(crate::node::directory::NodeTombstone, cellule_store::ETag)> { - directory - .load(fenced.claimant(), now_ms) - .await? - .ok_or(Error::Fenced)?; - let path = directory.layout.node_path(fenced.session().as_bytes()); - let Some((NodeRecord::Tombstone(current), token)) = directory.load_record_at(&path).await? - else { - return Err(Error::Fenced); - }; - if current.fenced()? != *fenced || now_ms >= fenced.claim_expires_at_ms() { - return Err(Error::Fenced); - } - Ok((*current, token)) -} - pub(crate) fn logical_now(now_ms: i64, started: std::time::Instant) -> Result { now_ms .checked_add( diff --git a/crates/cellule-runtime/src/node/bundle/tests/faults.rs b/crates/cellule-runtime/src/node/bundle/tests/faults.rs index fadc2927..2ba8fe40 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/faults.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/faults.rs @@ -122,6 +122,16 @@ impl ObjectStore for ReplyFault { } async fn get_opts(&self, path: &Path, opts: GetOptions) -> object_store::Result { let mode = self.mode.load(Ordering::SeqCst); + if mode == 17 + && path.as_ref().ends_with(".root") + && self + .mode + .compare_exchange(17, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + // Exercise a real I/O wait beyond the production claim lifetime. + tokio::time::sleep(std::time::Duration::from_secs(33)).await; + } if mode == 16 && path.as_ref().ends_with("/control.json") { self.control_started.fetch_add(1, Ordering::SeqCst); self.node_started.notify_one(); diff --git a/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs index 495fb8ea..e90608ec 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs @@ -125,6 +125,7 @@ async fn fence(f: &Fixture) -> crate::node::FencedNodeSession { enum Scenario { Pruned, Complete, + SlowRecovery, PartialRoot, LostRootReply, LostCatalogReply, @@ -141,6 +142,7 @@ impl Scenario { matches!( self, Self::Complete + | Self::SlowRecovery | Self::PartialRoot | Self::LostRootReply | Self::LostCatalogReply @@ -173,6 +175,11 @@ async fn recovery_materializes_full_fleet_suffix_and_closes_quiet_binding_before verify_recovery(Scenario::Complete).await; } +#[tokio::test] +async fn live_claimant_renews_recovery_during_root_verification_beyond_claim_lifetime() { + verify_recovery(Scenario::SlowRecovery).await; +} + #[tokio::test] async fn recovery_retries_after_one_root_cas_without_replacing_the_original_manifest() { verify_recovery(Scenario::PartialRoot).await; @@ -617,10 +624,38 @@ async fn verify_recovery(scenario: Scenario) { cells = discovered.cells; coordinator = RecoveryCoordinator::new(resumed, manifests.clone()); } - let completed = coordinator - .recover_and_seal(&f.directory, completion_fence, cells, NOW + 31_002) - .await - .unwrap(); + let completed = if matches!(scenario, Scenario::SlowRecovery) { + faults.mode.store(17, std::sync::atomic::Ordering::SeqCst); + let now = NOW + 31_002; + let started = std::time::Instant::now(); + let mut claimant = f + .directory + .load(completion_fence.claimant(), now) + .await + .unwrap() + .unwrap(); + let recovery = coordinator.recover_and_seal(&f.directory, completion_fence, cells, now); + tokio::pin!(recovery); + loop { + tokio::select! { + result = &mut recovery => break result.unwrap(), + () = tokio::time::sleep(std::time::Duration::from_secs(5)) => { + let current_now = now + i64::try_from(started.elapsed().as_millis()).unwrap(); + let mut next = claimant.advertisement().clone(); + next.issued_at_ms = current_now; + next.expires_at_ms = current_now + 30_000; + next.progress += 1; + next.signature = SigningKey::from_bytes(&[10; 32]).sign(&next.signing_bytes().unwrap()).to_bytes(); + claimant = f.directory.refresh(&claimant, next, current_now).await.unwrap(); + } + } + } + } else { + coordinator + .recover_and_seal(&f.directory, completion_fence, cells, NOW + 31_002) + .await + .unwrap() + }; assert_eq!( completed.controls.len(), if matches!(scenario, Scenario::SealInterrupted) { diff --git a/crates/cellule-runtime/src/node/directory/mod.rs b/crates/cellule-runtime/src/node/directory/mod.rs index c277627a..806ec409 100644 --- a/crates/cellule-runtime/src/node/directory/mod.rs +++ b/crates/cellule-runtime/src/node/directory/mod.rs @@ -586,10 +586,7 @@ impl NodeTombstone { } pub(super) fn renew(mut self, fenced: &FencedNodeSession, now_ms: i64) -> Result { - if self.session != fenced.session - || self.claimant != Some(fenced.claimant) - || self.claim_generation != fenced.claim_generation - { + if self.fenced()? != *fenced { return Err(Error::Fenced); } let current_expiry = self diff --git a/crates/cellule-runtime/src/node/directory/recovery.rs b/crates/cellule-runtime/src/node/directory/recovery.rs index c642dbf2..01b919a7 100644 --- a/crates/cellule-runtime/src/node/directory/recovery.rs +++ b/crates/cellule-runtime/src/node/directory/recovery.rs @@ -6,6 +6,14 @@ use super::*; +/// Verified immutable closure, bound to this store and exact failed-session head. +pub(crate) struct PreparedRecoverySeal { + session: SessionId, + head: Option, + path: object_store::path::Path, + store_identity: u64, +} + impl NodeDirectory { /// Reads live scheduler membership and primes the advisory recovery scan. /// @@ -501,34 +509,72 @@ impl NodeDirectory { }; let renewed = (*current).renew(fenced, now_ms)?; let proof = renewed.fenced()?; - self.layout + if let Err(source) = self + .layout .store() .update(&path, Bytes::from(renewed.encode()?), token) - .await?; + .await + { + // A lost reply can credit only this exact renewal, never another + // claimant, generation, log or terminal catalog selection. + let actual = self.load_record_at(&path).await?; + if !matches!(actual, Some((NodeRecord::Tombstone(current), _)) if current.fenced().is_ok_and(|current| current == proof)) + { + return Err(source.into()); + } + } Ok(proof) } + pub(crate) async fn prepare_recovery_seal( + &self, + fenced: &FencedNodeSession, + ) -> Result { + crate::node::bundle::store::ensure_session_drained( + &self.layout, + fenced.session(), + fenced.bundle_head(), + ) + .await?; + Ok(PreparedRecoverySeal { + session: fenced.session(), + head: fenced.bundle_head(), + path: self.layout.node_path(fenced.session().as_bytes()), + store_identity: self.layout.immutable_cache_identity(), + }) + } + /// Seals an exact recovery claim after every affected Cell pins its overlay. pub(crate) async fn seal_recovery( &self, fenced: &FencedNodeSession, recovery_manifest: Option, now_ms: i64, + ) -> Result { + let prepared = self.prepare_recovery_seal(fenced).await?; + self.seal_recovery_prepared(fenced, recovery_manifest, now_ms, &prepared) + .await + } + + pub(crate) async fn seal_recovery_prepared( + &self, + fenced: &FencedNodeSession, + recovery_manifest: Option, + now_ms: i64, + prepared: &PreparedRecoverySeal, ) -> Result { let path = self.layout.node_path(fenced.session.as_bytes()); let Some((NodeRecord::Tombstone(current), token)) = self.load_record_at(&path).await? else { return Err(Error::Fenced); }; - // A recovery manifest is not terminal bundle closure. Keep the whole - // failed boot fenced until every original binding is materialized and - // checkpointed; lower-level callers cannot bypass the coordinator. - crate::node::bundle::store::ensure_session_drained( - &self.layout, - current.session, - current.bundle, - ) - .await?; + if prepared.session != current.session + || prepared.head != current.bundle + || prepared.path != path + || prepared.store_identity != self.layout.immutable_cache_identity() + { + return Err(Error::Fenced); + } if let Some(log) = ¤t.log && matches!(log.phase(), NodeLogPhase::Sealed | NodeLogPhase::Retired) && log.recovery_manifest() == recovery_manifest @@ -538,6 +584,9 @@ impl NodeDirectory { log: log.clone(), }); } + if current.fenced()? != *fenced { + return Err(Error::Fenced); + } let sealed = (*current).seal(fenced, recovery_manifest, now_ms)?; let log = sealed .log diff --git a/crates/cellule-runtime/src/node/log_recovery/claim.rs b/crates/cellule-runtime/src/node/log_recovery/claim.rs new file mode 100644 index 00000000..9f112ff3 --- /dev/null +++ b/crates/cellule-runtime/src/node/log_recovery/claim.rs @@ -0,0 +1,126 @@ +//! One cancellation-scoped renewal loop for a managed recovery attempt. +use super::*; +use crate::node::directory::{NodeRecord, NodeTombstone}; +use tokio::sync::{Mutex, MutexGuard}; + +pub(crate) struct RecoveryClaimLease<'a> { + directory: &'a NodeDirectory, + current: Mutex, + now_ms: i64, + started: std::time::Instant, +} + +impl<'a> RecoveryClaimLease<'a> { + pub(crate) async fn new( + directory: &'a NodeDirectory, + fenced: FencedNodeSession, + now_ms: i64, + ) -> Result { + let lease = Self { + directory, + current: Mutex::new(fenced), + now_ms, + started: std::time::Instant::now(), + }; + lease.verify().await?; + Ok(lease) + } + + pub(crate) fn now(&self) -> Result { + crate::node::bundle::recovery::logical_now(self.now_ms, self.started) + } + + pub(in crate::node) async fn lock_verified( + &self, + ) -> Result<( + MutexGuard<'_, FencedNodeSession>, + NodeTombstone, + cellule_store::ETag, + )> { + let current = self.current.lock().await; + let (record, token) = self.check(¤t).await?; + Ok((current, record, token)) + } + + pub(crate) async fn verify(&self) -> Result { + let (current, _, _) = self.lock_verified().await?; + Ok(current.clone()) + } + + async fn check( + &self, + fenced: &FencedNodeSession, + ) -> Result<(NodeTombstone, cellule_store::ETag)> { + self.directory + .load(fenced.claimant(), self.now()?) + .await? + .ok_or(Error::Fenced)?; + let path = self.directory.layout.node_path(fenced.session().as_bytes()); + let Some((NodeRecord::Tombstone(current), token)) = + self.directory.load_record_at(&path).await? + else { + return Err(Error::Fenced); + }; + if current.fenced()? != *fenced || self.now()? >= fenced.claim_expires_at_ms() { + return Err(Error::Fenced); + } + Ok((*current, token)) + } + + async fn keep_alive(&self) -> Result<()> { + loop { + let expiry = self.current.lock().await.claim_expires_at_ms(); + let lifetime = crate::node::log_state::RECOVERY_CLAIM_LIFETIME_MS; + let delay = expiry + .saturating_sub(self.now()?) + .saturating_sub(lifetime * 2 / 3); + tokio::time::sleep(std::time::Duration::from_millis( + u64::try_from(delay.max(1)) + .map_err(|_| Error::Capacity("recovery renewal delay"))?, + )) + .await; + // Renewal and terminal node CAS share this lock. Immutable data + // verification and Cell materialization never hold it. + let (mut current, _, _) = self.lock_verified().await?; + *current = self + .directory + .refresh_recovery_claim(¤t, self.now()?) + .await?; + self.check(¤t).await?; + } + } + + pub(crate) async fn run( + &self, + work: impl std::future::Future>, + ) -> Result { + // No detached heartbeat survives success, failure or cancellation. + tokio::select! { + biased; + result = work => result, + result = self.keep_alive() => { + result?; + Err(Error::Node("recovery renewal loop stopped")) + } + } + } + + pub(crate) async fn seal(&self, manifest: Option) -> Result { + let snapshot = self.verify().await?; + let prepared = self.directory.prepare_recovery_seal(&snapshot).await?; + // The immutable terminal inventory was checked outside the renewal + // lock. Its exact head is checked again by the final authority CAS. + let (current, _, _) = self.lock_verified().await?; + let sealed = self + .directory + .seal_recovery_prepared(¤t, manifest, self.now()?, &prepared) + .await?; + let takeover = NodeTakeoverProof::after_recovery(¤t, &sealed)?; + Ok(CompletedNodeRecoverySeal { sealed, takeover }) + } +} + +pub(crate) struct CompletedNodeRecoverySeal { + pub(crate) sealed: SealedNodeLog, + pub(crate) takeover: NodeTakeoverProof, +} diff --git a/crates/cellule-runtime/src/node/log_recovery/mod.rs b/crates/cellule-runtime/src/node/log_recovery/mod.rs index e2046937..56a02b28 100644 --- a/crates/cellule-runtime/src/node/log_recovery/mod.rs +++ b/crates/cellule-runtime/src/node/log_recovery/mod.rs @@ -25,12 +25,14 @@ const MAX_RECOVERY_CATALOG_HEAD_READS: usize = 32; const MAX_RECOVERY_PAGE_BYTES: u64 = 1 << 20; const MAX_RECOVERY_PAGE_FRAMES: usize = 4_096; +mod claim; pub mod retirement; mod tail; mod witness; use tail::*; +pub(crate) use claim::RecoveryClaimLease; use witness::*; /// Bounded work counters emitted for one node-log recovery attempt. @@ -537,7 +539,10 @@ impl RecoveryCoordinator { cells: Vec, sealed: SealedSession, ) -> Result { - Ok(self.recover_attempt(fenced, cells, sealed).await?.result) + Ok(self + .recover_attempt(fenced, cells, sealed, None) + .await? + .result) } async fn recover_attempt( @@ -545,6 +550,7 @@ impl RecoveryCoordinator { fenced: FencedNodeSession, cells: Vec, sealed: SealedSession, + claim: Option<&RecoveryClaimLease<'_>>, ) -> Result { self.recovery.validate_fence(&fenced)?; let mut bases = Vec::with_capacity(cells.len()); @@ -583,6 +589,7 @@ impl RecoveryCoordinator { &cells, sealed.durable_through, self.recovery.limits, + claim, ) .await?; if let Some(drain) = &drain { @@ -741,27 +748,32 @@ impl RecoveryCoordinator { cells: Vec, now_ms: i64, ) -> Result { - let started = std::time::Instant::now(); self.recovery.validate_fence(&fenced)?; - let witness = self.recovery.ensure_sealed_bounded().await?; - let attempt = self.recover_attempt(fenced.clone(), cells, witness).await?; - let Some(drain) = attempt.drain else { - return self - .finish(directory, fenced, attempt.result.controls, now_ms) - .await; - }; - let manifest = pinned_manifest(&attempt.result.controls, &fenced, self.recovery.log_epoch)?; - let (fresh, controls) = drain - .complete(directory, &self.manifests, &fenced, now_ms, started) - .await?; - let now = crate::node::bundle::recovery::logical_now(now_ms, started)?; - let sealed = directory.seal_recovery(&fresh, manifest, now).await?; - let takeover = NodeTakeoverProof::after_recovery(&fenced, &sealed)?; - Ok(CompletedNodeRecovery { - sealed, - controls, - takeover, - }) + let claim = RecoveryClaimLease::new(directory, fenced.clone(), now_ms).await?; + claim + .run(async { + let witness = self.recovery.ensure_sealed_bounded().await?; + let attempt = self + .recover_attempt(fenced.clone(), cells, witness, Some(&claim)) + .await?; + let manifest = + pinned_manifest(&attempt.result.controls, &fenced, self.recovery.log_epoch)?; + let controls = match attempt.drain { + Some(drain) => { + drain + .complete(directory, &self.manifests, &fenced, &claim) + .await? + } + None => attempt.result.controls, + }; + let completed = claim.seal(manifest).await?; + Ok(CompletedNodeRecovery { + sealed: completed.sealed, + controls, + takeover: completed.takeover, + }) + }) + .await } /// Seals one still-current claim after all recovered controls were pinned. diff --git a/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs b/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs index 7433265a..04af7cfb 100644 --- a/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs +++ b/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs @@ -211,6 +211,20 @@ async fn run_member( ) .await .map_err(Arc::new); + if let Err(error) = &result { + tracing::warn!( + leader_session = ?leader, + log_epoch = epoch, + member = ?member, + first_sequence = requests.first().map(|request| request.first), + last_sequence = last, + covered_through = covered, + frame_count, + bytes, + error = ?error, + "follower group append failed" + ); + } // One canonical follower append covers the delivered group and its // single fsync. Every original request receives the same verified watermark. for request in requests { diff --git a/crates/cellule-runtime/src/node/tests/candidates.rs b/crates/cellule-runtime/src/node/tests/candidates.rs index 4b786df8..ccc97e0e 100644 --- a/crates/cellule-runtime/src/node/tests/candidates.rs +++ b/crates/cellule-runtime/src/node/tests/candidates.rs @@ -249,6 +249,12 @@ async fn expired_enrolled_log_becomes_a_renewable_recovery_claim() { .unwrap(); assert_eq!(renewed.claim_generation(), fenced.claim_generation()); assert!(renewed.claim_expires_at_ms() > fenced.claim_expires_at_ms()); + assert!(matches!( + directory + .refresh_recovery_claim(&fenced, NOW_MS + 15_001) + .await, + Err(Error::Fenced) + )); let sealed = directory .seal_recovery(&renewed, None, NOW_MS + 15_001) .await diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 9c6446fd..3589d7dc 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -6,7 +6,65 @@ uses an 8-vCPU owner. Fresh verification on three separate 1-vCPU/2-GiB VMs has not passed latency and backlog qualification. The separate 2K-write/20K-read qualification remains open. -## Fresh PR #76 write verification +## Latest 750-write/s PR #76 verification + +October 12, 2026 UTC. The fresh matched cases use 2,000 Cells and three +separate 1-vCPU/2-GiB Linux VMs, with native disk-backed node volumes. Both +use 96-byte SQL ledger rows, a 30-second warmup, a 300-second write window, +the unchanged 64-MiB retained-work budget and 128/128 client limits. Workload, +auditor and runner bytes match. Main `c79834d` includes only the common +request-deadline correction; the PR binary is built from `a0860db`, whose +production code matches PR head `89d4599`. + +| Fresh run | Completed inside window / offered | Errors / dropped offers | Scheduled p50 / p95 / p99, all attempts | Warm / cold ACK audits | +| --- | ---: | ---: | --- | --- | +| Main + common deadline correction | 85,803 / 225,000 (286.01/s) | 61,824 / 77,258 | 89.2 / 1,280.4 / 1,966.6 ms | All 99,711 pass / all 99,711 pass | +| PR #76 | 98,015 / 225,000 (326.72/s) | 405 / 126,580 | 279.3 / 2,463.0 / 4,110.5 ms | 110,861 checked, 73,262 errors / not reached | + +Successful-write scheduled p50/p95/p99, calculated directly from the request +journals using nearest rank, are 106.488/1,269.665/1,868.134 ms on main and +277.601/2,434.809/3,933.745 ms on the PR. All failed attempts and drops remain +qualification failures. These rates are delivered throughput at 750 offered +writes/s; they do not establish sustainable capacity. + +Both cases fail the existing delivery, 50-ms p99 and publication-backlog gates. +The last three one-minute segments have positive unpublished-byte slopes of +20,225 B/s on main and 205,167 B/s on the PR, and positive oldest-debt-age +slopes. Main's joined drain takes 90.62 seconds. The PR's warm audit reports +HTTP 503 fencing refusals; its owner then exceeds the unchanged 120-second +drain deadline and is stopped with exit 137. Both followers exit 0. No node +reports an OOM or swap activity. A refused audit response does not establish +that the acknowledged data was lost. + +The PR records one follower-append failure near the end of the timed window, +disables fleet proofs and enters rotation. The original failure cause was not +logged at that boundary. A failure-only warning now preserves the member, +sequence range and original error for the next reproduction; successful appends +do not emit it. The original case, all 110,861 ACKs, provider objects and native +volumes remain retained for canonical recovery investigation. + +Each window has 17 usable host/guest samples with zero collection errors. +Host CPU idle ranges are 23.13–62.60% on main and 8.39–70.71% on the PR. +Compilers appear in both traces; another native benchmark also appears during +the PR window. Neither window qualifies as an idle-host comparison. The +controller stops after the failed PR audit; it does not admit the second pair, +higher write rates or read/mixed cases. No performance gain or causal rollback +verdict is established. The original 2,000-Cell 750→1,000→1,500→2,000 ramp, +read/mixed qualification and original maintenance-fence diagnosis remain open. + +The retained `fresh-write-comparison-v30.json` and `.md` reports under +`cellule-origin-comparison-20261011-v19` reconcile every request journal, +original histogram, ACK manifest, publication frontier and resource window. +The PR case is under `cellule-grant-current-20261011-v18`. + +The latest Compose CI run `38217748520` also retains a failed leased +local-command/c16 gate: main 553.509/s, PR 498.1505/s, ratio 0.899986 against +the unchanged 0.90 minimum. Its p95 ratio is 1.491617 and p99 ratio 0.854865. +All four pairs remain included. This hosted routing fixture publishes per-cell +roots and does not test the three-VM follower-log capacity path. Rust CI run +`38217748541` passes; that does not override the performance gate failure. + +## Earlier low-rate PR #76 write diagnostics October 12, 2026 UTC. These new runs use three separate Linux VMs, each with 1 vCPU and 2 GiB, on the shared workstation. Each has fresh native disk-backed @@ -346,10 +404,42 @@ failed fixture invocation and all source/binary/resource evidence remain in lease's expiry. These cases qualify one unpublished ACK each after process loss, not physical power loss, peer partition, or throughput. -The missing support VM and four raw TCP tunnels were restored. The matched -750-write/s comparison remains queued behind the unchanged idle-host gate and -exclusive fixture admission. The original full ramp, read/mixed tests and -conditional rollback remain outstanding. +The missing support VM and four raw TCP tunnels were restored. The first matched +750-write/s cases subsequently ran and failed qualification as recorded above; +the controller stopped after the PR audit failed. The original full ramp, +read/mixed tests and conditional rollback remain outstanding. + +A subsequent diagnostic attempts canonical recovery of that retained PR case, +using its original provider and follower disks. All native state was archived +before the attempt. Recovery discovers 1,968 still-owned Cells, then returns +`Fenced` after 42.40 seconds; no ACK recovery qualification is claimed. The +diagnostic driver's initial cleanup stopped the provider before joining the +followers, causing their later storage errors. The corrected dependency order +joins followers first. Both the recovery failure and failed cleanup remain +retained under `cellule-retained-audit-20261012-v31`. The original two-hour +request identities have expired; the diagnostic does not change clocks or +identities to manufacture an exact-retry pass. + +A second retained-case attempt reproduces the recovery failure. Its targeted +probe shows the same claimant, claim generation, recovering log and bundle head; +the current time is 11.180 seconds beyond the 30-second claim expiry. This +identifies a recovery-claim lifecycle failure, independently of the original +append failure and intermittent maintenance CI failure. + +The managed recovery path now renews that exact claim during verification and +materialization, without changing its lifetime or permitting expired claims. +Renewal and terminal authority CAS are serialized; immutable terminal inventory +verification runs outside that lock. A real 33-second root-read regression test +fails before the fix and passes afterward. All 18 focused recovery tests, +80 node tests and strict runtime test lint pass in an isolated Linux snapshot. +The fixed native-VM diagnostic completes in 158.45 seconds, including restoring +all 2,000 exact roots and joining the runtime and owner. Both followers and the +provider also join with exit 0, with no OOM. The offline audit checks every one +of the 110,861 original ACKs against the restored row, exact receipt position, +runtime metadata and canonical stored outcome, with zero violations. It takes +207.87 seconds. The original capacity and HTTP-retry failures remain failures; +this diagnostic does not establish a performance improvement or a fresh HTTP +exact-retry pass. All original journals, hashes and failed attempts remain. The maintenance investigation also reproduces a distinct capture race: a real same-boot heartbeat between two directory observations caused `Fenced` through From d2c87edfdb111580c843578b007f74923a61284d Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 10:02:15 -0700 Subject: [PATCH 25/31] fix: adopt exact completed recovery on retry --- .../docs/failover-and-followers.md | 4 + .../src/node/log_recovery/completed.rs | 92 +++++++++++++++++++ .../src/node/log_recovery/mod.rs | 7 ++ .../runtime/lifecycle/ownership/recovery.rs | 17 ++++ docs/publication-progress-measurements.md | 56 ++++++++++- 5 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 crates/cellule-runtime/src/node/log_recovery/completed.rs diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index ce023daa..809d9c3f 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -1646,6 +1646,10 @@ terminal node CAS lock and ends with the attempt, including cancellation. A dead claimant, changed proof or expired claim still rejects. Terminal inventory verification runs outside that lock; sealing checks its exact immutable head against fresh authority before the final CAS. +A retry after a committed seal adopts the exact recorded result without a new +claim. It checks current controls, the complete original manifest and each +referenced overlay, then rechecks canonical closure and the live claimant. A +stale control or missing payload rejects; adoption writes no recovery state. Partial-root retries reuse the original digest-verified manifest and match every remaining tail's scope, predecessor, complete physical/logical range and bundle digest. A materialized root ahead of the old catalog is only a verified base; diff --git a/crates/cellule-runtime/src/node/log_recovery/completed.rs b/crates/cellule-runtime/src/node/log_recovery/completed.rs new file mode 100644 index 00000000..4aed55b4 --- /dev/null +++ b/crates/cellule-runtime/src/node/log_recovery/completed.rs @@ -0,0 +1,92 @@ +//! Read-only adoption of an already committed recovery result. +use super::*; + +impl RecoveryCoordinator { + pub(super) async fn resume_completed( + &self, + directory: &NodeDirectory, + fenced: &FencedNodeSession, + cells: &[RecoveryCell], + now_ms: i64, + ) -> Result> { + let started = std::time::Instant::now(); + let recovered = match directory + .recovered_session(fenced.node(), fenced.session(), fenced.claimant(), now_ms) + .await + { + Ok(recovered) => recovered, + Err(Error::Control("node session log recovery is incomplete")) => return Ok(None), + Err(source) => return Err(source), + }; + let sealed = SealedNodeLog { + session: fenced.session(), + log: recovered.log().cloned().ok_or(Error::Fenced)?, + }; + let takeover = NodeTakeoverProof::after_recovery(fenced, &sealed)?; + let mut controls = Vec::new(); + for cell in cells { + let current = cell + .authority + .load(cell.observed.value().cell) + .await? + .ok_or(Error::Fenced)?; + if current.value() != cell.observed.value() + || *cell.application.as_bytes() == [0; 16] + || current.value().ltx_root().is_none() + || current.value().owner.as_ref().map(|owner| owner.session) + != Some(fenced.session()) + { + return Err(Error::Fenced); + } + if current.value().recovery.is_some() { + controls.push(current); + } + } + let manifest = pinned_manifest(&controls, fenced, self.recovery.log_epoch)?; + if sealed.log().recovery_manifest() != manifest { + return Err(Error::Fenced); + } + if let Some(manifest) = manifest { + let inventory = self + .manifests + .load_manifest(fenced.session(), self.recovery.log_epoch, manifest) + .await?; + if inventory.cells().len() != controls.len() { + return Err(Error::Fenced); + } + for pin in inventory.cells() { + cells + .iter() + .find(|cell| { + cell.application == pin.application + && cell.observed.value().cell == pin.cell + && cell.observed.value().incarnation == pin.incarnation + && cell.observed.value().epoch == pin.cell_epoch + && cell.observed.value().recovery.as_ref() == Some(&pin.recovery) + }) + .ok_or(Error::Fenced)?; + // A terminal pointer does not prove its immutable payload is + // still available. Reopen the exact original overlay. + self.manifests + .for_application(pin.application) + .load_overlay(pin.cell, pin.incarnation, &pin.recovery) + .await?; + } + } + let now = crate::node::bundle::recovery::logical_now(now_ms, started)?; + let current = directory + .recovered_session(fenced.node(), fenced.session(), fenced.claimant(), now) + .await?; + if current != recovered { + return Err(Error::Fenced); + } + // Sealing clears the transient recovery claim. A repeated successful + // call adopts only the exact terminal result; it renews no old claim + // and performs no Cell, manifest or node authority writes. + Ok(Some(CompletedNodeRecovery { + sealed, + controls, + takeover, + })) + } +} diff --git a/crates/cellule-runtime/src/node/log_recovery/mod.rs b/crates/cellule-runtime/src/node/log_recovery/mod.rs index 56a02b28..a40da591 100644 --- a/crates/cellule-runtime/src/node/log_recovery/mod.rs +++ b/crates/cellule-runtime/src/node/log_recovery/mod.rs @@ -26,6 +26,7 @@ const MAX_RECOVERY_PAGE_BYTES: u64 = 1 << 20; const MAX_RECOVERY_PAGE_FRAMES: usize = 4_096; mod claim; +mod completed; pub mod retirement; mod tail; mod witness; @@ -749,6 +750,12 @@ impl RecoveryCoordinator { now_ms: i64, ) -> Result { self.recovery.validate_fence(&fenced)?; + if let Some(completed) = self + .resume_completed(directory, &fenced, &cells, now_ms) + .await? + { + return Ok(completed); + } let claim = RecoveryClaimLease::new(directory, fenced.clone(), now_ms).await?; claim .run(async { diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs index ee377e0e..b4c2c535 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs @@ -263,6 +263,23 @@ async fn recover_retained_tail(rooted: bool, fail_at: u8, interrupted: bool) { completed.sealed.log().phase(), cellule_runtime::node::log_state::NodeLogPhase::Sealed ); + // A terminal recovery retry may adopt the exact attached result, but it + // cannot certify the stale pre-recovery control under that same seal. + assert!(matches!( + resumed + .recover_and_seal( + &directory, + fenced.clone(), + vec![cellule_runtime::node::log_recovery::RecoveryCell { + application: fixture.target.application(), + authority: authority.clone(), + observed: stale.clone(), + }], + 10_003, + ) + .await, + Err(cellule_runtime::Error::Fenced) + )); let repeated = resumed .recover_and_seal( &directory, diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 3589d7dc..c832e1fb 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -6,6 +6,56 @@ uses an 8-vCPU owner. Fresh verification on three separate 1-vCPU/2-GiB VMs has not passed latency and backlog qualification. The separate 2K-write/20K-read qualification remains open. +## Fresh hosted routing comparison + +October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main +`c79834d` with merge revision `0816881` (parents main and PR head `9e68a3e`). +It retains four alternating pairs with the same harness on one hosted Linux +runner. Each command lane executes 1,024 commands and each warm query lane +executes 4,096 queries per run. The table reports medians of all four runs at +concurrency 16; latency is measured from invocation, without paced offers. + +| Lane | Main / PR operations per second | Main p50 / p95 / p99 | PR p50 / p95 / p99 | +| --- | ---: | --- | --- | +| Local writes | 619.80 / 616.28 | 24.54 / 36.15 / 41.99 ms | 24.63 / 35.38 / 43.35 ms | +| Forwarded writes | 497.19 / 504.74 | 29.45 / 40.15 / 45.69 ms | 29.15 / 39.68 / 44.48 ms | +| Local warm reads | 16,297.29 / 16,524.87 | 0.501 / 0.897 / 1.039 ms | 0.491 / 0.883 / 0.959 ms | +| Forwarded warm reads | 4,261.77 / 4,300.75 | 2.030 / 3.480 / 4.283 ms | 2.051 / 3.471 / 4.205 ms | + +The leased routing gate passes. Local writes deliver 0.57% less throughput and +3.24% higher p99; forwarded writes deliver 1.52% more throughput and 2.65% lower +p99. These small differences do not establish a general improvement. The +expired-burst local-query lane also retains a p99 alert (1.574 times main), +within the unchanged 2.0 gate. All lanes and the earlier failed comparison +remain included. + +The same run's object-only gate also passes. At concurrency 16, local writes +measure 424.25/s on main and 426.34/s on the PR, with p99 56.76 and 53.34 ms. +Forwarded writes measure 371.73/s and 367.92/s, with p99 58.54 and 59.50 ms. +Forwarded reads retain a p99 alert: 9.55 ms on main and 10.70 ms on the PR +(1.120 times main). Their throughput is 1,930.15/s and 1,870.17/s. These +results also show mixed small changes rather than a consistent gain. + +This fixture publishes per-cell roots. It does not run the three-VM shared-log +workload, enforce its 64-MiB profile, or qualify its sustainable capacity. +The raw `manifest.json`, `comparison.json`, binaries and all request logs are +retained under `cellule-ci-routing-9e-20261012-v37/{leased,object-only}`, from +[the leased CI job](https://github.com/crabbuild/cellule/actions/runs/38274456201). +The matched three-VM 750-write/s comparison is still waiting for five +continuous idle minutes on the workstation. Higher-rate, read and mixed-load +qualification remain open. + +Rust CI run `38274456048` at `9e68a3e` fails four idempotent recovery-retry +tests: the new renewal guard tries to validate a claim that the committed seal +has already cleared. An isolated Linux run reproduces all four failures. The +correction adopts only the canonical terminal result, checks exact controls, +manifest and referenced overlays, and performs no recovery authority writes. +All eight focused recovery tests pass, including rejection of stale controls; +all 1,317 runtime tests and doctests pass with all features. Strict all-target +runtime Clippy passes. Original CI logs and both local runs remain under +`cellule-recovery-retry-20261012-v36`. This is a recovery correctness fix; +the original intermittent maintenance `Fenced` cause remains unproved. + ## Latest 750-write/s PR #76 verification October 12, 2026 UTC. The fresh matched cases use 2,000 Cells and three @@ -57,12 +107,12 @@ The retained `fresh-write-comparison-v30.json` and `.md` reports under original histogram, ACK manifest, publication frontier and resource window. The PR case is under `cellule-grant-current-20261011-v18`. -The latest Compose CI run `38217748520` also retains a failed leased +The earlier Compose CI run `38217748520` also retains a failed leased local-command/c16 gate: main 553.509/s, PR 498.1505/s, ratio 0.899986 against the unchanged 0.90 minimum. Its p95 ratio is 1.491617 and p99 ratio 0.854865. All four pairs remain included. This hosted routing fixture publishes per-cell -roots and does not test the three-VM follower-log capacity path. Rust CI run -`38217748541` passes; that does not override the performance gate failure. +roots and does not test the three-VM follower-log capacity path. Earlier Rust +CI run `38217748541` passes; that does not override the performance gate failure. ## Earlier low-rate PR #76 write diagnostics From 48ff24be14652384b85756c60358c7da86ac1dfa Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 10:50:27 -0700 Subject: [PATCH 26/31] fix: preserve follower grants across receiver clock skew --- .../examples/fleet/remote_fault_tests.rs | 207 +++++++++++++++++- crates/cellule-runtime/docs/append-grants.md | 7 + .../src/follower/grant/tests.rs | 28 +++ .../src/node/append_grant/mod.rs | 11 +- docs/publication-progress-measurements.md | 43 ++++ docs/remote-follower-tail-verification.md | 40 +++- scripts/verify-remote-follower-tail.py | 115 +++++++++- 7 files changed, 425 insertions(+), 26 deletions(-) diff --git a/crates/cellule-axum/examples/fleet/remote_fault_tests.rs b/crates/cellule-axum/examples/fleet/remote_fault_tests.rs index f7f8e1d0..9f8bd944 100644 --- a/crates/cellule-axum/examples/fleet/remote_fault_tests.rs +++ b/crates/cellule-axum/examples/fleet/remote_fault_tests.rs @@ -1,4 +1,4 @@ -//! Real signed follower RPCs, unpublished ACK, killed owner and canonical recovery. +//! Signed follower RPCs, unpublished ACK, killed owner, partitions and recovery. //! The external driver owns three distinct VM/container roles and their native disks. use super::*; use cellule_ltx::{CellReplica, Host, Limits}; @@ -31,6 +31,7 @@ const PAYLOAD: Digest = Digest::from_bytes([156; 32]); struct HeldData { inner: Arc, armed: AtomicBool, + held_root: AtomicBool, entered: tokio::sync::Notify, } impl std::fmt::Display for HeldData { @@ -46,8 +47,15 @@ impl ObjectStore for HeldData { payload: object_store::PutPayload, options: object_store::PutOptions, ) -> object_store::Result { - if path.as_ref().ends_with(".nodebundle") && self.armed.load(Ordering::Acquire) { - self.entered.notify_one(); + if self.armed.load(Ordering::Acquire) + && (path.as_ref().ends_with(".nodebundle") || path.as_ref().ends_with(".root")) + { + if path.as_ref().ends_with(".root") { + self.held_root.store(true, Ordering::Release); + } + if path.as_ref().ends_with(".nodebundle") { + self.entered.notify_one(); + } // A killed owner cannot select this immutable upload. Authority and // enrollment operations still use the real strongly consistent store. std::future::pending::<()>().await; @@ -96,6 +104,16 @@ impl ObjectStore for HeldData { } } +async fn wait_driver(device: &std::path::Path, name: &str) { + tokio::time::timeout(Duration::from_secs(60), async { + while !device.join(name).exists() { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); +} + fn target() -> CellTarget { CellTarget::new( TenantId::from_bytes([61; 16]), @@ -119,8 +137,18 @@ fn marker(name: &str, value: serde_json::Value) { #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[ignore = "requires isolated RustFS, pinned TLS and an external driver owning three separate 1-vCPU/2-GiB VMs"] async fn remote_crash_fixture_child() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing_subscriber::filter::LevelFilter::WARN) + .with_writer(std::io::stderr) + .try_init(); // Refuse an incomplete invocation rather than quietly substituting memory. let role = std::env::var("CELLULE_REMOTE_FAULT_ROLE").unwrap(); + let fault = + std::env::var("CELLULE_REMOTE_FAULT_KIND").unwrap_or_else(|_| "owner-process-loss".into()); + assert!(matches!( + fault.as_str(), + "owner-process-loss" | "peer-partition" + )); std::env::var("CELLULE_TEST_ENDPOINT").unwrap(); let device = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); let config = Config::from_env(false).unwrap().unwrap(); @@ -129,6 +157,7 @@ async fn remote_crash_fixture_child() { let held = Arc::new(HeldData { inner: remote.inner().clone(), armed: AtomicBool::new(false), + held_root: AtomicBool::new(false), entered: tokio::sync::Notify::new(), }); let store = if role == "owner" { @@ -152,6 +181,34 @@ async fn remote_crash_fixture_child() { return; } assert_eq!(config.index, 0); + let before = (role == "recover").then(|| { + serde_json::from_str::( + &std::env::var("CELLULE_REMOTE_FAULT_ACK").unwrap(), + ) + .unwrap() + }); + if let Some(before) = &before { + let predecessor = SessionId::from_bytes( + *uuid::Uuid::parse_str(before["session"].as_str().unwrap()) + .unwrap() + .as_bytes(), + ); + // Check actual authority before enrolling a fourth live session. The + // owner can heartbeat after writing its first ACK marker. + let directory = + NodeDirectory::new(layout.clone(), config.tls().unwrap().fleet(), CODE, CODE); + tokio::time::timeout(Duration::from_secs(60), async { + while directory + .is_live(predecessor, clock().unwrap()) + .await + .unwrap() + { + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .unwrap(); + } let session = SessionId::from_bytes(*uuid::Uuid::now_v7().as_bytes()); let pool = SqlWorkerPool::new(1, 8) .unwrap() @@ -219,9 +276,19 @@ async fn remote_crash_fixture_child() { Ok(HandlerOutcome::Success(1i64.to_be_bytes().to_vec())) }), ) - .await - .unwrap() - .unwrap(); + .await; + if outcome.is_err() { + eprintln!( + "Initial remote ACK timed out; held root: {}; metrics: {}", + held.held_root.load(Ordering::Acquire), + metrics.snapshot() + ); + eprintln!( + "Initial remote ACK progress: {:?}", + runtime.node_durability().unwrap().1.progress() + ); + } + let outcome = outcome.unwrap().unwrap(); assert_eq!( outcome, StoredOutcome::Success { @@ -260,12 +327,86 @@ async fn remote_crash_fixture_child() { "held_bundle_upload": true }), ); + if fault == "peer-partition" { + wait_driver(&device, "partition-installed").await; + let submitted = metrics.snapshot()["node_log_submission"]["assigned_frames"] + .as_u64() + .unwrap(); + let executed = Arc::new(AtomicBool::new(false)); + let execution = executed.clone(); + let now = clock().unwrap(); + let second = handle.execute( + MutationIdentity { + request_id: RequestId::from_bytes([157; 16]), + issued_at_ms: now, + expires_at_ms: now.checked_add(120_000).unwrap(), + }, + Digest::from_bytes([158; 32]), + now, + 1_024, + 1_024, + move |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + execution.store(true, Ordering::Release); + Ok(HandlerOutcome::Success(2i64.to_be_bytes().to_vec())) + }, + ); + tokio::pin!(second); + let captured = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if executed.load(Ordering::Acquire) + && metrics.snapshot()["node_log_submission"]["assigned_frames"] + .as_u64() + .unwrap() + > submitted + { + break; + } + tokio::select! { + result = &mut second => panic!("partitioned command finished before capture: {result:?}"), + () = tokio::time::sleep(Duration::from_millis(20)) => {} + } + } + }); + captured.await.unwrap(); + let read = handle.query(64, 64, |tx| { + let value: i64 = tx.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }); + tokio::pin!(read); + let mut command_error = None; + let mut read_error = None; + let observing = tokio::time::sleep(Duration::from_secs(2)); + tokio::pin!(observing); + loop { + tokio::select! { + result = &mut second, if command_error.is_none() => { + command_error = Some(format!("{:?}", result.unwrap_err())); + } + result = &mut read, if read_error.is_none() => { + read_error = Some(format!("{:?}", result.unwrap_err())); + } + () = &mut observing => break, + } + } + let current = authority.load(target.cell_id()).await.unwrap().unwrap(); + assert_eq!(current.value().ltx_root().unwrap().commit_sequence, 0); + marker( + "partition-gated.json", + serde_json::json!({ + "second_handler_executed": true, "second_capture_assigned": true, + "second_success": false, "read_success": false, + "command_error": command_error, "read_error": read_error, + "bucket_commit_sequence": 0, + }), + ); + std::future::pending::<()>().await; + } std::future::pending::<()>().await; return; } assert_eq!(role, "recover"); - let before: serde_json::Value = - serde_json::from_str(&std::env::var("CELLULE_REMOTE_FAULT_ACK").unwrap()).unwrap(); + let before = before.unwrap(); let predecessor = SessionId::from_bytes( *uuid::Uuid::parse_str(before["session"].as_str().unwrap()) .unwrap() @@ -284,7 +425,7 @@ async fn remote_crash_fixture_child() { .unwrap(); // Expired advertisements cannot be loaded as live authority. The fencing // CAS returns the exact original membership that recovery must account for. - let fenced = directory + let mut fenced = directory .claim_expired(predecessor, session, clock().unwrap()) .await .unwrap(); @@ -306,16 +447,48 @@ async fn remote_crash_fixture_child() { session, config.tls().unwrap(), peers, - metrics, + metrics.clone(), ) .unwrap(), ); - let recovery = NodeLogRecovery::from_fenced(transport, &fenced, Limits::default()).unwrap(); + let mut recovery = + NodeLogRecovery::from_fenced(transport.clone(), &fenced, Limits::default()).unwrap(); let manifests = RecoveryManifestStore::new(layout, Limits::default()); - let inventory = recoverable_cells(&catalog, &authority, predecessor, 1) + let mut inventory = recoverable_cells(&catalog, &authority, predecessor, 1) .await .unwrap(); assert_eq!(inventory.len(), 1); + if fault == "peer-partition" { + let result = RecoveryCoordinator::new(recovery, manifests.clone()) + .recover_and_seal(&directory, fenced, inventory, clock().unwrap()) + .await; + assert!(matches!( + result, + Err(Error::Node( + "active node log has no complete follower witness" + )) + )); + let current = authority.load(target.cell_id()).await.unwrap().unwrap(); + assert_eq!(current.value().owner.as_ref().unwrap().session, predecessor); + assert_eq!(current.value().ltx_root().unwrap().commit_sequence, 0); + assert!(current.value().recovery.is_none()); + marker( + "recovery-refused.json", + serde_json::json!({ + "missing_witness_refused": true, "bucket_commit_sequence": 0, + "no_takeover": true, "no_overlay": true, + }), + ); + wait_driver(&device, "one-witness-reachable").await; + fenced = directory + .claim_expired(predecessor, session, clock().unwrap()) + .await + .unwrap(); + recovery = NodeLogRecovery::from_fenced(transport, &fenced, Limits::default()).unwrap(); + inventory = recoverable_cells(&catalog, &authority, predecessor, 1) + .await + .unwrap(); + } let completed = RecoveryCoordinator::new(recovery, manifests.clone()) .recover_and_seal(&directory, fenced, inventory, clock().unwrap()) .await @@ -364,6 +537,16 @@ async fn remote_crash_fixture_child() { .unwrap(), 1i64.to_be_bytes() ); + if fault == "peer-partition" { + marker( + "recovered-before-drain.json", + serde_json::json!({ + "value": 1, "commit_sequence": 1, "exact_retry": true, + "single_reachable_witness": true, + }), + ); + wait_driver(&device, "partition-healed").await; + } handle.drain().await.unwrap(); runtime.shutdown().await.unwrap(); owner.stop().await.unwrap(); diff --git a/crates/cellule-runtime/docs/append-grants.md b/crates/cellule-runtime/docs/append-grants.md index d02b409a..56a383f7 100644 --- a/crates/cellule-runtime/docs/append-grants.md +++ b/crates/cellule-runtime/docs/append-grants.md @@ -29,6 +29,13 @@ monotonic horizon starts before those reads; wall-clock rollback and slow I/O cannot extend it. This is a bounded authorization horizon, not a device persistence or clock-skew qualification. +Sender verification permits a receiver issue time ahead of its own wall clock +within the existing five-minute node-advertisement skew bound. It still rejects +negative times, expired grants and invalid signed lifetimes. The sender's cached +monotonic horizon starts before issuance I/O and uses the original signed +lifetime, so that allowance adds no lifetime. Receiver issuance, native append +release, sealing and lease checks retain their original strict local horizons. + The signed pruning floor is the lower of fresh authoritative object coverage and the sequence immediately before the requested window. Publication can advance past frames already queued by the source; retaining their first diff --git a/crates/cellule-runtime/src/follower/grant/tests.rs b/crates/cellule-runtime/src/follower/grant/tests.rs index b71e0ac1..0dd8fcf5 100644 --- a/crates/cellule-runtime/src/follower/grant/tests.rs +++ b/crates/cellule-runtime/src/follower/grant/tests.rs @@ -146,6 +146,34 @@ impl Fixture { } } +#[tokio::test] +async fn receiver_clock_ahead_preserves_the_exact_signed_append_window() { + let mut f = Fixture::new().await; + let grant = f.grant(1).await.unwrap(); + // A sender can receive a freshly issued grant before its own wall clock + // reaches the receiver's issue time. Native authorization still uses the + // receiver's original monotonic horizon and terminal lease. + let decoded = NodeAppendGrant::verify(&grant.encode(), &f.receiver, NOW - 1).unwrap(); + assert_eq!(decoded.digest(), grant.digest()); + assert_eq!(decoded.issued_at_ms(), NOW); + assert_eq!(decoded.expires_at_ms(), NOW + APPEND_GRANT_LIFETIME_MS); + assert!(!decoded.valid_at(NOW - 1)); + assert!(NodeAppendGrant::verify(&grant.encode(), &f.receiver, NOW - 5 * 60_000 - 1,).is_err()); + assert!( + NodeAppendGrant::verify(&grant.encode(), &f.receiver, decoded.expires_at_ms()).is_err() + ); + f.backend.reset(); + let frame = f.frame(1); + assert_eq!( + f.append(&decoded, vec![frame]) + .await + .unwrap() + .durable_through, + 1 + ); + assert!(f.backend.requests().is_empty()); +} + #[tokio::test] async fn signed_grant_binds_every_byte_and_amortizes_directory_reads() { let mut f = Fixture::new().await; diff --git a/crates/cellule-runtime/src/node/append_grant/mod.rs b/crates/cellule-runtime/src/node/append_grant/mod.rs index bf5c514b..f8ff01bd 100644 --- a/crates/cellule-runtime/src/node/append_grant/mod.rs +++ b/crates/cellule-runtime/src/node/append_grant/mod.rs @@ -24,6 +24,8 @@ pub struct NodeAppendGrant { impl NodeAppendGrant { /// Verifies the exact receiver signature and the caller's original boot pin. + /// Receiver issue times use the same bounded clock skew as advertisements; + /// this cannot extend the receiver's registered monotonic authorization. pub fn verify(encoded: &[u8], receiver: &NodeAdvertisement, now_ms: i64) -> Result { if encoded.len() != BODY_BYTES + 64 || &encoded[..8] != MAGIC { return Err(Error::PeerAuthorization("invalid append grant encoding")); @@ -46,7 +48,11 @@ impl NodeAppendGrant { || grant.receiver_certificate() != receiver.certificate || grant.receiver_key() != receiver.public_key || grant.expires_at_ms() > receiver.expires_at_ms - || !grant.valid_at(now_ms) + || now_ms < 0 + || grant.issued_at_ms() > now_ms.saturating_add(MAX_CLOCK_SKEW_MS) + // This is the sender's clock, while issuance and native proof + // release use the receiver's clock and original monotonic horizon. + || !grant.valid_at(now_ms.max(grant.issued_at_ms())) || grant.last_sequence().checked_sub(grant.first_sequence()) != Some(APPEND_GRANT_SEQUENCES - 1) || grant.first_sequence() == 0 @@ -111,7 +117,8 @@ impl NodeAppendGrant { pub fn expires_at_ms(&self) -> i64 { self.number(368) as i64 } - /// Whether wall-clock policy still permits this signed grant. + /// Whether the receiver's wall clock still permits this signed grant. + /// Senders use [`Self::verify`] for bounded differences between clocks. #[must_use] pub fn valid_at(&self, now_ms: i64) -> bool { self.issued_at_ms() >= 0 diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index c832e1fb..15ff3d6b 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -6,6 +6,49 @@ uses an 8-vCPU owner. Fresh verification on three separate 1-vCPU/2-GiB VMs has not passed latency and backlog qualification. The separate 2K-write/20K-read qualification remains open. +## Fresh receiver-clock and peer-partition verification + +October 12, 2026 UTC. A three-VM partition repeat failed before the partition +was installed: sender verification rejected a signed append grant and disabled +fleet proofs. An isolated diagnostic recorded both receiver issue times ahead +of the sender's clock by 31 and 52 ms. Signature, boot, certificate, key, +ensemble and sequence-window checks passed; only the issue-time check failed. +The same grant implementation exists byte-for-byte on main `c79834d`. + +Sender verification now applies the existing node-advertisement clock-skew +bound. The signed lifetime, sender's original monotonic cache horizon, +receiver's strict local validity, native fsync gate and terminal fencing remain +unchanged. A signed-grant/native-append regression test fails before the change +with just 1 ms of clock difference. All nine grant tests pass afterward, and +all 903 runtime library tests pass (four environment-dependent tests ignored). +Strict runtime all-target/all-feature Clippy and SQL-example test Clippy pass. +The SQL example's 16 ordinary tests pass (eight environment-dependent tests +ignored). The remote fault child runs explicitly through the external driver. + +Two final partition repeats pass using the original mTLS and signed-grant +transport on three separate 1-vCPU/2-GiB VMs with native ext4. Each obtains one +unpublished ACK, blocks both actual follower TCP paths, verifies that a captured +second write and its observed read cannot return success, and kills the owner. +Recovery refuses both unreachable followers without taking over the Cell or +attaching an overlay. Reaching one complete original follower then recovers the +ACK and exact retry while the other path remains blocked. Both paths are healed +before joined shutdown; all owned roles become terminal without OOM. Actual +iptables packet-drop counters are retained at each partition stage. + +The original failures, diagnostic source/binaries and two final cases +(`peer-partition-v35-r9` and `r10`) remain under +`cellule-peer-partition-20261012-v35`. The before/after regression and complete +runtime library logs remain under `cellule-recovery-retry-20261012-v36`. +Temporary debug instrumentation is absent from the corrected source. This is +process-loss and follower-path verification, not physical power-loss or capacity +qualification. It does not identify the unlogged append failure in the earlier +750-write/s run or the original intermittent maintenance `Fenced` cause. + +The next matched capacity comparison applies the common transport correction +to both main and the PR, preserving the original workload, budgets, guards and +every-ACK audits. The workstation remains busy; no corrected capacity result or +performance improvement is claimed. + ## Fresh hosted routing comparison October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main diff --git a/docs/remote-follower-tail-verification.md b/docs/remote-follower-tail-verification.md index f0b5281d..c2809855 100644 --- a/docs/remote-follower-tail-verification.md +++ b/docs/remote-follower-tail-verification.md @@ -1,7 +1,8 @@ # Remote unpublished-tail verification -This fixture checks a follower-acknowledged write after owner process loss, -using the SQL example's original signed mTLS transport and the public recovery +This fixture checks a follower-acknowledged write after owner process loss or +a follower TCP partition, using the SQL example's original signed mTLS transport +and the public recovery coordinator. It requires three separate Linux VMs and a support Docker host. It is an environment-dependent fault test, not a throughput benchmark. @@ -14,6 +15,9 @@ It is an environment-dependent fault test, not a throughput benchmark. | Owner process loss | SIGKILL after the ACK; successor uses a different fresh volume | | Recovery authority | Actual predecessor lease expiry, fencing CAS, remote sealing and complete tail recovery | | Exact retry | Original result and commit sequence; retry handler fails if executed | +| Partitioned output | After blocking both follower paths, a captured second mutation and its observed read cannot return success | +| Missing recovery witness | Recovery refuses both unreachable followers before installing an overlay or taking over the Cell | +| One complete witness | Reaching one original sealed follower recovers the first ACK while the second path remains blocked | | Joined resources | Successor and followers exit successfully; all owned containers become terminal without OOM | The support host runs the pinned RustFS provider. Configure loopback routes @@ -52,6 +56,29 @@ records, logs, native-storage/resource observations, and the recovered result outside the source checkout. Completed containers and volumes remain available for inspection. Run it separately from performance measurements on these hosts. +For the partition case: + +```sh +python3 scripts/verify-remote-follower-tail.py \ + --artifacts "$HOME/Workspace/crabbuild-target/cellule-remote-tail" \ + --binary /absolute/path/to/linux-sql-test-executable \ + --context support-context \ + --node-contexts owner-context follower-1-context follower-2-context \ + --tag peer-partition-r1 \ + --fault peer-partition \ + --owner-ssh-config /absolute/path/to/owner-vm/ssh.config \ + --owner-ssh-host owner-vm-alias +``` + +The SSH account needs passwordless access to `sudo iptables`. The driver checks +that SSH and Docker reach the same owner kernel, then installs two uniquely +tagged OUTPUT rules for the follower TCP ports. Packet counters must show actual +drops on both paths. It restores one path for recovery, verifies the original +ACK and exact retry while the other remains blocked, then heals that path before +joining the successor and followers. Cleanup removes only its own rules, and +stops the provider after all node roles become terminal. All markers are fsynced +on the roles' native volumes. + Two fresh cases passed with one unpublished ACK per case on the three-VM profile. The original failed fixture invocation is retained: successor enrollment occurred before predecessor lease expiry, exceeding the example's @@ -59,7 +86,14 @@ three-live-session bound. The corrected driver waits for actual expiry without changing that bound or spoofing time. See [the measurement record](publication-progress-measurements.md). -This checks process loss, not physical power loss or remote-peer partitions. +Two corrected peer-partition cases also pass with one unpublished ACK each. +Earlier failed cases remain retained, including a pre-partition grant rejection +caused by receiver issue times ahead of the sender's clock. See the same +measurement record for the diagnosis, correction, native packet counters and +per-case limits. + +This checks process loss and, when selected, the two follower TCP paths. It does +not simulate physical power loss or partition the object-store path. The existing [device-backed fault tests](../crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs) cover separate object-store TCP-partition cases. Neither fixture establishes capacity, latency improvement, or the cause of the original intermittent diff --git a/scripts/verify-remote-follower-tail.py b/scripts/verify-remote-follower-tail.py index eb382dad..10d71f7c 100755 --- a/scripts/verify-remote-follower-tail.py +++ b/scripts/verify-remote-follower-tail.py @@ -5,16 +5,22 @@ import pathlib import subprocess import shutil +import shlex import time ROOT = pathlib.Path(__file__).resolve().parent.parent -parser = argparse.ArgumentParser(description='Verify a remote unpublished ACK after owner process loss on three separate 1-vCPU/2-GiB Docker hosts.') +parser = argparse.ArgumentParser(description='Verify a remote unpublished ACK after owner loss or follower TCP partition on three separate 1-vCPU/2-GiB Docker hosts.') parser.add_argument('--artifacts', type=pathlib.Path, required=True) parser.add_argument('--binary', type=pathlib.Path, required=True, help='release SQL example test executable, not the ordinary SQL server') parser.add_argument('--context', required=True, help='separate provider/client support Docker context') parser.add_argument('--node-contexts', nargs=3, required=True, metavar=('OWNER', 'FOLLOWER1', 'FOLLOWER2')) parser.add_argument('--tag', required=True, help='new isolated object prefix and container-name suffix') +parser.add_argument('--fault', choices=['owner-process-loss', 'peer-partition'], default='owner-process-loss') +parser.add_argument('--owner-ssh-config', type=pathlib.Path, help='SSH configuration for the owner VM; required for peer-partition') +parser.add_argument('--owner-ssh-host', help='owner VM SSH alias; required for peer-partition') args = parser.parse_args() +if args.fault == 'peer-partition' and (not args.owner_ssh_config or not args.owner_ssh_host): + parser.error('peer-partition requires owner VM SSH configuration and alias') BASE = args.artifacts.resolve() if BASE.is_relative_to(ROOT): parser.error('evidence must be outside the source checkout') @@ -37,8 +43,9 @@ STORE = 'ghcr.io/rustfs/rustfs@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858' SELECTOR = 'fleet::remote_fault_tests::remote_crash_fixture_child' owned = [] +partition_rules = set() status = dict(phase='setup', started_at=time.time(), performance_qualification=False, - fault='owner process SIGKILL after remote fleet ACK before bundle publication', + fault=args.fault, transport='original pinned mTLS + signed grants + sealed remote tail', scope='three independent 1-CPU/2-GiB VMs, native ext4; not physical power loss', joins=[], ack_count=0, audit_count=0) @@ -67,6 +74,47 @@ def wait_until(predicate, seconds, label): raise RuntimeError('Fixture guard expired: ' + label) +def owner_ssh(*command): + result = subprocess.run(['ssh', '-F', str(args.owner_ssh_config.resolve()), + '-o', 'BatchMode=yes', args.owner_ssh_host, shlex.join(command)], + text=True, capture_output=True, timeout=25) + if result.returncode: + raise RuntimeError('Owner VM SSH command failed: ' + result.stderr) + return result.stdout + + +def partition_rule(port, install): + rule = ['OUTPUT', '-p', 'tcp', '-d', '127.0.0.1', '--dport', str(port), + '-m', 'comment', '--comment', 'cellule-' + TAG + '-partition', '-j', 'DROP'] + if install: + # Record before the command: an SSH reply loss may leave the rule installed. + partition_rules.add(port) + owner_ssh('sudo', '-n', 'iptables', '-I', *rule) + else: + owner_ssh('sudo', '-n', 'iptables', '-D', *rule) + partition_rules.remove(port) + + +def partition_snapshot(label, ports): + raw = owner_ssh('sudo', '-n', 'iptables', '-L', 'OUTPUT', '-n', '-v', '-x') + (CASE / (label + '-iptables.txt')).write_text(raw) + counters = {} + for line in raw.splitlines(): + if 'cellule-' + TAG + '-partition' not in line: + continue + for port in ports: + if 'dpt:' + str(port) in line: + counters[port] = int(line.split()[0]) + assert all(counters.get(port, 0) > 0 for port in ports), 'no actual dropped TCP packets' + status.setdefault('partition_observations', []).append(dict(label=label, dropped_packets=counters)) + save() + + +def signal(context, name, file): + code = "import os,pathlib; p=pathlib.Path('/scratch')/" + repr(file) + "; f=p.open('x'); f.write('ready\\n'); f.flush(); os.fsync(f.fileno()); f.close()" + docker(context, 'exec', name, 'python3', '-c', code) + + def native_snapshot(context, name): code = """import pathlib,json,subprocess p=pathlib.Path('/proc') @@ -94,7 +142,7 @@ def start_role(role, index): if docker(context, 'volume', 'inspect', volume, check=False).returncode == 0: raise RuntimeError('Native state volume is not fresh: ' + volume) docker(context, 'volume', 'create', '--label', 'cellule.fault.artifacts=' + str(CASE), volume) - args = ['run', '-d', '--name', name, '--label', 'cellule.fault.artifacts=' + str(CASE), + run_arguments = ['run', '-d', '--name', name, '--label', 'cellule.fault.artifacts=' + str(CASE), '--network', 'host', '--cpus', '1', '--memory', '2g', '--memory-swap', '2g', '--ulimit', 'nofile=65536:65536', '-v', volume + ':/scratch', '-v', str(BASE) + ':/work:ro', '-e', 'AWS_ACCESS_KEY_ID=benchmark_access', '-e', 'AWS_SECRET_ACCESS_KEY=benchmark_secret_private', @@ -102,15 +150,16 @@ def start_role(role, index): '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', 'CELLULE_TEST_PREFIX=' + TAG, '-e', 'CELLULE_TEST_DEVICE_PATH=/scratch', '-e', 'TMPDIR=/scratch', '-e', 'CELLULE_AXUM_FLEET_DIR=/scratch/fleet', '-e', 'CELLULE_REMOTE_FAULT_ROLE=' + role, + '-e', 'CELLULE_REMOTE_FAULT_KIND=' + args.fault, '-e', 'TOKIO_WORKER_THREADS=1', '-e', 'RUST_BACKTRACE=1'] if index: - args += ['-e', 'CELLULE_AXUM_FOLLOWER=' + str(index)] + run_arguments += ['-e', 'CELLULE_AXUM_FOLLOWER=' + str(index)] if role == 'recover': - args += ['-e', 'CELLULE_REMOTE_FAULT_ACK=' + json.dumps(status['ack'])] - args += [RUST, 'sh', '-c', f'mkdir -m 700 /scratch/fleet && cp /work/tls/ca.crt /work/tls/node-{index}.crt /work/tls/node-{index}.key /scratch/fleet/ && exec /work/bin/sql_fault {SELECTOR} --exact --ignored --nocapture'] + run_arguments += ['-e', 'CELLULE_REMOTE_FAULT_ACK=' + json.dumps(status['ack'])] + run_arguments += [RUST, 'sh', '-c', f'mkdir -m 700 /scratch/fleet && cp /work/tls/ca.crt /work/tls/node-{index}.crt /work/tls/node-{index}.key /scratch/fleet/ && exec /work/bin/sql_fault {SELECTOR} --exact --ignored --nocapture'] # Track ownership before the RPC: a failed Docker response can be ambiguous. owned.append((context, name)) - docker(context, *args) + docker(context, *run_arguments) status.setdefault('roles', []).append(dict(role=role, index=index, context=context, name=name, volume=volume)) save() return context, name @@ -118,7 +167,12 @@ def start_role(role, index): def marker(context, name, file): result = docker(context, 'exec', name, 'cat', '/scratch/' + file, check=False) - return json.loads(result.stdout) if result.returncode == 0 else None + if result.returncode == 0: + return json.loads(result.stdout) + state = json.loads(docker(context, 'inspect', name).stdout)[0]['State'] + if not state['Running']: + raise RuntimeError(f'Fixture role exited before {file}: {name}, exit {state["ExitCode"]}') + return None def joined(context, name, seconds): @@ -176,10 +230,25 @@ def joined(context, name, seconds): before = {name: native_snapshot(context, name) for context, name in [owner, *followers]} assert len({value['boot_id'] for value in before.values()}) == 3 (CASE / 'native-before-crash.json').write_text(json.dumps(before, indent=2) + '\n') + if args.fault == 'peer-partition': + assert owner_ssh('cat', '/proc/sys/kernel/random/boot_id').strip() == before[owner[1]]['boot_id'] + partition_rule(8090, True) + partition_rule(8100, True) + signal(*owner, 'partition-installed') + status['phase'] = 'checking_partition_output_gate' + save() + wait_until(lambda: marker(*owner, 'partition-gated.json') is not None, 10, 'partitioned output gate') + gated = marker(*owner, 'partition-gated.json') + assert gated['second_handler_executed'] and gated['second_capture_assigned'] + assert not gated['second_success'] and not gated['read_success'] + assert gated['bucket_commit_sequence'] == 0 + (CASE / 'partition-gated.json').write_text(json.dumps(gated, indent=2) + '\n') status['phase'] = 'killing_owner' save() docker(*owner[:1], 'kill', '--signal', 'KILL', owner[1]) assert joined(*owner, 10) == 137 + if args.fault == 'peer-partition': + partition_snapshot('both-followers-blocked', [8090, 8100]) # The embedding example bounds its live inventory to the three-node fleet. # Enroll the replacement only after actual old-boot lease expiry, using # wall time; never fake a future recovery timestamp or relax that bound. @@ -194,6 +263,25 @@ def joined(context, name, seconds): (CASE / 'native-successor.json').write_text(json.dumps(successor_native, indent=2) + '\n') status['phase'] = 'recovering_remote_tail' save() + if args.fault == 'peer-partition': + wait_until(lambda: marker(*successor, 'recovery-refused.json') is not None, + 65, 'recovery refuses missing complete follower witness') + refused = marker(*successor, 'recovery-refused.json') + assert refused['missing_witness_refused'] and refused['no_takeover'] and refused['no_overlay'] + assert refused['bucket_commit_sequence'] == 0 + (CASE / 'recovery-refused.json').write_text(json.dumps(refused, indent=2) + '\n') + partition_snapshot('recovery-refused', [8090, 8100]) + partition_rule(8090, False) + signal(*successor, 'one-witness-reachable') + wait_until(lambda: marker(*successor, 'recovered-before-drain.json') is not None, + 45, 'one complete reachable follower recovers original ACK') + recovered = marker(*successor, 'recovered-before-drain.json') + assert recovered['value'] == recovered['commit_sequence'] == 1 + assert recovered['exact_retry'] and recovered['single_reachable_witness'] + (CASE / 'recovered-before-drain.json').write_text(json.dumps(recovered, indent=2) + '\n') + partition_snapshot('one-follower-still-blocked', [8100]) + partition_rule(8100, False) + signal(*successor, 'partition-healed') assert joined(*successor, 100) == 0 # Read the marker from the stopped container without reusing its database. docker(successor[0], 'cp', successor[1] + ':/scratch/recovered.json', str(CASE / 'recovered.json')) @@ -215,7 +303,13 @@ def joined(context, name, seconds): finally: # Even a failed scenario has to join every accepted process and retain logs. cleanup_errors = [] - for context, name in owned: + for port in sorted(partition_rules.copy()): + try: + partition_rule(port, False) + except Exception as error: + cleanup_errors.append(dict(partition_port=port, error=repr(error))) + # Keep the real provider alive until node shutdown has joined its I/O. + for context, name in reversed(owned): try: inspection = docker(context, 'inspect', name, check=False) if inspection.returncode: @@ -230,6 +324,9 @@ def joined(context, name, seconds): terminal = json.loads(docker(context, 'inspect', name).stdout)[0] assert not terminal['State']['Running'] and not terminal['State']['OOMKilled'] (CASE / (name + '-terminal.json')).write_text(json.dumps(terminal, indent=2) + '\n') + if status['phase'] == 'passed': + expected_exit = 137 if name == owner[1] else 0 + assert terminal['State']['ExitCode'] == expected_exit except Exception as error: cleanup_errors.append(dict(context=context, name=name, error=repr(error))) if cleanup_errors: From 14dcb55937a463119e7a8fff4946ceee78db8d86 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 11:17:02 -0700 Subject: [PATCH 27/31] test: isolate lifecycle snapshots from process disk usage --- crates/cellule-host/tests/node/lifecycle.rs | 7 +++++++ docs/publication-progress-measurements.md | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/crates/cellule-host/tests/node/lifecycle.rs b/crates/cellule-host/tests/node/lifecycle.rs index 32b92a07..6ba468d7 100644 --- a/crates/cellule-host/tests/node/lifecycle.rs +++ b/crates/cellule-host/tests/node/lifecycle.rs @@ -91,6 +91,10 @@ async fn operational_pressure_recovers_acquisition_without_clearing_a_cordon() { async fn node_shutdown_is_idempotent_and_returns_stopped_state() { let pool = SqlWorkerPool::new(1, 1).unwrap(); let host = ReplicaHost::default(); + let process_disk = host.local_disk_budget(); + // Parallel tests charge the default process budget. This fixture owns an + // independent envelope with the same capacity for its exact stats snapshot. + let host = host.with_local_disk_budget(DiskBudget::new(process_disk.capacity())); let node = CellNodeBuilder::new(application()) .with_runtime(pool, 16 * 1024 * 1024) .with_replica_host(host) @@ -102,7 +106,10 @@ async fn node_shutdown_is_idempotent_and_returns_stopped_state() { let starting = node.status(); assert_eq!(starting.state(), NodeState::Starting); assert!(!starting.is_shutting_down()); + // Force the shared-budget interleaving that originally made CI flaky. + let unrelated_disk = process_disk.try_reserve(4096).unwrap(); assert_eq!(starting.stats(), node.stats()); + drop(unrelated_disk); assert!(node.start().is_err()); node.install_task_group(CancellationToken::new(), CancellationToken::new()) .unwrap(); diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 15ff3d6b..57640695 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -49,6 +49,18 @@ to both main and the PR, preserving the original workload, budgets, guards and every-ACK audits. The workstation remains busy; no corrected capacity result or performance improvement is claimed. +Rust CI run `38286916209` at `48ff24b` fails the host lifecycle test's startup +snapshot equality, before shutdown. The default Host disk budget is +process-shared, and another parallel test changes its usage between the two +samples. An unrelated 4,096-byte reservation reproduces the same assertion +deterministically. The fixture now owns a private budget with the same capacity +and retains both the forced interleaving and the complete stats assertion. +The focused test, all 131 host node tests with default parallel execution, and +strict all-target/all-feature host Clippy pass. Original CI and local +before/after evidence remain retained under +`cellule-host-starting-stats-20261012-v42`. This fixture change leaves the +prepared main/PR benchmark binaries and all performance gates unchanged. + ## Fresh hosted routing comparison October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main From df37b68c6c558164aa9bbb1047978d3e4936a8b0 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 11:38:48 -0700 Subject: [PATCH 28/31] test: isolate captured-open concurrency admission --- crates/cellule-ltx/tests/host/hooks/prepare.rs | 8 +++++++- docs/publication-progress-measurements.md | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/cellule-ltx/tests/host/hooks/prepare.rs b/crates/cellule-ltx/tests/host/hooks/prepare.rs index 9f52561b..3b4ef74c 100644 --- a/crates/cellule-ltx/tests/host/hooks/prepare.rs +++ b/crates/cellule-ltx/tests/host/hooks/prepare.rs @@ -512,7 +512,13 @@ async fn prepare_opens_captured_segments_concurrently() { Limits::default(), ) .unwrap() - .with_host(Host::default().with_executor(Arc::new(DelayedExecutor { delay }))); + .with_host( + // This test measures the three-open wave, so reserve its own three job + // slots instead of competing for the process-wide, CPU-sized pool. + Host::default() + .with_job_slots(Arc::new(tokio::sync::Semaphore::new(3))) + .with_executor(Arc::new(DelayedExecutor { delay })), + ); let started = tokio::time::Instant::now(); replica.prepare(None, &captured, 1, 1).await.unwrap(); diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 57640695..c25bd0ff 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -61,6 +61,17 @@ before/after evidence remain retained under `cellule-host-starting-stats-20261012-v42`. This fixture change leaves the prepared main/PR benchmark binaries and all performance gates unchanged. +Rust CI run `38289876672` at `14dcb55` fails the LTX concurrent-open test at +600 ms against its 500-ms assertion. The unchanged test reproduces 20/20 times +with two available job slots: three opens require two waves. Its fixture now +owns three job slots, so the assertion measures independent opens without +competing for the default process-wide pool. The same 500-ms assertion passes +100/100 times, and all 76 LTX host tests pass with default parallel execution +(three environment-dependent tests ignored). CI logs and the isolated +before/after evidence remain under `cellule-ltx-open-wave-20261012-v43`. +This test fixture change leaves production admission limits and the prepared +performance binaries unchanged. + ## Fresh hosted routing comparison October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main From 90a2fab3227cb0ad20b4249aae5011eafbcab27b Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 12:42:47 -0700 Subject: [PATCH 29/31] fix: keep maintenance blocked after inventory changes --- crates/cellule-host/minion/README.md | 5 + .../persisted/inventory_change.rs | 231 ++++++++++++++++++ .../scenario/follower_tests/persisted/mod.rs | 1 + .../minion/scenario/observation/mod.rs | 47 +++- docs/publication-progress-measurements.md | 54 ++++ 5 files changed, 329 insertions(+), 9 deletions(-) create mode 100644 crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs diff --git a/crates/cellule-host/minion/README.md b/crates/cellule-host/minion/README.md index 6acc81d1..fb889a50 100644 --- a/crates/cellule-host/minion/README.md +++ b/crates/cellule-host/minion/README.md @@ -195,6 +195,11 @@ eight service/receipt checks are reported separately from zero fleet writer moves. This qualifies an in-process reader/follower combination; busy primitive owners, process/provider failures and reboot remain required by the fleet plan. +A live role change during a follower-policy inventory scan invalidates that +observation. The reference observer discards its role and policy proofs and +keeps maintenance blocked until a fresh complete pass succeeds. Fencing, +transport and authority failures retain their original source errors. + The overload command creates three independently leased CellNodes over shared strict in-memory object storage, initializes twelve real SQLite Cells, and acknowledges a command on each. A held seven-GiB disk admission token causes the existing actor's diff --git a/crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs b/crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs new file mode 100644 index 00000000..db4987f3 --- /dev/null +++ b/crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs @@ -0,0 +1,231 @@ +//! A real actor activation invalidates the follower-policy inventory interval. +use super::*; +use cellule_host::fleet::{ + FleetAdapterFuture, FleetNodeSnapshot, FleetSnapshotRequest, FleetSnapshotSubject, + FleetSnapshotTransport, +}; + +struct HeldInventory { + native: Snapshots, + source_cells: AtomicUsize, + gate: Mutex< + Option<( + tokio::sync::oneshot::Sender<()>, + tokio::sync::oneshot::Receiver<()>, + )>, + >, +} + +fn fleet(fixture: &ManagedFixture) -> crate::scenario::adapters::LocalFleet { + crate::scenario::adapters::LocalFleet { + nodes: fixture.nodes.clone(), + journal: fixture.native.journal.clone(), + boots: fixture.boots.clone(), + records: fixture.records.clone(), + reader_verifier: None, + capture_sequence: std::sync::atomic::AtomicU64::new(0), + lose_release_replies: false, + lost_release_replies: AtomicUsize::new(0), + drop_closed_finalize_replies: AtomicUsize::new(0), + expired_receiver_cleanups: AtomicUsize::new(0), + } +} + +impl FleetSnapshotTransport for HeldInventory { + fn capture<'a>( + &'a self, + request: &'a FleetSnapshotRequest, + end: Instant, + ) -> FleetAdapterFuture<'a, Arc> { + Box::pin(async move { + // Hold the fresh category check after the original complete page. + // Both replies come from the real node; no proof bytes are altered. + if request.node() == node_id(0) + && request.subject() == &FleetSnapshotSubject::Cells(None) + && self.source_cells.fetch_add(1, Ordering::AcqRel) == 1 + { + let gate = self.gate.lock().unwrap().take().unwrap(); + gate.0.send(()).unwrap(); + gate.1.await.unwrap(); + } + self.native.capture(request, end).await + }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reference_observer_changed_inventory_blocks_policy_until_a_fresh_complete_scan() { + let (fixture, capture, policy) = setup().await; + let publication = publish(&fixture, &capture, policy).await; + let record = publication.record().unwrap().clone(); + let target = CellTarget::new( + TenantId::from_bytes([1; 16]), + scope().application, + application::NAMESPACE, + &[2], + ) + .unwrap(); + let app = application::compile().unwrap(); + let catalog = CellCatalog::new(fixture.native.layout.clone(), target.tenant()) + .provision( + CatalogEntry::new( + &target, + CatalogRole::Sql, + app.registry().module_digests()[0], + 1, + ) + .unwrap(), + ) + .await + .unwrap(); + let incarnation = IncarnationId::from_bytes([2; 16]); + let authority = CellAuthority::new(fixture.native.layout.clone()); + let initial = authority + .create_initial(&catalog, incarnation, owner(0)) + .await + .unwrap(); + let replica = CellReplica::new( + fixture.native.layout.clone(), + *target.cell_id().as_bytes(), + *incarnation.as_bytes(), + limits(), + ) + .unwrap(); + let mut records = fixture + .records + .iter() + .map(|(cell, record)| { + ( + *cell, + Record { + target: record.target.clone(), + incarnation: record.incarnation, + catalog: record.catalog.clone(), + replica: record.replica.clone(), + authority: record.authority.clone(), + }, + ) + }) + .collect::>(); + records.insert( + target.cell_id(), + Record { + target, + incarnation, + catalog: catalog.clone(), + replica: replica.clone(), + authority: authority.clone(), + }, + ); + let mut fleet = fleet(&fixture); + let roster = aggregate::roster(&fixture).await; + let (entered, paused) = tokio::sync::oneshot::channel(); + let (resume, released) = tokio::sync::oneshot::channel(); + let snapshots = Arc::new(HeldInventory { + native: Snapshots::new(&fixture), + source_cells: AtomicUsize::new(0), + gate: Mutex::new(Some((entered, released))), + }); + let mut observation = Box::pin(crate::scenario::observation::observe_with_snapshots( + &fleet, + &roster, + deadline(), + snapshots.clone(), + )); + tokio::select! { + result = &mut observation => panic!("completed before held recheck: {}", result.is_ok()), + result = paused => result.unwrap(), + } + let handle = fixture.nodes[0] + .runtime() + .bootstrap( + catalog, + replica, + authority, + initial, + fixture.native.root.path().join("second.sqlite"), + |tx| { + tx.execute_batch( + "CREATE TABLE counter(value INTEGER); INSERT INTO counter VALUES (31)", + )?; + Ok(()) + }, + ) + .await + .unwrap(); + resume.send(()).unwrap(); + let changed = observation.await; + fleet.records = Arc::new(records); + let fresh = crate::scenario::observation::observe(&fleet, &roster, deadline()).await; + let unchanged = fixture.native.journal.load_snapshot(scope()).await.unwrap(); + let stored_record = stored(&fixture, record.digest().unwrap()).await; + let pages = snapshots.source_cells.load(Ordering::Acquire); + drop((handle, capture, fleet, snapshots)); + fixture.finish().await; + + // Join all native owners before asserting either observer result. + assert_eq!(unchanged, *roster.snapshot()); + assert_eq!(stored_record, record); + assert_eq!(pages, 2); + let changed = changed.unwrap(); + assert!(changed.role_coverage().is_none()); + let missing = changed.maintenance_policy_coverage().unwrap(); + assert!(!missing.is_complete()); + assert_eq!( + (missing.progress().required, missing.progress().checked), + (1, 0) + ); + let fresh = fresh.unwrap(); + let checked = fresh.maintenance_policy_coverage().unwrap(); + assert!(checked.is_complete()); + assert_eq!( + (checked.progress().required, checked.progress().checked), + (1, 1) + ); + assert_eq!( + checked.obligations()[0].status(), + cellule_host::fleet::FleetMaintenancePolicyStatus::Follower(record.digest().unwrap()), + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reference_observer_fenced_inventory_preserves_the_original_source_error() { + let (fixture, capture, policy) = setup().await; + let publication = publish(&fixture, &capture, policy).await; + let record = publication.record().unwrap().clone(); + let fleet = fleet(&fixture); + let roster = aggregate::roster(&fixture).await; + let (entered, paused) = tokio::sync::oneshot::channel(); + let (resume, released) = tokio::sync::oneshot::channel(); + let snapshots = Arc::new(HeldInventory { + native: Snapshots::new(&fixture), + source_cells: AtomicUsize::new(0), + gate: Mutex::new(Some((entered, released))), + }); + let mut observation = Box::pin(crate::scenario::observation::observe_with_snapshots( + &fleet, + &roster, + deadline(), + snapshots.clone(), + )); + tokio::select! { + result = &mut observation => panic!("completed before held recheck: {}", result.is_ok()), + result = paused => result.unwrap(), + } + fixture.boots[0].guard.as_ref().unwrap().fence(); + resume.send(()).unwrap(); + let result = observation.await; + let stored_record = stored(&fixture, record.digest().unwrap()).await; + drop((capture, fleet, snapshots)); + fixture.finish().await; + assert_eq!(stored_record, record); + let error = result.err().unwrap().downcast::().unwrap(); + let Error::Facility { name, source } = *error else { + panic!("native snapshot source error was flattened"); + }; + assert_eq!(name, "fleet-native-snapshot-transport"); + assert!(matches!( + **source.downcast_ref::>().unwrap(), + Error::Fenced, + )); +} diff --git a/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs b/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs index 922cc05f..e6e63921 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs @@ -10,6 +10,7 @@ use cellule_host::{ }; use cellule_runtime::fleet::operations::{FollowerEvacuationRecord, FollowerReplacementPolicy}; mod cancelled_settlement; +mod inventory_change; mod maintenance; mod observation; mod races; diff --git a/crates/cellule-host/minion/scenario/observation/mod.rs b/crates/cellule-host/minion/scenario/observation/mod.rs index 86163b35..069cefc7 100644 --- a/crates/cellule-host/minion/scenario/observation/mod.rs +++ b/crates/cellule-host/minion/scenario/observation/mod.rs @@ -7,7 +7,7 @@ use cellule_host::fleet::{ FleetFollowerEvacuationVerifier, FleetFollowerReferences, FleetMaintenanceEnrollments, FleetNodeInventory, FleetNodeInventoryScan, FleetNodeSnapshot, FleetObservation, FleetOwnedCell, FleetRoleCoverage, FleetRoster, FleetSnapshotNativePage, FleetSnapshotRequest, - FleetSnapshotSubject, + FleetSnapshotSubject, FleetSnapshotTransport, }; use cellule_runtime::control::{Control, ControlState}; use cellule_runtime::fleet::operations::{ @@ -51,7 +51,24 @@ pub(super) async fn observe( roster: &FleetRoster, deadline: Instant, ) -> JournalResult { - observe_inner(fleet, roster, deadline, None).await + observe_inner( + fleet, + roster, + deadline, + None, + Arc::new(adapters::LocalSnapshots::new(fleet.nodes.clone())), + ) + .await +} + +#[cfg(test)] +pub(super) async fn observe_with_snapshots( + fleet: &adapters::LocalFleet, + roster: &FleetRoster, + deadline: Instant, + snapshots: Arc, +) -> JournalResult { + observe_inner(fleet, roster, deadline, None, snapshots).await } pub(super) async fn observe_with_failed_boot_closure( @@ -67,6 +84,7 @@ pub(super) async fn observe_with_failed_boot_closure( roster, deadline, Some((request, processes, claimant)), + Arc::new(adapters::LocalSnapshots::new(fleet.nodes.clone())), ) .await } @@ -80,8 +98,9 @@ async fn observe_inner( &dyn FleetFailedBootProcesses, SessionId, )>, + snapshots: Arc, ) -> JournalResult { - let capture = collect(fleet, roster, deadline).await?; + let mut capture = collect(fleet, roster, deadline).await?; let mut reader_checks = Vec::new(); let mut follower_checks = Vec::new(); let mut finished = capture.finished; @@ -91,13 +110,23 @@ async fn observe_inner( .collect_maintenance(fleet.journal.as_ref(), original, roster, deadline, clock) .await?; } - let follower_verifier = FleetFollowerEvacuationVerifier::new( - fleet.boots[0].directory.clone(), - Arc::new(adapters::LocalSnapshots::new(fleet.nodes.clone())), - ); - follower_checks = follower_verifier + let follower_verifier = + FleetFollowerEvacuationVerifier::new(fleet.boots[0].directory.clone(), snapshots); + match follower_verifier .collect_maintenance(fleet.journal.as_ref(), original, roster, deadline, clock) - .await?; + .await + { + Ok(checks) => follower_checks = checks, + Err(cellule_runtime::Error::Node("native inventory category changed")) => { + // A live role change invalidates this interval, not the fleet. + // Discard its proofs; the next driver pass must collect them + // afresh before maintenance can settle or finalize any boot. + capture.complete = false; + capture.role_coverage = None; + reader_checks.clear(); + } + Err(source) => return Err(source.into()), + } roster.confirm(fleet.journal.as_ref(), deadline).await?; finished = clock()?; } diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index c25bd0ff..62a2f13f 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -72,6 +72,26 @@ before/after evidence remain under `cellule-ltx-open-wave-20261012-v43`. This test fixture change leaves production admission limits and the prepared performance binaries unchanged. +Rust CI run `38292326937` at `df37b68` passes the workspace tests, then fails +the original combined maintenance scenario with +`Node("native inventory category changed")` during observer confirmation. +A held native recheck and real second-Cell activation reproduce that error +5/5 times. The reference observer now discards that interval's role and policy +proofs, leaves maintenance blocked, and requires a fresh complete collection. +The seven-category verifier and its fencing/authority checks remain unchanged. + +Both the invalidated-inventory and fenced-node source-error tests pass 20 +repeats, and both original maintenance scenarios pass 10 repeats. Strict +example/test Clippy passes. The broader persisted-observer suite records +27 passes and two failures: a snapshot interval refusal in the new test's +fresh observation, and an operational sample timestamp ahead of its +advertisement in the real-expiry restart test. Both remain retained and open; +the full suite is not qualified. Evidence remains under +`cellule-maintenance-inventory-20261012-v46`. This diagnoses the fresh inventory +race, not the original unlogged intermittent maintenance `Fenced` cause. +Only the reference example/tests/docs change; prepared SQL benchmark code and +all performance gates remain unchanged. + ## Fresh hosted routing comparison October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main @@ -122,6 +142,40 @@ runtime Clippy passes. Original CI logs and both local runs remain under `cellule-recovery-retry-20261012-v36`. This is a recovery correctness fix; the original intermittent maintenance `Fenced` cause remains unproved. +## Fresh hosted routing repeats + +October 12, 2026 UTC. The newest Compose run `38292326894` completes four +alternating pairs with main `c79834d` and merge `0820461` (PR head `df37b68`). +Both measured binaries are byte-identical to runs `38286916365` and +`38289876505`; only test fixtures and docs changed between these revisions. +All 242,192 raw latency samples per mode and every four-run median reconcile +in each run. The table reports the newest run, with invocation latency. + +| Leased lane, concurrency 16 | Main / PR operations per second | Main p50 / p95 / p99 | PR p50 / p95 / p99 | +| --- | ---: | --- | --- | +| Local writes | 541.56 / 543.36 | 27.93 / 40.21 / 46.86 ms | 28.00 / 39.97 / 45.16 ms | +| Forwarded writes | 432.28 / 430.42 | 34.19 / 46.06 / 49.30 ms | 34.15 / 46.17 / 49.53 ms | +| Local warm reads | 12,037.26 / 12,129.43 | 0.686 / 1.248 / 1.373 ms | 0.680 / 1.238 / 1.343 ms | +| Forwarded warm reads | 3,604.01 / 3,539.87 | 2.424 / 4.288 / 5.396 ms | 2.463 / 4.464 / 5.877 ms | + +The paired signs vary across hosted runs on identical binaries: + +| Run, four alternating pairs | Local write throughput / p99 change | Forwarded write throughput / p99 change | +| --- | ---: | ---: | +| `38286916365` | -1.65% / +7.24% | +6.59% / -21.68% | +| `38289876505` | -0.79% / +7.12% | -3.65% / +1.00% | +| `38292326894` | +0.33% / -3.62% | -0.43% / +0.46% | + +Both original gates pass in the newest run; its expired-burst forwarded-query +lane retains a p95 alert (1.103 times main). Passing those gates does not establish an improvement. +All lanes, object-only results, samples, binaries and manifests remain retained +under `cellule-ci-routing-df-20261012-v48`, with the earlier runs under +`cellule-ci-routing-48-20261012-v44` and `cellule-ci-routing-14-20261012-v47`. +[The newest CI run](https://github.com/crabbuild/cellule/actions/runs/38292326894) +uses per-cell roots and does not qualify the three-VM shared-log workload. +The matched idle-host capacity comparison and conditional rollback decision +remain open. + ## Latest 750-write/s PR #76 verification October 12, 2026 UTC. The fresh matched cases use 2,000 Cells and three From 820b9e5bf625cc24073cb31b6250cdb06db043ac Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 13:19:31 -0700 Subject: [PATCH 30/31] docs: retain fresh routing and VM clock evidence --- docs/publication-progress-measurements.md | 79 +++++++++++++++++------ 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index 62a2f13f..f3ec373e 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -85,13 +85,41 @@ repeats, and both original maintenance scenarios pass 10 repeats. Strict example/test Clippy passes. The broader persisted-observer suite records 27 passes and two failures: a snapshot interval refusal in the new test's fresh observation, and an operational sample timestamp ahead of its -advertisement in the real-expiry restart test. Both remain retained and open; -the full suite is not qualified. Evidence remains under +advertisement in the real-expiry restart test. That failed attempt remains +retained; the unchanged clock-discipline repeat is recorded below. Evidence +remains under `cellule-maintenance-inventory-20261012-v46`. This diagnoses the fresh inventory race, not the original unlogged intermittent maintenance `Fenced` cause. Only the reference example/tests/docs change; prepared SQL benchmark code and all performance gates remain unchanged. +## Fresh VM clock verification + +October 12, 2026 UTC. A 30-second read-only support-VM probe captured a +100.1-ms backward wall-clock step while monotonic time advanced normally. Its +sample timestamp matches the Lima guest-agent `SyncTime` adjustment in the +guest journal. All three target VMs also logged approximately 100-ms clock +adjustments. Installed Lima 2.1.4 has a matching +[upstream issue](https://github.com/lima-vm/lima/issues/5543). + +Only the four owned test VMs receive a reversible guest-agent systemd drop-in +removing `CAP_SYS_TIME`. Existing NTP services continue running. Five-minute +wall/monotonic probes then observe no backward steps in 53,610 total samples. +Original boot IDs, active guest agents and synchronized NTP are verified; +VM resources, Cellule code, original timestamps, deadlines and guards remain +unchanged. The performance controller resumes its original quiet-start policy. + +The exact frozen binary from the earlier failed persisted-observer run passes +all 29 tests in each of three repeats, including both timestamp-refusal cases +and the invalid-clock rejection test. The earlier 27-pass/two-failure attempt +remains retained. Clock steps are confirmed; their occurrence at those two +original test failures was not captured. This does not identify the original +unlogged maintenance `Fenced` cause or establish a performance improvement. +Before/after journals, every clock sample, capability/boot/NTP checks, rollback +procedure, unchanged binary hash and test logs remain under +`cellule-clock-discipline-20261012-v49`. No corrected idle-host capacity case +has yet been admitted. + ## Fresh hosted routing comparison October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main @@ -144,19 +172,20 @@ the original intermittent maintenance `Fenced` cause remains unproved. ## Fresh hosted routing repeats -October 12, 2026 UTC. The newest Compose run `38292326894` completes four -alternating pairs with main `c79834d` and merge `0820461` (PR head `df37b68`). -Both measured binaries are byte-identical to runs `38286916365` and -`38289876505`; only test fixtures and docs changed between these revisions. -All 242,192 raw latency samples per mode and every four-run median reconcile -in each run. The table reports the newest run, with invocation latency. +October 12, 2026 UTC. The newest Compose run `38299468996` completes four +alternating pairs with main `c79834d` and merge `5868ab8` (PR head `90a2fab`). +Both measured binaries are byte-identical to runs `38286916365`, `38289876505` +and `38292326894`; test fixtures, the reference observer and docs changed +between these revisions. All 242,192 raw latency samples per mode, publication +coverage and every four-run median reconcile through the canonical verifier. +The table reports the newest run, with invocation latency. | Leased lane, concurrency 16 | Main / PR operations per second | Main p50 / p95 / p99 | PR p50 / p95 / p99 | | --- | ---: | --- | --- | -| Local writes | 541.56 / 543.36 | 27.93 / 40.21 / 46.86 ms | 28.00 / 39.97 / 45.16 ms | -| Forwarded writes | 432.28 / 430.42 | 34.19 / 46.06 / 49.30 ms | 34.15 / 46.17 / 49.53 ms | -| Local warm reads | 12,037.26 / 12,129.43 | 0.686 / 1.248 / 1.373 ms | 0.680 / 1.238 / 1.343 ms | -| Forwarded warm reads | 3,604.01 / 3,539.87 | 2.424 / 4.288 / 5.396 ms | 2.463 / 4.464 / 5.877 ms | +| Local writes | 560.15 / 551.65 | 27.169 / 39.343 / 43.549 ms | 27.286 / 40.199 / 45.155 ms | +| Forwarded writes | 547.41 / 531.65 | 26.811 / 36.641 / 40.205 ms | 27.745 / 38.516 / 42.283 ms | +| Local warm reads | 12,348.99 / 12,296.51 | 0.665 / 1.247 / 1.392 ms | 0.663 / 1.251 / 1.376 ms | +| Forwarded warm reads | 3,306.44 / 3,287.31 | 2.624 / 4.633 / 5.532 ms | 2.654 / 4.578 / 5.913 ms | The paired signs vary across hosted runs on identical binaries: @@ -165,16 +194,26 @@ The paired signs vary across hosted runs on identical binaries: | `38286916365` | -1.65% / +7.24% | +6.59% / -21.68% | | `38289876505` | -0.79% / +7.12% | -3.65% / +1.00% | | `38292326894` | +0.33% / -3.62% | -0.43% / +0.46% | - -Both original gates pass in the newest run; its expired-burst forwarded-query -lane retains a p95 alert (1.103 times main). Passing those gates does not establish an improvement. -All lanes, object-only results, samples, binaries and manifests remain retained -under `cellule-ci-routing-df-20261012-v48`, with the earlier runs under -`cellule-ci-routing-48-20261012-v44` and `cellule-ci-routing-14-20261012-v47`. -[The newest CI run](https://github.com/crabbuild/cellule/actions/runs/38292326894) +| `38299468996` | -1.52% / +3.69% | -2.88% / +5.17% | + +Both original gates pass in the newest run, with no leased latency alerts. +Object-only forwarded writes retain a p99 alert: 34.312 ms on main and +41.089 ms on the PR (1.1975 times main), with throughput 766.83/s and 738.01/s. +Object-only expired-burst forwarded/local reads also retain p99 alerts of +1.1510/1.3721 times main. Passing the original coarse gates does not establish +an improvement. The previous run's leased expired-burst forwarded-query p95 +alert remains retained with that run. + +All lanes, object-only results, samples, binaries and manifests remain under +`cellule-ci-routing-90a-20261012-v50`; previous runs remain under +`cellule-ci-routing-df-20261012-v48`, `cellule-ci-routing-48-20261012-v44` and +`cellule-ci-routing-14-20261012-v47`. +[The newest CI run](https://github.com/crabbuild/cellule/actions/runs/38299468996) uses per-cell roots and does not qualify the three-VM shared-log workload. The matched idle-host capacity comparison and conditional rollback decision -remain open. +remain open. Rust CI run +[`38299468998`](https://github.com/crabbuild/cellule/actions/runs/38299468998) +at PR head `90a2fab` passes both workspace and MSRV jobs. ## Latest 750-write/s PR #76 verification From b60ac4ba3ad544032b25b596d698a47e64fc9320 Mon Sep 17 00:00:00 2001 From: forhappy Date: Mon, 12 Oct 2026 14:09:15 -0700 Subject: [PATCH 31/31] test: assert receiver reclamation after joined shutdown Resolve replies can retain completion bytes after the request slot is released, and native work remains admitted while the activated cell serves. Assert the retired receipt separately and keep exact whole-runtime zero accounting at the joined shutdown boundary. Verified 1,000 clean repeats, nine receiver tests, all 246 runtime integration tests, strict Clippy, and static documentation checks. Retain the fresh hosted routing regressions and gains without claiming three-VM capacity. --- .../tests/runtime/lifecycle/receiver.rs | 8 ++- docs/publication-progress-measurements.md | 64 ++++++++++++++----- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs b/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs index f3968acb..42ab2212 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs @@ -230,10 +230,16 @@ async fn prepared_credit_transfers_into_exact_root_sql_and_one_fenced_writer() { .is_none() ); retire(&receiver, &prepared).await; - assert_eq!(receiver.stats().retained_bytes(), 0); + assert!(matches!( + prepared.state(), + Err(cellule_runtime::Error::RuntimeClosed) + )); assert_eq!(receiver.stats().active_cells(), 1); activated.drain().await.unwrap(); receiver.shutdown().await.unwrap(); + // Resolve completion and native background work retain their own bytes + // while serving. The joined shutdown is the boundary for total reclamation. + assert_eq!(receiver.stats().retained_bytes(), 0); assert_eq!(receiver.stats().active_cells(), 0); assert_eq!(receiver.stats().resident_bytes(), 0); assert_eq!(receiver.stats().file_descriptors(), 0); diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md index f3ec373e..dbb8bdc0 100644 --- a/docs/publication-progress-measurements.md +++ b/docs/publication-progress-measurements.md @@ -72,6 +72,30 @@ before/after evidence remain under `cellule-ltx-open-wave-20261012-v43`. This test fixture change leaves production admission limits and the prepared performance binaries unchanged. +Rust CI run `38303478642` at `820b9e5` fails the receiver-credit test's +whole-runtime zero-byte assertion while its activated Cell is still serving: +112 bytes remain. The unchanged test reproduces that exact failure once in +188 attempts. A resolve completion owns 48 input bytes and 64 result bytes; +its reply releases the request slot before those reservations drop. Holding +that actual completion boundary briefly in an isolated diagnostic reproduces +the exact failure 5/5 times. A subsequent FIFO-only probe still fails 3/5 times +with other live native reservations, and that failed approach remains retained. + +The fixture now checks that the prepared receipt is closed and the activated +Cell remains serving, then asserts exact zero retained bytes after its existing +joined shutdown. Every original zero-resource, exact-root, SQL and fenced-writer +assertion remains. The corrected test passes 5/5 diagnostic runs and 1,000/1,000 +uninstrumented repeats with four parallel processes. All nine receiver tests, +all 246 runtime integration tests and strict integration-target Clippy pass; +six tests require their documented external environments and remain ignored. +An initial direct-executable suite omitted Cargo's movement-probe environment +and failed four process tests; its log is retained, and the complete suite +passes when launched through Cargo. Temporary instrumentation is absent from +the workspace and clean verification snapshot. Evidence remains under +`cellule-ci-failure-820-20261012-v51`. This fixture change does not alter +production accounting or the prepared performance binaries, and does not +diagnose the older unlogged maintenance `Fenced` failure. + Rust CI run `38292326937` at `df37b68` passes the workspace tests, then fails the original combined maintenance scenario with `Node("native inventory category changed")` during observer confirmation. @@ -172,20 +196,20 @@ the original intermittent maintenance `Fenced` cause remains unproved. ## Fresh hosted routing repeats -October 12, 2026 UTC. The newest Compose run `38299468996` completes four -alternating pairs with main `c79834d` and merge `5868ab8` (PR head `90a2fab`). +October 12, 2026 UTC. The newest Compose run `38303478631` completes four +alternating pairs with main `c79834d` and merge `ab5fcda` (PR head `820b9e5`). Both measured binaries are byte-identical to runs `38286916365`, `38289876505` -and `38292326894`; test fixtures, the reference observer and docs changed +and `38292326894`, and to `38299468996`; test fixtures, the reference observer and docs changed between these revisions. All 242,192 raw latency samples per mode, publication coverage and every four-run median reconcile through the canonical verifier. The table reports the newest run, with invocation latency. | Leased lane, concurrency 16 | Main / PR operations per second | Main p50 / p95 / p99 | PR p50 / p95 / p99 | | --- | ---: | --- | --- | -| Local writes | 560.15 / 551.65 | 27.169 / 39.343 / 43.549 ms | 27.286 / 40.199 / 45.155 ms | -| Forwarded writes | 547.41 / 531.65 | 26.811 / 36.641 / 40.205 ms | 27.745 / 38.516 / 42.283 ms | -| Local warm reads | 12,348.99 / 12,296.51 | 0.665 / 1.247 / 1.392 ms | 0.663 / 1.251 / 1.376 ms | -| Forwarded warm reads | 3,306.44 / 3,287.31 | 2.624 / 4.633 / 5.532 ms | 2.654 / 4.578 / 5.913 ms | +| Local writes | 950.69 / 873.67 | 15.710 / 24.102 / 31.714 ms | 15.652 / 26.950 / 34.894 ms | +| Forwarded writes | 714.00 / 740.91 | 19.450 / 30.864 / 43.050 ms | 19.609 / 28.342 / 36.139 ms | +| Local warm reads | 28,048.23 / 28,349.27 | 0.282 / 0.510 / 0.540 ms | 0.284 / 0.503 / 0.540 ms | +| Forwarded warm reads | 7,183.39 / 7,161.29 | 1.185 / 2.258 / 2.760 ms | 1.183 / 2.334 / 2.865 ms | The paired signs vary across hosted runs on identical binaries: @@ -195,25 +219,31 @@ The paired signs vary across hosted runs on identical binaries: | `38289876505` | -0.79% / +7.12% | -3.65% / +1.00% | | `38292326894` | +0.33% / -3.62% | -0.43% / +0.46% | | `38299468996` | -1.52% / +3.69% | -2.88% / +5.17% | - -Both original gates pass in the newest run, with no leased latency alerts. -Object-only forwarded writes retain a p99 alert: 34.312 ms on main and -41.089 ms on the PR (1.1975 times main), with throughput 766.83/s and 738.01/s. -Object-only expired-burst forwarded/local reads also retain p99 alerts of -1.1510/1.3721 times main. Passing the original coarse gates does not establish -an improvement. The previous run's leased expired-burst forwarded-query p95 -alert remains retained with that run. +| `38303478631` | -8.10% / +10.03% | +3.77% / -16.05% | + +Both original coarse gates pass in the newest run. Leased local writes retain +p95/p99 alerts of 1.1181/1.1003 times main; leased expired-burst local reads +retain a p99 alert of 1.1752 times main. Object-only local reads at concurrency +one and fresh-client local reads at concurrency 16 retain p99 alerts of +1.6335/1.1014 times main. Object-only concurrency-16 local/forwarded writes +achieve 821.90/694.18 per second, against main's 828.60/690.97; their p99s are +32.361/33.037 ms, against 36.882/33.999 ms. Passing the original coarse gates +does not establish an improvement. All earlier alerts, including the previous +run's object-only forwarded-write p99 regression, remain retained. All lanes, object-only results, samples, binaries and manifests remain under -`cellule-ci-routing-90a-20261012-v50`; previous runs remain under +`cellule-ci-routing-820-20261012-v52`; previous runs remain under +`cellule-ci-routing-90a-20261012-v50`, `cellule-ci-routing-df-20261012-v48`, `cellule-ci-routing-48-20261012-v44` and `cellule-ci-routing-14-20261012-v47`. -[The newest CI run](https://github.com/crabbuild/cellule/actions/runs/38299468996) +[The newest CI run](https://github.com/crabbuild/cellule/actions/runs/38303478631) uses per-cell roots and does not qualify the three-VM shared-log workload. The matched idle-host capacity comparison and conditional rollback decision remain open. Rust CI run [`38299468998`](https://github.com/crabbuild/cellule/actions/runs/38299468998) at PR head `90a2fab` passes both workspace and MSRV jobs. +The follow-up `820b9e5` workspace run fails the receiver fixture described above; +its passing performance jobs do not override that CI failure. ## Latest 750-write/s PR #76 verification