diff --git a/crates/cellule-axum/examples/fleet/mod.rs b/crates/cellule-axum/examples/fleet/mod.rs index d35a59ab..88fd2e76 100644 --- a/crates/cellule-axum/examples/fleet/mod.rs +++ b/crates/cellule-axum/examples/fleet/mod.rs @@ -13,6 +13,8 @@ use std::{ }; mod authority; +#[cfg(test)] +mod remote_fault_tests; mod server; #[cfg(test)] mod tests; @@ -101,7 +103,8 @@ impl Config { metrics: Arc, ) -> Result { let tls = self.tls()?; - let directory = NodeDirectory::new(layout, tls.fleet(), code, code); + let directory = NodeDirectory::new(layout, tls.fleet(), code, code) + .with_telemetry(runtime.telemetry_handle()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .map_err(transport_error)?; diff --git a/crates/cellule-axum/examples/fleet/remote_fault_tests.rs b/crates/cellule-axum/examples/fleet/remote_fault_tests.rs new file mode 100644 index 00000000..9f8bd944 --- /dev/null +++ b/crates/cellule-axum/examples/fleet/remote_fault_tests.rs @@ -0,0 +1,561 @@ +//! Signed follower RPCs, unpublished ACK, killed owner, partitions and recovery. +//! The external driver owns three distinct VM/container roles and their native disks. +use super::*; +use cellule_ltx::{CellReplica, Host, Limits}; +use cellule_runtime::{ + CatalogRole, CellTarget, MutationIdentity, NamespaceId, SqlWorkerPool, TenantId, + cell::{ + catalog::{CatalogEntry, CellCatalog}, + executor::{HandlerOutcome, StoredOutcome}, + }, + control::{Owner as CellOwner, authority::CellAuthority}, + identity::{IncarnationId, RequestId}, + node::{ + log_recovery::{NodeLogRecovery, RecoveryCoordinator, recoverable_cells}, + log_transport::NodeLogTransport, + }, + recovery::manifest::RecoveryManifestStore, +}; +use cellule_store::Store; +use futures_util::stream::BoxStream; +use object_store::{ObjectStore, path::Path}; +use std::sync::atomic::{AtomicBool, Ordering}; + +const APPLICATION: ApplicationId = ApplicationId::from_bytes([62; 16]); +const CODE: Digest = Digest::from_bytes([65; 32]); +const INCARNATION: IncarnationId = IncarnationId::from_bytes([64; 16]); +const REQUEST: RequestId = RequestId::from_bytes([155; 16]); +const PAYLOAD: Digest = Digest::from_bytes([156; 32]); + +#[derive(Debug)] +struct HeldData { + inner: Arc, + armed: AtomicBool, + held_root: AtomicBool, + entered: tokio::sync::Notify, +} +impl std::fmt::Display for HeldData { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HeldData") + } +} +#[async_trait::async_trait] +impl ObjectStore for HeldData { + async fn put_opts( + &self, + path: &Path, + payload: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + if self.armed.load(Ordering::Acquire) + && (path.as_ref().ends_with(".nodebundle") || path.as_ref().ends_with(".root")) + { + if path.as_ref().ends_with(".root") { + self.held_root.store(true, Ordering::Release); + } + if path.as_ref().ends_with(".nodebundle") { + self.entered.notify_one(); + } + // A killed owner cannot select this immutable upload. Authority and + // enrollment operations still use the real strongly consistent store. + std::future::pending::<()>().await; + } + self.inner.put_opts(path, payload, options).await + } + async fn put_multipart_opts( + &self, + path: &Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts( + &self, + path: &Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.inner.get_opts(path, options).await + } + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +async fn wait_driver(device: &std::path::Path, name: &str) { + tokio::time::timeout(Duration::from_secs(60), async { + while !device.join(name).exists() { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap(); +} + +fn target() -> CellTarget { + CellTarget::new( + TenantId::from_bytes([61; 16]), + APPLICATION, + NamespaceId::from_bytes([63; 16]), + b"remote-follower-tail", + ) + .unwrap() +} +fn marker(name: &str, value: serde_json::Value) { + let root = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); + let path = root.join(name); + let mut file = std::fs::File::create(path).unwrap(); + use std::io::Write as _; + file.write_all(serde_json::to_string_pretty(&value).unwrap().as_bytes()) + .unwrap(); + file.sync_all().unwrap(); + println!("Remote fault marker: {name}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[ignore = "requires isolated RustFS, pinned TLS and an external driver owning three separate 1-vCPU/2-GiB VMs"] +async fn remote_crash_fixture_child() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing_subscriber::filter::LevelFilter::WARN) + .with_writer(std::io::stderr) + .try_init(); + // Refuse an incomplete invocation rather than quietly substituting memory. + let role = std::env::var("CELLULE_REMOTE_FAULT_ROLE").unwrap(); + let fault = + std::env::var("CELLULE_REMOTE_FAULT_KIND").unwrap_or_else(|_| "owner-process-loss".into()); + assert!(matches!( + fault.as_str(), + "owner-process-loss" | "peer-partition" + )); + std::env::var("CELLULE_TEST_ENDPOINT").unwrap(); + let device = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); + let config = Config::from_env(false).unwrap().unwrap(); + let metrics = Arc::new(QueryMetrics::default()); + let (remote, prefix) = super::super::example_storage().await.unwrap(); + let held = Arc::new(HeldData { + inner: remote.inner().clone(), + armed: AtomicBool::new(false), + held_root: AtomicBool::new(false), + entered: tokio::sync::Notify::new(), + }); + let store = if role == "owner" { + Store::new(held.clone()) + } else { + remote + }; + let layout = cellule_ltx::CellStorageLayout::new(store, prefix, *APPLICATION.as_bytes()); + if role == "follower" { + assert!(config.index == 1 || config.index == 2); + let port = 8080 + u16::from(config.index) * 10; + config + .serve_follower( + layout, + CODE, + format!("127.0.0.1:{port}").parse().unwrap(), + metrics, + ) + .await + .unwrap(); + return; + } + assert_eq!(config.index, 0); + let before = (role == "recover").then(|| { + serde_json::from_str::( + &std::env::var("CELLULE_REMOTE_FAULT_ACK").unwrap(), + ) + .unwrap() + }); + if let Some(before) = &before { + let predecessor = SessionId::from_bytes( + *uuid::Uuid::parse_str(before["session"].as_str().unwrap()) + .unwrap() + .as_bytes(), + ); + // Check actual authority before enrolling a fourth live session. The + // owner can heartbeat after writing its first ACK marker. + let directory = + NodeDirectory::new(layout.clone(), config.tls().unwrap().fleet(), CODE, CODE); + tokio::time::timeout(Duration::from_secs(60), async { + while directory + .is_live(predecessor, clock().unwrap()) + .await + .unwrap() + { + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .unwrap(); + } + let session = SessionId::from_bytes(*uuid::Uuid::now_v7().as_bytes()); + let pool = SqlWorkerPool::new(1, 8) + .unwrap() + .with_native_memory_limit(512 << 20) + .unwrap(); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + pool, + 64 << 20, + session, + Host::default(), + ) + .unwrap(); + runtime.install_telemetry(metrics.clone()).unwrap(); + let owner = config + .start_owner( + layout.clone(), + CODE, + session, + APPLICATION, + &runtime, + metrics.clone(), + ) + .await + .unwrap(); + let directory = owner.directory(); + let target = target(); + let catalog = CellCatalog::new(layout.clone(), target.tenant()); + let authority = CellAuthority::new(layout.clone()); + let replica = CellReplica::new( + layout.clone(), + *target.cell_id().as_bytes(), + *INCARNATION.as_bytes(), + Limits::default(), + ) + .unwrap(); + let current_owner = CellOwner { + session, + endpoint: owner.endpoint.clone(), + }; + if role == "owner" { + let proof = catalog + .provision(CatalogEntry::new(&target, CatalogRole::Application, CODE, 1).unwrap()) + .await + .unwrap(); + let initial = authority + .create_initial(&proof, INCARNATION, current_owner) + .await + .unwrap(); + let handle = runtime.bootstrap(proof, replica, authority.clone(), initial, + device.join("owner.sqlite"), |tx| { + tx.execute_batch("CREATE TABLE counter(value INTEGER NOT NULL); INSERT INTO counter VALUES(0)")?; + Ok(()) + }).await.unwrap(); + held.armed.store(true, Ordering::Release); + let now = clock().unwrap(); + let identity = MutationIdentity { + request_id: REQUEST, + issued_at_ms: now, + expires_at_ms: now.checked_add(120_000).unwrap(), + }; + let outcome = tokio::time::timeout( + Duration::from_secs(20), + handle.execute(identity, PAYLOAD, now, 1_024, 1_024, |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(1i64.to_be_bytes().to_vec())) + }), + ) + .await; + if outcome.is_err() { + eprintln!( + "Initial remote ACK timed out; held root: {}; metrics: {}", + held.held_root.load(Ordering::Acquire), + metrics.snapshot() + ); + eprintln!( + "Initial remote ACK progress: {:?}", + runtime.node_durability().unwrap().1.progress() + ); + } + let outcome = outcome.unwrap().unwrap(); + assert_eq!( + outcome, + StoredOutcome::Success { + result: 1i64.to_be_bytes().to_vec(), + commit_sequence: 1 + } + ); + tokio::time::timeout(Duration::from_secs(5), held.entered.notified()) + .await + .unwrap(); + let original = authority.load(target.cell_id()).await.unwrap().unwrap(); + assert_eq!(original.value().ltx_root().unwrap().commit_sequence, 0); + let advertised = directory + .load(session, clock().unwrap()) + .await + .unwrap() + .unwrap(); + assert!(advertised.advertisement().log().unwrap().active()); + assert_eq!( + advertised + .advertisement() + .bundle_head() + .map(|head| head.selected_through()) + .unwrap_or(0), + 0 + ); + marker( + "acknowledged.json", + serde_json::json!({ + "session": uuid::Uuid::from_bytes(*session.as_bytes()).to_string(), + "issued_at_ms": identity.issued_at_ms, "expires_at_ms": identity.expires_at_ms, + "commit_sequence": 1, "bucket_commit_sequence": 0, + "native_epoch": advertised.advertisement().log().unwrap().epoch(), + "required_followers": advertised.advertisement().log().unwrap().members().len(), + "owner_expires_at_ms": advertised.advertisement().expires_at_ms(), + "held_bundle_upload": true + }), + ); + if fault == "peer-partition" { + wait_driver(&device, "partition-installed").await; + let submitted = metrics.snapshot()["node_log_submission"]["assigned_frames"] + .as_u64() + .unwrap(); + let executed = Arc::new(AtomicBool::new(false)); + let execution = executed.clone(); + let now = clock().unwrap(); + let second = handle.execute( + MutationIdentity { + request_id: RequestId::from_bytes([157; 16]), + issued_at_ms: now, + expires_at_ms: now.checked_add(120_000).unwrap(), + }, + Digest::from_bytes([158; 32]), + now, + 1_024, + 1_024, + move |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + execution.store(true, Ordering::Release); + Ok(HandlerOutcome::Success(2i64.to_be_bytes().to_vec())) + }, + ); + tokio::pin!(second); + let captured = tokio::time::timeout(Duration::from_secs(5), async { + loop { + if executed.load(Ordering::Acquire) + && metrics.snapshot()["node_log_submission"]["assigned_frames"] + .as_u64() + .unwrap() + > submitted + { + break; + } + tokio::select! { + result = &mut second => panic!("partitioned command finished before capture: {result:?}"), + () = tokio::time::sleep(Duration::from_millis(20)) => {} + } + } + }); + captured.await.unwrap(); + let read = handle.query(64, 64, |tx| { + let value: i64 = tx.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }); + tokio::pin!(read); + let mut command_error = None; + let mut read_error = None; + let observing = tokio::time::sleep(Duration::from_secs(2)); + tokio::pin!(observing); + loop { + tokio::select! { + result = &mut second, if command_error.is_none() => { + command_error = Some(format!("{:?}", result.unwrap_err())); + } + result = &mut read, if read_error.is_none() => { + read_error = Some(format!("{:?}", result.unwrap_err())); + } + () = &mut observing => break, + } + } + let current = authority.load(target.cell_id()).await.unwrap().unwrap(); + assert_eq!(current.value().ltx_root().unwrap().commit_sequence, 0); + marker( + "partition-gated.json", + serde_json::json!({ + "second_handler_executed": true, "second_capture_assigned": true, + "second_success": false, "read_success": false, + "command_error": command_error, "read_error": read_error, + "bucket_commit_sequence": 0, + }), + ); + std::future::pending::<()>().await; + } + std::future::pending::<()>().await; + return; + } + assert_eq!(role, "recover"); + let before = before.unwrap(); + let predecessor = SessionId::from_bytes( + *uuid::Uuid::parse_str(before["session"].as_str().unwrap()) + .unwrap() + .as_bytes(), + ); + tokio::time::timeout(Duration::from_secs(60), async { + while directory + .is_live(predecessor, clock().unwrap()) + .await + .unwrap() + { + tokio::time::sleep(Duration::from_millis(250)).await; + } + }) + .await + .unwrap(); + // Expired advertisements cannot be loaded as live authority. The fencing + // CAS returns the exact original membership that recovery must account for. + let mut fenced = directory + .claim_expired(predecessor, session, clock().unwrap()) + .await + .unwrap(); + let members = fenced.log().unwrap().members().to_vec(); + assert_eq!(members.len(), 2); + let mut peers = Vec::new(); + for member in members { + if let Some(peer) = directory + .resolve_node(member, clock().unwrap()) + .await + .unwrap() + { + peers.push(peer); + } + } + let transport: Arc = Arc::new( + transport::Transport::new( + directory.clone(), + session, + config.tls().unwrap(), + peers, + metrics.clone(), + ) + .unwrap(), + ); + let mut recovery = + NodeLogRecovery::from_fenced(transport.clone(), &fenced, Limits::default()).unwrap(); + let manifests = RecoveryManifestStore::new(layout, Limits::default()); + let mut inventory = recoverable_cells(&catalog, &authority, predecessor, 1) + .await + .unwrap(); + assert_eq!(inventory.len(), 1); + if fault == "peer-partition" { + let result = RecoveryCoordinator::new(recovery, manifests.clone()) + .recover_and_seal(&directory, fenced, inventory, clock().unwrap()) + .await; + assert!(matches!( + result, + Err(Error::Node( + "active node log has no complete follower witness" + )) + )); + let current = authority.load(target.cell_id()).await.unwrap().unwrap(); + assert_eq!(current.value().owner.as_ref().unwrap().session, predecessor); + assert_eq!(current.value().ltx_root().unwrap().commit_sequence, 0); + assert!(current.value().recovery.is_none()); + marker( + "recovery-refused.json", + serde_json::json!({ + "missing_witness_refused": true, "bucket_commit_sequence": 0, + "no_takeover": true, "no_overlay": true, + }), + ); + wait_driver(&device, "one-witness-reachable").await; + fenced = directory + .claim_expired(predecessor, session, clock().unwrap()) + .await + .unwrap(); + recovery = NodeLogRecovery::from_fenced(transport, &fenced, Limits::default()).unwrap(); + inventory = recoverable_cells(&catalog, &authority, predecessor, 1) + .await + .unwrap(); + } + let completed = RecoveryCoordinator::new(recovery, manifests.clone()) + .recover_and_seal(&directory, fenced, inventory, clock().unwrap()) + .await + .unwrap(); + assert_eq!(completed.controls.len(), 1); + let proof = catalog.lookup(target.cell_id()).await.unwrap().unwrap(); + let control = completed.controls.into_iter().next().unwrap(); + let handle = runtime + .takeover_restored( + proof, + replica, + authority, + control, + completed.takeover, + manifests, + device.join("recovered.sqlite"), + current_owner, + ) + .await + .unwrap(); + let identity = MutationIdentity { + request_id: REQUEST, + issued_at_ms: before["issued_at_ms"].as_i64().unwrap(), + expires_at_ms: before["expires_at_ms"].as_i64().unwrap(), + }; + let retry = handle + .execute(identity, PAYLOAD, clock().unwrap(), 1_024, 1_024, |_| { + Err(Error::Control("recovered request executed twice")) + }) + .await + .unwrap(); + assert_eq!( + retry, + StoredOutcome::Success { + result: 1i64.to_be_bytes().to_vec(), + commit_sequence: 1 + } + ); + assert_eq!( + handle + .query(64, 64, |tx| { + let value: i64 = tx.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }) + .await + .unwrap(), + 1i64.to_be_bytes() + ); + if fault == "peer-partition" { + marker( + "recovered-before-drain.json", + serde_json::json!({ + "value": 1, "commit_sequence": 1, "exact_retry": true, + "single_reachable_witness": true, + }), + ); + wait_driver(&device, "partition-healed").await; + } + handle.drain().await.unwrap(); + runtime.shutdown().await.unwrap(); + owner.stop().await.unwrap(); + marker( + "recovered.json", + serde_json::json!({ + "commit_sequence": 1, "value": 1, "exact_retry": true, + "reexecution": false, "runtime_joined": true, "owner_joined": true, + "follower_transport": "original signed mTLS capacity transport" + }), + ); +} diff --git a/crates/cellule-axum/examples/fleet/tests.rs b/crates/cellule-axum/examples/fleet/tests.rs index 59be10cc..7f7d97f2 100644 --- a/crates/cellule-axum/examples/fleet/tests.rs +++ b/crates/cellule-axum/examples/fleet/tests.rs @@ -126,3 +126,35 @@ fn grant_issuance_and_append_tokens_have_separate_signed_shapes() { request.grant.pop(); assert!(wire::validate(&request, 1000).is_err()); } + +#[test] +fn sender_deadline_passes_receiver_horizon_with_clock_headroom_and_preserves_grant_expiry() { + let mut request = wire::Request { + sender: vec![1; 16], + leader: vec![1; 16], + member: vec![2; 16], + epoch: 1, + operation: 5, + first_sequence: 1, + deadline_ms: wire::request_deadline(1000, 0).unwrap(), + ..Default::default() + }; + // The real three-VM request failed this first receiver check with a 1-ms + // lag. Exercise the same producer/consumer seam, including its full buffer. + assert!(wire::validate(&request, 999).is_ok()); + assert!(wire::validate(&request, 0).is_ok()); + assert!(wire::validate(&request, -1).is_err()); + request.operation = 1; + request.frames = vec![vec![3; 128]]; + request.grant = vec![7; 32]; + request.deadline_ms = wire::request_deadline(1000, 2000).unwrap(); + assert!(wire::validate(&request, 1000).is_ok()); + assert!(matches!( + wire::validate(&request, 2000), + Err(Error::PeerAuthorization("expired capacity log request")) + )); + assert!(matches!( + wire::request_deadline(i64::MAX, 0), + Err(Error::Deadline) + )); +} diff --git a/crates/cellule-axum/examples/fleet/transport.rs b/crates/cellule-axum/examples/fleet/transport.rs index 74a419a6..320cef2b 100644 --- a/crates/cellule-axum/examples/fleet/transport.rs +++ b/crates/cellule-axum/examples/fleet/transport.rs @@ -102,12 +102,7 @@ impl Transport { }; request.sender = self.sender.as_bytes().to_vec(); request.member = member.as_bytes().to_vec(); - let deadline = clock()?.checked_add(10_000).ok_or(Error::Deadline)?; - request.deadline_ms = if request.deadline_ms == 0 { - deadline - } else { - request.deadline_ms.min(deadline) - }; + request.deadline_ms = wire::request_deadline(clock()?, request.deadline_ms)?; let phase = std::time::Instant::now(); let body = request.encode_to_vec(); let digest = blake3::hash(&body); @@ -217,7 +212,13 @@ impl NodeLogTransport for Transport { .collect(); for attempt in 0..2 { let valid = cached.as_ref().is_some_and(|grant| { - std::time::Instant::now() < grant.expires + // Leave time for transport and native fsync before the + // follower rechecks authority. Its signed expiry remains + // unchanged and can still reject a slower append. + grant + .expires + .saturating_duration_since(std::time::Instant::now()) + > Duration::from_secs(1) && grant .authorization .authorize( diff --git a/crates/cellule-axum/examples/fleet/wire.rs b/crates/cellule-axum/examples/fleet/wire.rs index c580018d..622180d7 100644 --- a/crates/cellule-axum/examples/fleet/wire.rs +++ b/crates/cellule-axum/examples/fleet/wire.rs @@ -82,6 +82,18 @@ fn signing_bytes(body: &[u8], domain: &[u8]) -> Vec { bytes } +pub(super) fn request_deadline(now: i64, previous: i64) -> Result { + // Leave clock headroom below the receiver's ten-second horizon. Using the + // whole horizon rejected a real grant request when its clock lagged by 1 ms. + // A grant's own expiry remains an earlier, independent limit. + let deadline = now.checked_add(9_000).ok_or(Error::Deadline)?; + Ok(if previous == 0 { + deadline + } else { + previous.min(deadline) + }) +} + pub(super) fn validate(request: &Request, now: i64) -> Result<()> { if request.sender.len() != 16 || request.member.len() != 16 diff --git a/crates/cellule-axum/examples/sql.rs b/crates/cellule-axum/examples/sql.rs index 549995d9..c8084504 100644 --- a/crates/cellule-axum/examples/sql.rs +++ b/crates/cellule-axum/examples/sql.rs @@ -156,6 +156,7 @@ struct Order { struct ServiceState { app: ApplicationHandle, targets: Arc>, + metrics: Arc, placement: sql_placement::Placement, } @@ -179,6 +180,18 @@ async fn create_order( app: Cellule, State(state): State, Json(input): Json, +) -> Result<(StatusCode, CellJson), HttpError> { + let result = create_order_inner(app, &state, input).await; + if let Err(error) = &result { + state.metrics.http_failure(error, true); + } + result +} + +async fn create_order_inner( + app: Cellule, + state: &ServiceState, + input: CreateOrder, ) -> Result<(StatusCode, CellJson), HttpError> { let target = state.target(input.id)?; let sql = app.sql::(target)?; @@ -231,7 +244,11 @@ async fn get_order( HttpPath(id): HttpPath, MinimumReceipt(minimum): MinimumReceipt, ) -> Result>, HttpError> { - read_order(&app, state.target(id)?, id, minimum).await + let result = async { read_order(&app, state.target(id)?, id, minimum).await }.await; + if let Err(error) = &result { + state.metrics.http_failure(error, false); + } + result } async fn read_order( @@ -500,6 +517,17 @@ async fn main() -> ExampleResult<()> { }, None => serde_json::Value::Null, }; + sample["retained_work"] = match runtime.retained_work() { + Ok(snapshot) => serde_json::json!({ + "total": snapshot.used.retained_bytes(), + "capacity": snapshot.limit.retained_bytes(), + "working_materializations": snapshot.working_materializations, + "stages": cellule_runtime::fleet::resource::RetentionStage::ALL.iter() + .map(|stage| (stage.label().to_owned(), snapshot.retained_by_stage[*stage as usize])) + .collect::>(), + }), + Err(error) => serde_json::json!({"error": error.to_string()}), + }; sample["runtime"] = serde_json::json!({ "active_cells": stats.active_cells(), "retained_bytes": stats.retained_bytes(), @@ -518,6 +546,7 @@ async fn main() -> ExampleResult<()> { .with_state(ServiceState { app: typed, targets: Arc::new(targets.clone()), + metrics: query_metrics.clone(), placement, }); // This local fixture binds only loopback. A product installs its own diff --git a/crates/cellule-axum/examples/sql_metrics/mod.rs b/crates/cellule-axum/examples/sql_metrics/mod.rs index 9bde586d..7444150a 100644 --- a/crates/cellule-axum/examples/sql_metrics/mod.rs +++ b/crates/cellule-axum/examples/sql_metrics/mod.rs @@ -10,6 +10,7 @@ use std::{ }; mod capture; +mod publication; mod storage; mod submission; use capture::CaptureMetrics; @@ -42,6 +43,7 @@ pub(super) struct QueryMetrics { selected_roots: AtomicU64, materialized_commits: AtomicU64, shared: SharedMetrics, + publication: publication::Metrics, follower_frames: AtomicU64, follower_sync_calls: AtomicU64, follower_failures: AtomicU64, @@ -314,6 +316,35 @@ impl CellTelemetry for QueryMetrics { self.writes.objects.fetch_add(objects, Ordering::Relaxed); self.writes.bytes.fetch_add(bytes, Ordering::Relaxed); } + fn node_publication_started( + &self, + phase: cellule_runtime::fleet::telemetry::NodePublicationPhase, + ) { + self.publication.started(phase); + } + fn node_publication_phase( + &self, + phase: cellule_runtime::fleet::telemetry::NodePublicationPhase, + elapsed: Duration, + outcome: cellule_runtime::fleet::telemetry::PublicationPhaseOutcome, + ) { + self.publication.completed(phase, elapsed, outcome); + } + fn node_publication_cohort( + &self, + cohort: cellule_runtime::fleet::telemetry::NodePublicationCohort, + ) { + self.publication.cohort(cohort); + } + fn node_publication_fallback( + &self, + reason: cellule_runtime::fleet::telemetry::PublicationPipelineFallback, + ) { + self.publication.fallback(reason); + } + fn admission_refused(&self, reason: cellule_runtime::fleet::telemetry::AdmissionRefusal) { + self.publication.refused(reason); + } fn shared_publication(&self, timing: SharedPublicationTiming) { self.shared.cohorts.fetch_add(1, Ordering::Relaxed); self.shared @@ -374,6 +405,10 @@ impl CellTelemetry for QueryMetrics { } impl QueryMetrics { + pub(super) fn http_failure(&self, error: &cellule_axum::HttpError, write: bool) { + self.publication.http_failure(error, write); + } + pub(super) fn peer_phase(&self, phase: PeerPhase, elapsed: Duration) { self.peer[phase as usize].observe(elapsed); } @@ -425,6 +460,7 @@ impl QueryMetrics { ] { histograms.insert(name.into(), histogram.raw()); } + histograms.extend(self.publication.histograms()); histograms.extend(self.capture.window_snapshot()); histograms.extend(self.submission.window_snapshot()); value["histograms"] = histograms.into(); @@ -498,6 +534,7 @@ impl QueryMetrics { .collect(); serde_json::json!({ "schema_version": 3, + "node_publication": self.publication.snapshot(), "sample_session": self.sample_session.get_or_init(uuid::Uuid::now_v7).to_string(), "sample_elapsed_ns": nanos(self.sample_origin.get_or_init(std::time::Instant::now).elapsed()), "host_capacity": self.host_capacity, diff --git a/crates/cellule-axum/examples/sql_metrics/publication.rs b/crates/cellule-axum/examples/sql_metrics/publication.rs new file mode 100644 index 00000000..50da32cc --- /dev/null +++ b/crates/cellule-axum/examples/sql_metrics/publication.rs @@ -0,0 +1,198 @@ +//! Managed bundle phases and retained-work pressure, with finite labels. +use super::*; +use cellule_runtime::fleet::telemetry::{ + AdmissionRefusal, NodePublicationCohort, NodePublicationPhase, PublicationPhaseOutcome, + PublicationPipelineFallback, +}; + +#[derive(Default)] +struct Phase { + active: AtomicU64, + succeeded: AtomicU64, + failed: AtomicU64, + cancelled: AtomicU64, + elapsed: Histogram, +} + +#[derive(Default)] +pub(super) struct Metrics { + phases: [Phase; NodePublicationPhase::ALL.len()], + http_failures: [[AtomicU64; FAILURE_LABELS.len()]; 2], + refusals: [AtomicU64; AdmissionRefusal::ALL.len()], + fallbacks: [AtomicU64; 4], + cohorts: AtomicU64, + captures: AtomicU64, + frames: AtomicU64, + bytes: AtomicU64, + checkpoints: AtomicU64, + max_frames: AtomicU64, + max_bytes: AtomicU64, +} + +const FAILURE_LABELS: [&str; 13] = [ + "publication_backlog", + "pending_publication_bytes", + "node_retained_bytes", + "mailbox_requests", + "mailbox_bytes", + "other_capacity", + "fenced", + "draining", + "closed", + "deadline", + "pending_publication", + "other", + "outcome_unknown", +]; + +fn failure_class(error: &(dyn std::error::Error + 'static)) -> usize { + let mut source = Some(error); + for _ in 0..16 { + let Some(error) = source else { + break; + }; + if let Some(runtime) = error.downcast_ref::() { + use cellule_runtime::Error; + if let Error::Shared(original) = runtime { + return failure_class(original.as_ref()); + } + return match runtime { + Error::Capacity("publication backlog") => 0, + Error::Capacity("pending publication bytes") => 1, + Error::Capacity("node retained bytes") => 2, + Error::Capacity("Cell mailbox requests") => 3, + Error::Capacity("Cell mailbox bytes") => 4, + Error::Capacity(_) => 5, + Error::Fenced => 6, + Error::CellDraining => 7, + Error::RuntimeClosed => 8, + Error::Deadline => 9, + Error::PendingPublication => 10, + _ => 11, + }; + } + source = error.source(); + } + 11 +} + +impl Metrics { + pub(super) fn http_failure(&self, error: &cellule_axum::HttpError, write: bool) { + let reason = if error.code() == "outcome_unknown" { + 12 + } else { + failure_class(error) + }; + self.http_failures[usize::from(write)][reason].fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn started(&self, phase: NodePublicationPhase) { + self.phases[phase as usize] + .active + .fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn completed( + &self, + phase: NodePublicationPhase, + elapsed: Duration, + outcome: PublicationPhaseOutcome, + ) { + let phase = &self.phases[phase as usize]; + let counter = match outcome { + PublicationPhaseOutcome::Succeeded => &phase.succeeded, + PublicationPhaseOutcome::Failed => &phase.failed, + PublicationPhaseOutcome::Cancelled => &phase.cancelled, + }; + phase.elapsed.observe(elapsed); + counter.fetch_add(1, Ordering::Relaxed); + phase.active.fetch_sub(1, Ordering::Relaxed); + } + + pub(super) fn refused(&self, reason: AdmissionRefusal) { + self.refusals[reason as usize].fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn fallback(&self, reason: PublicationPipelineFallback) { + self.fallbacks[reason as usize].fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn cohort(&self, cohort: NodePublicationCohort) { + self.cohorts.fetch_add(1, Ordering::Relaxed); + self.captures + .fetch_add(cohort.captures as u64, Ordering::Relaxed); + self.frames + .fetch_add(cohort.frames as u64, Ordering::Relaxed); + self.bytes.fetch_add(cohort.bytes as u64, Ordering::Relaxed); + self.checkpoints + .fetch_add(cohort.checkpoints as u64, Ordering::Relaxed); + self.max_frames + .fetch_max(cohort.frames as u64, Ordering::Relaxed); + self.max_bytes + .fetch_max(cohort.bytes as u64, Ordering::Relaxed); + } + + pub(super) fn histograms(&self) -> serde_json::Map { + NodePublicationPhase::ALL + .iter() + .map(|phase| { + ( + format!("node_publication_{}", phase.label()), + self.phases[*phase as usize].elapsed.raw(), + ) + }) + .collect() + } + + pub(super) fn snapshot(&self) -> serde_json::Value { + let read = |value: &AtomicU64| value.load(Ordering::Relaxed); + let phases = NodePublicationPhase::ALL + .iter() + .map(|phase| { + let value = &self.phases[*phase as usize]; + ( + phase.label().to_owned(), + serde_json::json!({ + "active": read(&value.active), "succeeded": read(&value.succeeded), + "failed": read(&value.failed), "cancelled": read(&value.cancelled), + "elapsed": value.elapsed.snapshot(), + }), + ) + }) + .collect::>(); + let http_failures = ["reads", "writes"] + .iter() + .enumerate() + .map(|(kind, label)| { + ( + (*label).to_owned(), + FAILURE_LABELS + .iter() + .enumerate() + .map(|(reason, label)| { + ( + (*label).to_owned(), + read(&self.http_failures[kind][reason]).into(), + ) + }) + .collect::>() + .into(), + ) + }) + .collect::>(); + serde_json::json!({ + "phases": phases, + "http_failures": http_failures, + "admission_refusals": AdmissionRefusal::ALL.iter().map(|reason| ( + reason.label().to_owned(), read(&self.refusals[*reason as usize]).into() + )).collect::>(), + "pipeline_fallbacks": {"checkpoints": read(&self.fallbacks[0]), + "capacity": read(&self.fallbacks[1]), "no_successor": read(&self.fallbacks[2]), + "headroom": read(&self.fallbacks[3])}, + "cohorts": {"count": read(&self.cohorts), "captures": read(&self.captures), + "frames": read(&self.frames), "native_bytes": read(&self.bytes), + "checkpoints": read(&self.checkpoints), "max_frames": read(&self.max_frames), + "max_native_bytes": read(&self.max_bytes)}, + }) + } +} diff --git a/crates/cellule-axum/examples/sql_metrics/tests.rs b/crates/cellule-axum/examples/sql_metrics/tests.rs index 6156bebb..c0712f35 100644 --- a/crates/cellule-axum/examples/sql_metrics/tests.rs +++ b/crates/cellule-axum/examples/sql_metrics/tests.rs @@ -1,5 +1,56 @@ use super::*; +#[test] +fn node_publication_metrics_keep_phase_failures_and_http_sources_separate() { + use cellule_runtime::fleet::telemetry::{ + NodePublicationPhase as Phase, PublicationPhaseOutcome, + }; + let metrics = QueryMetrics::default(); + metrics.node_publication_started(Phase::Verification); + metrics.node_publication_phase( + Phase::Verification, + Duration::from_millis(3), + PublicationPhaseOutcome::Failed, + ); + metrics.node_publication_started(Phase::Checkpoint); + metrics.node_publication_phase( + Phase::Checkpoint, + Duration::from_millis(4), + PublicationPhaseOutcome::Cancelled, + ); + metrics.http_failure( + &cellule_axum::HttpError::from(cellule_runtime::Error::Shared(std::sync::Arc::new( + cellule_runtime::Error::Capacity("pending publication bytes"), + ))), + true, + ); + let snapshot = metrics.window_snapshot(); + assert_eq!( + snapshot["node_publication"]["phases"]["verification"]["failed"], + 1 + ); + assert_eq!( + snapshot["node_publication"]["phases"]["verification"]["active"], + 0 + ); + assert_eq!( + snapshot["node_publication"]["phases"]["checkpoint"]["cancelled"], + 1 + ); + assert_eq!( + snapshot["node_publication"]["http_failures"]["writes"]["pending_publication_bytes"], + 1 + ); + assert_eq!( + snapshot["node_publication"]["http_failures"]["reads"]["pending_publication_bytes"], + 0 + ); + assert_eq!( + snapshot["histograms"]["node_publication_verification"]["total_ns"], + 3_000_000 + ); +} + #[test] fn sparse_export_preserves_cumulative_histogram_indices_and_overflow() { let histogram = Histogram::default(); diff --git a/crates/cellule-host/docs/lifecycle.md b/crates/cellule-host/docs/lifecycle.md index e2f1aed8..599b9f55 100644 --- a/crates/cellule-host/docs/lifecycle.md +++ b/crates/cellule-host/docs/lifecycle.md @@ -215,6 +215,12 @@ expired operation or incomplete enrollment produces no settlement evidence. Complete foreign/native inventory, Pending producers, failed-owner recovery and terminal action handoff remain required before physical-node shutdown. +Directory rechecks accept heartbeat, follower activation and covered-prefix +advancement for the same signed boot and Open ensemble. Generation, issue time, +lease expiry, progress, activation and coverage cannot regress; epoch and members +remain exact. The capture retains its final observation. Closure, replacement or +authority loss still rejects the interval, including changes after a held read. + ### Persist and revalidate follower replacement evidence `FollowerReplacementPolicy` gives the application's nonzero member minimum a diff --git a/crates/cellule-host/minion/README.md b/crates/cellule-host/minion/README.md index 6acc81d1..fb889a50 100644 --- a/crates/cellule-host/minion/README.md +++ b/crates/cellule-host/minion/README.md @@ -195,6 +195,11 @@ eight service/receipt checks are reported separately from zero fleet writer moves. This qualifies an in-process reader/follower combination; busy primitive owners, process/provider failures and reboot remain required by the fleet plan. +A live role change during a follower-policy inventory scan invalidates that +observation. The reference observer discards its role and policy proofs and +keeps maintenance blocked until a fresh complete pass succeeds. Fencing, +transport and authority failures retain their original source errors. + The overload command creates three independently leased CellNodes over shared strict in-memory object storage, initializes twelve real SQLite Cells, and acknowledges a command on each. A held seven-GiB disk admission token causes the existing actor's diff --git a/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs b/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs index 1d5e411f..7eebdd09 100644 --- a/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_maintenance/mod.rs @@ -49,7 +49,9 @@ pub(super) async fn run( // These role-enabled futures contain native peer activation and complete // fleet capture. Their synchronous boxed constructors keep construction // temporaries out of the parent poll frame as well as its retained state. - let inputs = setup::initialize(root, &journal, nodes, boots, roles).await?; + let inputs = setup::initialize(root, &journal, nodes, boots, roles) + .await + .map_err(|source| failure("follower-maintenance-initialize", source))?; let version = journal.load_snapshot(scope()).await?.registry(); let page = journal.enrollments_page(version, None, 128).await?; let original = page @@ -121,8 +123,11 @@ pub(super) async fn run( // The driver owns cordon and evacuation transitions. No fixture writes a // ready-to-close row or an invented native role proof. loop { - let report = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&report)?; + let phase = "follower-maintenance-enter-evacuation"; + let report = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &report)?; for blocker in report.blockers { if !blockers.contains(&blocker) { blockers.push(blocker); @@ -143,14 +148,18 @@ pub(super) async fn run( } boots[1] .refresh_capacity(1, journal.as_ref(), deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-donor-refresh", source))?; if nodes[1].state() == NodeState::Stopped || !nodes[1].is_management_ready() { return Err(invalid("follower source stopped before replacement")); } // The missing replacement remains an explicit policy blocker and cannot // retire an uncovered original follower or authorize native Finalize. - let blocked = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&blocked)?; + let phase = "follower-maintenance-await-follower-policy"; + let blocked = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &blocked)?; if blocked .snapshot .head() @@ -171,7 +180,10 @@ pub(super) async fn run( } } if let Some(readers) = &inputs.readers { - readers.check_original(&journal, true).await?; + readers + .check_original(&journal, true) + .await + .map_err(|source| failure("follower-maintenance-original-reader", source))?; } let store = nodes[1] .try_owned_component::(cellule_host::FOLLOWER_STORE_COMPONENT)? @@ -195,11 +207,13 @@ pub(super) async fn run( // blocks finalization despite the donor having zero local writers. boots[3] .refresh_capacity(3, journal.as_ref(), deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-spare-refresh", source))?; let selected = boots[0] .directory .select_log_members(session(0), 1, clock()?, 4) - .await?; + .await + .map_err(|source| failure("follower-maintenance-select-members", source))?; if selected != [node_id(2), node_id(3)] { return Err(invalid( "follower replacement selection includes the donor or lacks redundancy", @@ -208,8 +222,15 @@ pub(super) async fn run( // This canonical actor barrier covers every acknowledged frame before the // existing supervisor fences each old member and closes the original epoch. inputs.coverage_hold.release(); - inputs.acknowledged.source.drain().await?; - let request = nodes[0].request_node_log_rotation(1)?; + inputs + .acknowledged + .source + .drain() + .await + .map_err(|source| failure("follower-maintenance-original-source-drain", source))?; + let request = nodes[0] + .request_node_log_rotation(1) + .map_err(|source| failure("follower-maintenance-request-rotation", source))?; let completion = tokio::time::timeout_at(deadline(), async { loop { let observed = request.observe()?; @@ -228,7 +249,9 @@ pub(super) async fn run( tokio::time::sleep(Duration::from_millis(10)).await; } }) - .await??; + .await + .map_err(|source| failure("follower-maintenance-rotation-wait", source))? + .map_err(|source| failure("follower-maintenance-rotation", source))?; if completion.retirement().barrier().covered_through() == 0 || completion.replacement_epoch() != 2 { @@ -236,7 +259,9 @@ pub(super) async fn run( "follower rotation lacks original tail coverage or the newer epoch", )); } - let renewed = service::resume(root, nodes, boots, &inputs).await?; + let renewed = service::resume(root, nodes, boots, &inputs) + .await + .map_err(|source| failure("follower-maintenance-resume", source))?; let current = journal.load_snapshot(scope()).await?; let policy = FollowerReplacementPolicy::new(scope(), 1, 2)?; journal @@ -249,7 +274,8 @@ pub(super) async fn run( .ok_or_else(|| invalid("follower maintenance operation disappeared"))?; let capture = nodes[0] .follower_evacuation(&original, operation, 2, deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-capture", source))?; if !Arc::ptr_eq(capture.rotation(), &completion) || capture.retired_members().len() != 2 || capture @@ -278,33 +304,53 @@ pub(super) async fn run( deadline(), clock, ) - .await?; + .await + .map_err(|source| failure("follower-maintenance-publication", source))?; let record = publication.record()?; let reader_capture = if let Some(readers) = &inputs.readers { // A follower proof alone cannot close a donor that also owns a reader. - let blocked = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&blocked)?; + let phase = "follower-maintenance-await-reader-policy"; + let blocked = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &blocked)?; readers .require_blocked(&blocked, &journal, &nodes[1]) - .await?; + .await + .map_err(|source| failure(phase, source))?; for blocker in blocked.blockers { if !blockers.contains(&blocker) { blockers.push(blocker); } } - readers.prepare_replacement(&journal, boots).await?; - let blocked = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&blocked)?; + readers + .prepare_replacement(&journal, boots) + .await + .map_err(|source| failure("follower-maintenance-prepare-reader", source))?; + let phase = "follower-maintenance-await-reader-ready"; + let blocked = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &blocked)?; readers .require_blocked(&blocked, &journal, &nodes[1]) - .await?; - Some(readers.complete_replacement(&journal, boots).await?) + .await + .map_err(|source| failure(phase, source))?; + Some( + readers + .complete_replacement(&journal, boots) + .await + .map_err(|source| failure("follower-maintenance-complete-reader", source))?, + ) } else { None }; let completed = loop { - let report = Box::pin(driver.reconcile_once(clock, deadline())).await?; - checked_report(&report)?; + let phase = "follower-maintenance-complete"; + let report = Box::pin(driver.reconcile_once(clock, deadline())) + .await + .map_err(|source| failure(phase, source))?; + checked_report(phase, &report)?; for blocker in report.blockers { if !blockers.contains(&blocker) { blockers.push(blocker); @@ -354,9 +400,14 @@ pub(super) async fn run( "published follower policy lost original retirement history", )); } - service::readback(root, &inputs, &renewed).await?; + service::readback(root, &inputs, &renewed) + .await + .map_err(|source| failure("follower-maintenance-writer-readback", source))?; if let Some((readers, capture)) = inputs.readers.as_ref().zip(reader_capture.as_ref()) { - readers.readback(&journal, capture).await?; + readers + .readback(&journal, capture) + .await + .map_err(|source| failure("follower-maintenance-reader-readback", source))?; } let roster = FleetRoster::collect(journal.as_ref(), &completed, deadline()).await?; let final_counts = observation::complete_counts(&fleet, &roster, deadline()) @@ -391,16 +442,32 @@ pub(super) async fn run( routed_activation_replays: 0, }) } -fn checked_report(report: &cellule_host::fleet::FleetReconcileReport) -> JournalResult<()> { - if let Some(failure) = report.failures.first() { - return Err(Box::new(failure.error.clone())); +fn checked_report( + phase: &'static str, + report: &cellule_host::fleet::FleetReconcileReport, +) -> JournalResult<()> { + if let Some(failed) = report.failures.first() { + return Err(failure( + phase, + Box::new(failed.error.clone()) as JournalError, + )); } - if let Some(failure) = &report.maintenance_failure { - return Err(Box::new(failure.clone())); + if let Some(failed) = &report.maintenance_failure { + return Err(failure(phase, Box::new(failed.clone()) as JournalError)); } Ok(()) } +// A finite scenario must keep the failing boundary and the original typed +// source together. Repeated Fenced errors without a phase cannot distinguish +// native rotation, resumed service and fresh policy verification. +pub(super) fn failure(name: &'static str, source: impl Into) -> JournalError { + Box::new(cellule_runtime::Error::Facility { + name, + source: source.into(), + }) +} + // Preserve the supervisor's shared source error when this finite command exits. #[derive(Debug)] struct RotationFailure(Arc); diff --git a/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs b/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs index 822d1952..38956581 100644 --- a/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs +++ b/crates/cellule-host/minion/scenario/follower_maintenance/setup.rs @@ -200,7 +200,8 @@ pub(super) fn initialize<'a>( .await?; boots[0] .refresh_capacity(0, journal.as_ref(), deadline()) - .await?; + .await + .map_err(|source| failure("follower-maintenance-leader-refresh", source))?; let handle = nodes[0] .runtime() .bootstrap( @@ -216,7 +217,8 @@ pub(super) fn initialize<'a>( Ok(()) }, ) - .await?; + .await + .map_err(|source| failure("follower-maintenance-bootstrap", source))?; let now = clock()?; let identity = MutationIdentity { request_id: RequestId::from_bytes([1; 16]), diff --git a/crates/cellule-host/minion/scenario/follower_tests/authority_race.rs b/crates/cellule-host/minion/scenario/follower_tests/authority_race.rs new file mode 100644 index 00000000..6632b346 --- /dev/null +++ b/crates/cellule-host/minion/scenario/follower_tests/authority_race.rs @@ -0,0 +1,154 @@ +//! Capture must keep its exact ensemble across a legitimate boot heartbeat. +use super::*; +use futures_util::stream::BoxStream; +use object_store::{ObjectStore, path::Path}; + +#[derive(Debug, Default)] +struct HeldNodeRead { + inner: InMemory, + path: Mutex>, + entered: tokio::sync::Notify, + resume: tokio::sync::Notify, +} +impl std::fmt::Display for HeldNodeRead { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HeldNodeRead") + } +} +#[async_trait::async_trait] +impl ObjectStore for HeldNodeRead { + async fn put_opts( + &self, + path: &Path, + payload: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + async fn get_opts( + &self, + path: &Path, + options: object_store::GetOptions, + ) -> object_store::Result { + let result = self.inner.get_opts(path, options).await; + let held = { + let mut target = self.path.lock().unwrap(); + if target.as_ref() == Some(path) { + target.take(); + true + } else { + false + } + }; + if held { + // The provider has captured the old exact record. Publish the real + // heartbeat before delivering it, then let capture reobserve it. + self.entered.notify_one(); + self.resume.notified().await; + } + result + } + async fn put_multipart_opts( + &self, + path: &Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn managed_follower_evacuation_preserves_replacement_across_captured_heartbeat() { + captured_change(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn managed_follower_evacuation_rejects_closed_replacement_after_captured_read() { + captured_change(true).await; +} + +async fn captured_change(close: bool) { + let store = Arc::new(HeldNodeRead::default()); + let fixture = ManagedFixture::with_store(4, 2, store.clone()).await; + let (original, operation) = evacuation::maintenance(&fixture).await; + evacuation::spare(&fixture).await; + evacuation::rotated(&fixture).await; + *store.path.lock().unwrap() = Some(fixture.native.layout.node_path(session(0).as_bytes())); + let (result, ()) = tokio::join!( + evacuation::evidence(&fixture, &original, &operation), + async { + tokio::time::timeout(Duration::from_secs(3), store.entered.notified()) + .await + .unwrap(); + if close { + fixture + .native + .node + .runtime() + .node_durability() + .unwrap() + .1 + .shutdown() + .await + .unwrap(); + } else { + fixture.boots[0] + .refresh_capacity( + 0, + fixture.native.journal.as_ref(), + Instant::now() + Duration::from_secs(3), + ) + .await + .unwrap(); + } + store.resume.notify_one(); + } + ); + let message = result.as_ref().err().map(|error| format!("{error:?}")); + let valid = if close { + matches!(result, Err(Error::Fenced)) + } else { + result.as_ref().is_ok_and(|capture| { + capture + .authority() + .log() + .is_some_and(|log| log.epoch() == 2 && log.members() == [node_id(2), node_id(3)]) + }) + }; + drop(result); + fixture.finish().await; + assert!( + valid, + "unexpected captured authority result: close={close} {message:?}" + ); +} diff --git a/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs b/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs index 0c6bb65c..ef1679b9 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/evacuation.rs @@ -107,7 +107,7 @@ pub(super) async fn rotated(fixture: &ManagedFixture) -> cellule_host::NodeLogRo request } -async fn evidence( +pub(super) async fn evidence( fixture: &ManagedFixture, original: &EnrollmentRecord, operation: &MaintenanceOperation, diff --git a/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs b/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs index 4319d8d1..db97d897 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/fixture/managed.rs @@ -17,12 +17,20 @@ impl ManagedFixture { } pub async fn with_members(count: usize, max_epochs: usize) -> Self { + Self::with_store(count, max_epochs, Arc::new(InMemory::new())).await + } + + pub async fn with_store( + count: usize, + max_epochs: usize, + store: Arc, + ) -> Self { assert!((3..=4).contains(&count)); assert!((1..=3).contains(&max_epochs)); let root = tempfile::tempdir().unwrap(); let app = application::compile().unwrap(); let layout = CellStorageLayout::new( - Store::new(Arc::new(InMemory::new())), + Store::new(store), ObjectPath::from("managed-followers"), [3; 16], ); diff --git a/crates/cellule-host/minion/scenario/follower_tests/mod.rs b/crates/cellule-host/minion/scenario/follower_tests/mod.rs index 84aad55c..2580cb7d 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/mod.rs @@ -29,6 +29,7 @@ use std::sync::{ mod fixture; use fixture::*; mod aggregate; +mod authority_race; mod coverage; mod evacuation; mod inventory; diff --git a/crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs b/crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs new file mode 100644 index 00000000..db4987f3 --- /dev/null +++ b/crates/cellule-host/minion/scenario/follower_tests/persisted/inventory_change.rs @@ -0,0 +1,231 @@ +//! A real actor activation invalidates the follower-policy inventory interval. +use super::*; +use cellule_host::fleet::{ + FleetAdapterFuture, FleetNodeSnapshot, FleetSnapshotRequest, FleetSnapshotSubject, + FleetSnapshotTransport, +}; + +struct HeldInventory { + native: Snapshots, + source_cells: AtomicUsize, + gate: Mutex< + Option<( + tokio::sync::oneshot::Sender<()>, + tokio::sync::oneshot::Receiver<()>, + )>, + >, +} + +fn fleet(fixture: &ManagedFixture) -> crate::scenario::adapters::LocalFleet { + crate::scenario::adapters::LocalFleet { + nodes: fixture.nodes.clone(), + journal: fixture.native.journal.clone(), + boots: fixture.boots.clone(), + records: fixture.records.clone(), + reader_verifier: None, + capture_sequence: std::sync::atomic::AtomicU64::new(0), + lose_release_replies: false, + lost_release_replies: AtomicUsize::new(0), + drop_closed_finalize_replies: AtomicUsize::new(0), + expired_receiver_cleanups: AtomicUsize::new(0), + } +} + +impl FleetSnapshotTransport for HeldInventory { + fn capture<'a>( + &'a self, + request: &'a FleetSnapshotRequest, + end: Instant, + ) -> FleetAdapterFuture<'a, Arc> { + Box::pin(async move { + // Hold the fresh category check after the original complete page. + // Both replies come from the real node; no proof bytes are altered. + if request.node() == node_id(0) + && request.subject() == &FleetSnapshotSubject::Cells(None) + && self.source_cells.fetch_add(1, Ordering::AcqRel) == 1 + { + let gate = self.gate.lock().unwrap().take().unwrap(); + gate.0.send(()).unwrap(); + gate.1.await.unwrap(); + } + self.native.capture(request, end).await + }) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reference_observer_changed_inventory_blocks_policy_until_a_fresh_complete_scan() { + let (fixture, capture, policy) = setup().await; + let publication = publish(&fixture, &capture, policy).await; + let record = publication.record().unwrap().clone(); + let target = CellTarget::new( + TenantId::from_bytes([1; 16]), + scope().application, + application::NAMESPACE, + &[2], + ) + .unwrap(); + let app = application::compile().unwrap(); + let catalog = CellCatalog::new(fixture.native.layout.clone(), target.tenant()) + .provision( + CatalogEntry::new( + &target, + CatalogRole::Sql, + app.registry().module_digests()[0], + 1, + ) + .unwrap(), + ) + .await + .unwrap(); + let incarnation = IncarnationId::from_bytes([2; 16]); + let authority = CellAuthority::new(fixture.native.layout.clone()); + let initial = authority + .create_initial(&catalog, incarnation, owner(0)) + .await + .unwrap(); + let replica = CellReplica::new( + fixture.native.layout.clone(), + *target.cell_id().as_bytes(), + *incarnation.as_bytes(), + limits(), + ) + .unwrap(); + let mut records = fixture + .records + .iter() + .map(|(cell, record)| { + ( + *cell, + Record { + target: record.target.clone(), + incarnation: record.incarnation, + catalog: record.catalog.clone(), + replica: record.replica.clone(), + authority: record.authority.clone(), + }, + ) + }) + .collect::>(); + records.insert( + target.cell_id(), + Record { + target, + incarnation, + catalog: catalog.clone(), + replica: replica.clone(), + authority: authority.clone(), + }, + ); + let mut fleet = fleet(&fixture); + let roster = aggregate::roster(&fixture).await; + let (entered, paused) = tokio::sync::oneshot::channel(); + let (resume, released) = tokio::sync::oneshot::channel(); + let snapshots = Arc::new(HeldInventory { + native: Snapshots::new(&fixture), + source_cells: AtomicUsize::new(0), + gate: Mutex::new(Some((entered, released))), + }); + let mut observation = Box::pin(crate::scenario::observation::observe_with_snapshots( + &fleet, + &roster, + deadline(), + snapshots.clone(), + )); + tokio::select! { + result = &mut observation => panic!("completed before held recheck: {}", result.is_ok()), + result = paused => result.unwrap(), + } + let handle = fixture.nodes[0] + .runtime() + .bootstrap( + catalog, + replica, + authority, + initial, + fixture.native.root.path().join("second.sqlite"), + |tx| { + tx.execute_batch( + "CREATE TABLE counter(value INTEGER); INSERT INTO counter VALUES (31)", + )?; + Ok(()) + }, + ) + .await + .unwrap(); + resume.send(()).unwrap(); + let changed = observation.await; + fleet.records = Arc::new(records); + let fresh = crate::scenario::observation::observe(&fleet, &roster, deadline()).await; + let unchanged = fixture.native.journal.load_snapshot(scope()).await.unwrap(); + let stored_record = stored(&fixture, record.digest().unwrap()).await; + let pages = snapshots.source_cells.load(Ordering::Acquire); + drop((handle, capture, fleet, snapshots)); + fixture.finish().await; + + // Join all native owners before asserting either observer result. + assert_eq!(unchanged, *roster.snapshot()); + assert_eq!(stored_record, record); + assert_eq!(pages, 2); + let changed = changed.unwrap(); + assert!(changed.role_coverage().is_none()); + let missing = changed.maintenance_policy_coverage().unwrap(); + assert!(!missing.is_complete()); + assert_eq!( + (missing.progress().required, missing.progress().checked), + (1, 0) + ); + let fresh = fresh.unwrap(); + let checked = fresh.maintenance_policy_coverage().unwrap(); + assert!(checked.is_complete()); + assert_eq!( + (checked.progress().required, checked.progress().checked), + (1, 1) + ); + assert_eq!( + checked.obligations()[0].status(), + cellule_host::fleet::FleetMaintenancePolicyStatus::Follower(record.digest().unwrap()), + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn reference_observer_fenced_inventory_preserves_the_original_source_error() { + let (fixture, capture, policy) = setup().await; + let publication = publish(&fixture, &capture, policy).await; + let record = publication.record().unwrap().clone(); + let fleet = fleet(&fixture); + let roster = aggregate::roster(&fixture).await; + let (entered, paused) = tokio::sync::oneshot::channel(); + let (resume, released) = tokio::sync::oneshot::channel(); + let snapshots = Arc::new(HeldInventory { + native: Snapshots::new(&fixture), + source_cells: AtomicUsize::new(0), + gate: Mutex::new(Some((entered, released))), + }); + let mut observation = Box::pin(crate::scenario::observation::observe_with_snapshots( + &fleet, + &roster, + deadline(), + snapshots.clone(), + )); + tokio::select! { + result = &mut observation => panic!("completed before held recheck: {}", result.is_ok()), + result = paused => result.unwrap(), + } + fixture.boots[0].guard.as_ref().unwrap().fence(); + resume.send(()).unwrap(); + let result = observation.await; + let stored_record = stored(&fixture, record.digest().unwrap()).await; + drop((capture, fleet, snapshots)); + fixture.finish().await; + assert_eq!(stored_record, record); + let error = result.err().unwrap().downcast::().unwrap(); + let Error::Facility { name, source } = *error else { + panic!("native snapshot source error was flattened"); + }; + assert_eq!(name, "fleet-native-snapshot-transport"); + assert!(matches!( + **source.downcast_ref::>().unwrap(), + Error::Fenced, + )); +} diff --git a/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs b/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs index 922cc05f..e6e63921 100644 --- a/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs +++ b/crates/cellule-host/minion/scenario/follower_tests/persisted/mod.rs @@ -10,6 +10,7 @@ use cellule_host::{ }; use cellule_runtime::fleet::operations::{FollowerEvacuationRecord, FollowerReplacementPolicy}; mod cancelled_settlement; +mod inventory_change; mod maintenance; mod observation; mod races; diff --git a/crates/cellule-host/minion/scenario/mod.rs b/crates/cellule-host/minion/scenario/mod.rs index 0195cff2..b9db2abb 100644 --- a/crates/cellule-host/minion/scenario/mod.rs +++ b/crates/cellule-host/minion/scenario/mod.rs @@ -225,14 +225,20 @@ async fn execute(scenario: Scenario) -> JournalResult { if let Err(error) = node.shutdown().await && cleanup_error.is_none() { - cleanup_error = Some(Box::new(error) as JournalError); + cleanup_error = Some(Box::new(cellule_runtime::Error::Facility { + name: "fleet-example-node-shutdown", + source: Box::new(error), + }) as JournalError); } } for boot in &boots { if let Err(error) = boot.withdraw(&journal).await && cleanup_error.is_none() { - cleanup_error = Some(error); + cleanup_error = Some(Box::new(cellule_runtime::Error::Facility { + name: "fleet-example-boot-withdrawal", + source: error, + }) as JournalError); } } if result.is_ok() && cleanup_error.is_none() { diff --git a/crates/cellule-host/minion/scenario/observation/mod.rs b/crates/cellule-host/minion/scenario/observation/mod.rs index 86163b35..069cefc7 100644 --- a/crates/cellule-host/minion/scenario/observation/mod.rs +++ b/crates/cellule-host/minion/scenario/observation/mod.rs @@ -7,7 +7,7 @@ use cellule_host::fleet::{ FleetFollowerEvacuationVerifier, FleetFollowerReferences, FleetMaintenanceEnrollments, FleetNodeInventory, FleetNodeInventoryScan, FleetNodeSnapshot, FleetObservation, FleetOwnedCell, FleetRoleCoverage, FleetRoster, FleetSnapshotNativePage, FleetSnapshotRequest, - FleetSnapshotSubject, + FleetSnapshotSubject, FleetSnapshotTransport, }; use cellule_runtime::control::{Control, ControlState}; use cellule_runtime::fleet::operations::{ @@ -51,7 +51,24 @@ pub(super) async fn observe( roster: &FleetRoster, deadline: Instant, ) -> JournalResult { - observe_inner(fleet, roster, deadline, None).await + observe_inner( + fleet, + roster, + deadline, + None, + Arc::new(adapters::LocalSnapshots::new(fleet.nodes.clone())), + ) + .await +} + +#[cfg(test)] +pub(super) async fn observe_with_snapshots( + fleet: &adapters::LocalFleet, + roster: &FleetRoster, + deadline: Instant, + snapshots: Arc, +) -> JournalResult { + observe_inner(fleet, roster, deadline, None, snapshots).await } pub(super) async fn observe_with_failed_boot_closure( @@ -67,6 +84,7 @@ pub(super) async fn observe_with_failed_boot_closure( roster, deadline, Some((request, processes, claimant)), + Arc::new(adapters::LocalSnapshots::new(fleet.nodes.clone())), ) .await } @@ -80,8 +98,9 @@ async fn observe_inner( &dyn FleetFailedBootProcesses, SessionId, )>, + snapshots: Arc, ) -> JournalResult { - let capture = collect(fleet, roster, deadline).await?; + let mut capture = collect(fleet, roster, deadline).await?; let mut reader_checks = Vec::new(); let mut follower_checks = Vec::new(); let mut finished = capture.finished; @@ -91,13 +110,23 @@ async fn observe_inner( .collect_maintenance(fleet.journal.as_ref(), original, roster, deadline, clock) .await?; } - let follower_verifier = FleetFollowerEvacuationVerifier::new( - fleet.boots[0].directory.clone(), - Arc::new(adapters::LocalSnapshots::new(fleet.nodes.clone())), - ); - follower_checks = follower_verifier + let follower_verifier = + FleetFollowerEvacuationVerifier::new(fleet.boots[0].directory.clone(), snapshots); + match follower_verifier .collect_maintenance(fleet.journal.as_ref(), original, roster, deadline, clock) - .await?; + .await + { + Ok(checks) => follower_checks = checks, + Err(cellule_runtime::Error::Node("native inventory category changed")) => { + // A live role change invalidates this interval, not the fleet. + // Discard its proofs; the next driver pass must collect them + // afresh before maintenance can settle or finalize any boot. + capture.complete = false; + capture.role_coverage = None; + reader_checks.clear(); + } + Err(source) => return Err(source.into()), + } roster.confirm(fleet.journal.as_ref(), deadline).await?; finished = clock()?; } diff --git a/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs b/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs index c37de3e9..08bcd6ba 100644 --- a/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs +++ b/crates/cellule-host/minion/scenario/successor_tests/routed/recovery_faults/inherited/setup.rs @@ -85,6 +85,9 @@ impl Inherited { ); capacity.follower_free_bytes = capacity.free_disk_bytes.min(64 << 20); capacity.log_protocol = cellule_runtime::node::NODE_LOG_PROTOCOL_VERSION; + // Derive expiry from the same instant: a second clock read can + // cross a millisecond boundary and exceed the maximum lifetime. + let issued_at = clock().unwrap(); let next = NodeAdvertisement::sign( ad.node(), ad.session(), @@ -95,8 +98,8 @@ impl Inherited { ad.release(), &ed25519_dalek::SigningKey::from_bytes(&[index as u8 + 1; 32]), ad.progress(), - clock().unwrap(), - clock().unwrap() + 30_000, + issued_at, + issued_at + 30_000, ad.module_digests().to_vec(), ad.peer_versions().to_vec(), ad.failure_domain().clone(), diff --git a/crates/cellule-host/src/durability/enrollment/maintenance.rs b/crates/cellule-host/src/durability/enrollment/maintenance.rs index f1a3dfa3..2f07cd30 100644 --- a/crates/cellule-host/src/durability/enrollment/maintenance.rs +++ b/crates/cellule-host/src/durability/enrollment/maintenance.rs @@ -370,7 +370,7 @@ impl FleetFollowerEnrollment { .inspect_log_enrollment(&replacement.attempt, now_ms()?) .await? .ok_or(Error::Fenced)?; - let authority = enrollment.enrollment().advertisement().clone(); + let mut authority = enrollment.enrollment().advertisement().clone(); let source = roster.boot(self.node, self.session)?; if source.intent().mode() != NodeMode::Active || replacement.members.iter().any(|member| { @@ -412,16 +412,22 @@ impl FleetFollowerEnrollment { .inspect_log_enrollment(&replacement.attempt, now_ms()?) .await? .ok_or(Error::Fenced)?; - if current.enrollment().advertisement() != &authority - || authority + let observed = current.enrollment().advertisement(); + // Heartbeat, activation and covered-prefix publication can advance + // this same ensemble during suspended journal/directory reads. + // Reobserve their monotone progress; never accept a new boot, closed + // epoch or changed membership as the original replacement witness. + if !same_ensemble(&authority, observed)? + || observed .log() .is_none_or(|log| log.phase() != NodeLogPhase::Open) - || authority + || observed .operational_sample() .is_none_or(|sample| sample.mode != NodeMode::Active) { return Err(Error::Fenced); } + authority.clone_from(observed); let (application, durability) = self.runtime.node_durability().ok_or(Error::Fenced)?; if application != self.scope.application || !node.is_ready() @@ -482,6 +488,25 @@ fn same_boot( && current.issued_at_ms() >= original.issued_at_ms()) } +fn same_ensemble( + original: &NodeAdvertisement, + current: &NodeAdvertisement, +) -> cellule_runtime::Result { + Ok(same_boot(original, current)? + && current.expires_at_ms() >= original.expires_at_ms() + && current.progress() >= original.progress() + && current.log().is_some_and(|log| { + original.log().is_some_and(|old| { + old.phase() == NodeLogPhase::Open + && log.phase() == NodeLogPhase::Open + && log.epoch() == old.epoch() + && log.members() == old.members() + && (!old.active() || log.active()) + && log.tiered_through() >= old.tiered_through() + }) + })) +} + /// Immutable identity shared by native capture and durable revalidation. pub(crate) fn boot_identity(node: &NodeAdvertisement) -> cellule_runtime::Result { let mut hash = blake3::Hasher::new(); diff --git a/crates/cellule-host/tests/node/lifecycle.rs b/crates/cellule-host/tests/node/lifecycle.rs index 32b92a07..6ba468d7 100644 --- a/crates/cellule-host/tests/node/lifecycle.rs +++ b/crates/cellule-host/tests/node/lifecycle.rs @@ -91,6 +91,10 @@ async fn operational_pressure_recovers_acquisition_without_clearing_a_cordon() { async fn node_shutdown_is_idempotent_and_returns_stopped_state() { let pool = SqlWorkerPool::new(1, 1).unwrap(); let host = ReplicaHost::default(); + let process_disk = host.local_disk_budget(); + // Parallel tests charge the default process budget. This fixture owns an + // independent envelope with the same capacity for its exact stats snapshot. + let host = host.with_local_disk_budget(DiskBudget::new(process_disk.capacity())); let node = CellNodeBuilder::new(application()) .with_runtime(pool, 16 * 1024 * 1024) .with_replica_host(host) @@ -102,7 +106,10 @@ async fn node_shutdown_is_idempotent_and_returns_stopped_state() { let starting = node.status(); assert_eq!(starting.state(), NodeState::Starting); assert!(!starting.is_shutting_down()); + // Force the shared-budget interleaving that originally made CI flaky. + let unrelated_disk = process_disk.try_reserve(4096).unwrap(); assert_eq!(starting.stats(), node.stats()); + drop(unrelated_disk); assert!(node.start().is_err()); node.install_task_group(CancellationToken::new(), CancellationToken::new()) .unwrap(); diff --git a/crates/cellule-ltx/docs/publication.md b/crates/cellule-ltx/docs/publication.md index 32668a53..d1d9ad8f 100644 --- a/crates/cellule-ltx/docs/publication.md +++ b/crates/cellule-ltx/docs/publication.md @@ -37,6 +37,14 @@ retry pins and rechecks the selected capture bytes, so path replacement cannot change an in-flight proposal. The host owns request admission, deadlines, and reconciliation after ambiguous results. +The in-memory CRB1 bundle encoder verifies each complete LTX segment before +copying its owned bytes into the envelope. It reparses the complete footer, +checks contiguous extents and duplicate identities, and requires every parsed +row to match the original verified metadata. This avoids a second LTX scan of +the same bytes while preserving the canonical encoding. External memory and +file decoders independently verify every segment; root preparation and fresh +origin verification retain their own checks. + `RootPreparation` identifies a native verified derivation while uploads may still be running. Its construction is private and it grants no uploaded-root, restore, serving, authority or acknowledgement rights. A `RootPreparationMetadata` future diff --git a/crates/cellule-ltx/src/bundle.rs b/crates/cellule-ltx/src/bundle.rs index e0d5821c..58069156 100644 --- a/crates/cellule-ltx/src/bundle.rs +++ b/crates/cellule-ltx/src/bundle.rs @@ -236,7 +236,24 @@ impl Bundle { bytes.extend(footer); bytes.extend(len.to_le_bytes()); bytes.extend(b"CRB1"); - Self::decode(bytes, limits) + let bytes = Bytes::from(bytes); + let decoded = memory_rows(&bytes, limits)?; + // Every complete segment was verified before its bytes moved into + // this private envelope. Reparse and validate the entire footer, then + // require its exact original rows; repeating the LTX scan adds no + // proof about these unchanged owned bytes. External decoders still + // independently verify every segment. + if decoded.len() != rows.len() + || decoded.iter().zip(&rows).any(|(decoded, original)| { + decoded.repository != original.repository + || decoded.epoch != original.epoch + || decoded.info != original.info + || decoded.offset != original.offset + }) + { + return Err(LtxError::LTXCorrupted); + } + Ok(Self::from_validated_memory(bytes, decoded)) } /// Verifies the complete envelope, every extent, and every segment digest. @@ -247,11 +264,7 @@ impl Bundle { /// Verifies a shared envelope without copying its complete body. pub fn decode_bytes(bytes: Bytes, limits: Limits) -> Result { let limits = limits.validate()?; - if bytes.len() as u64 > limits.max_plan_bytes { - return Err(LtxError::Limit(crate::LimitKind::BundleBytes)); - } - let (rows, payload_end) = decode_footer(&bytes, limits)?; - validate_row_layout(&rows, payload_end, limits)?; + let rows = memory_rows(&bytes, limits)?; for row in &rows { let start = usize::try_from(row.offset).map_err(|_| LtxError::LTXCorrupted)?; let end = start @@ -261,12 +274,16 @@ impl Bundle { .ok_or(LtxError::LTXCorrupted)?; crate::recovery::verify_segment(&bytes[start..end], &row.info, limits)?; } - Ok(Self { + Ok(Self::from_validated_memory(bytes, rows)) + } + + fn from_validated_memory(bytes: Bytes, rows: Vec) -> Self { + Self { digest: *blake3::hash(&bytes).as_bytes(), length: bytes.len() as u64, body: BundleBody::Memory(bytes), rows, - }) + } } /// Verifies an envelope from a caller-owned file without retaining its body in memory. @@ -437,6 +454,15 @@ impl Bundle { } } +fn memory_rows(bytes: &[u8], limits: Limits) -> Result> { + if bytes.len() as u64 > limits.max_plan_bytes { + return Err(LtxError::Limit(crate::LimitKind::BundleBytes)); + } + let (rows, payload_end) = decode_footer(bytes, limits)?; + validate_row_layout(&rows, payload_end, limits)?; + Ok(rows) +} + struct BundleUploadSource { body: BundleBody, length: u64, diff --git a/crates/cellule-ltx/tests/cell/bundle.rs b/crates/cellule-ltx/tests/cell/bundle.rs index 8cdf044c..1153ffaa 100644 --- a/crates/cellule-ltx/tests/cell/bundle.rs +++ b/crates/cellule-ltx/tests/cell/bundle.rs @@ -98,6 +98,76 @@ fn encode_rejects_a_duplicate_segment_identity() { assert!(Bundle::encode(vec![entry(), entry()], Limits::default()).is_err()); } +#[test] +fn encode_matches_the_complete_canonical_envelope_and_independent_decoder() { + let directory = tempfile::tempdir().unwrap(); + let (info, body) = segment(directory.path(), 4); + let entries = (1..=4) + .map(|id| BundleEntry::for_cell([id; 32], [id; 16], info.clone(), body.clone())) + .collect::>(); + let mut payload = Vec::new(); + let rows = entries + .iter() + .map(|entry| { + let row = BundleRow { + repository: entry.repository.clone(), + epoch: entry.epoch.clone(), + info: entry.info.clone(), + offset: payload.len() as u64, + }; + payload.extend_from_slice(&entry.bytes); + row + }) + .collect::>(); + let expected = envelope(&rows, payload); + let encoded = Bundle::encode(entries, Limits::default()).unwrap(); + assert_eq!(encoded.read_all().unwrap().as_ref(), expected); + assert_eq!(encoded.digest(), *blake3::hash(&expected).as_bytes()); + let decoded = Bundle::decode(expected, Limits::default()).unwrap(); + assert_eq!(decoded.digest(), encoded.digest()); + for (index, row) in rows.iter().enumerate() { + assert_eq!(decoded.rows()[index].info, row.info); + assert_eq!(decoded.rows()[index].offset, row.offset); + assert_eq!(decoded.read_segment(index).unwrap().as_ref(), body); + } +} + +#[test] +fn encode_still_verifies_ltx_after_its_outer_digest_is_rebound() { + let directory = tempfile::tempdir().unwrap(); + let (mut info, mut body) = segment(directory.path(), 5); + *body.last_mut().unwrap() ^= 1; + info.blake3 = *blake3::hash(&body).as_bytes(); + assert!( + Bundle::encode( + vec![BundleEntry::for_cell([5; 32], [5; 16], info, body)], + Limits::default(), + ) + .is_err() + ); +} + +#[test] +fn encode_includes_the_footer_in_the_complete_plan_bound() { + let directory = tempfile::tempdir().unwrap(); + let (info, body) = segment(directory.path(), 6); + let limits = Limits { + max_capture_bytes: info.size_bytes, + max_file_bytes: info.size_bytes, + max_plan_bytes: info.size_bytes, + ..Limits::default() + }; + assert!(matches!( + Bundle::encode( + vec![BundleEntry::for_cell([6; 32], [6; 16], info, body)], + limits, + ), + Err(cellule_ltx::LtxError::Limit( + cellule_ltx::LimitKind::BundleBytes + )) + )); +} + #[test] fn builder_rejects_more_than_the_segment_limit() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/cellule-ltx/tests/host/hooks/prepare.rs b/crates/cellule-ltx/tests/host/hooks/prepare.rs index 9f52561b..3b4ef74c 100644 --- a/crates/cellule-ltx/tests/host/hooks/prepare.rs +++ b/crates/cellule-ltx/tests/host/hooks/prepare.rs @@ -512,7 +512,13 @@ async fn prepare_opens_captured_segments_concurrently() { Limits::default(), ) .unwrap() - .with_host(Host::default().with_executor(Arc::new(DelayedExecutor { delay }))); + .with_host( + // This test measures the three-open wave, so reserve its own three job + // slots instead of competing for the process-wide, CPU-sized pool. + Host::default() + .with_job_slots(Arc::new(tokio::sync::Semaphore::new(3))) + .with_executor(Arc::new(DelayedExecutor { delay })), + ); let started = tokio::time::Instant::now(); replica.prepare(None, &captured, 1, 1).await.unwrap(); diff --git a/crates/cellule-runtime/docs/append-grants.md b/crates/cellule-runtime/docs/append-grants.md index d02b409a..56a383f7 100644 --- a/crates/cellule-runtime/docs/append-grants.md +++ b/crates/cellule-runtime/docs/append-grants.md @@ -29,6 +29,13 @@ monotonic horizon starts before those reads; wall-clock rollback and slow I/O cannot extend it. This is a bounded authorization horizon, not a device persistence or clock-skew qualification. +Sender verification permits a receiver issue time ahead of its own wall clock +within the existing five-minute node-advertisement skew bound. It still rejects +negative times, expired grants and invalid signed lifetimes. The sender's cached +monotonic horizon starts before issuance I/O and uses the original signed +lifetime, so that allowance adds no lifetime. Receiver issuance, native append +release, sealing and lease checks retain their original strict local horizons. + The signed pruning floor is the lower of fresh authoritative object coverage and the sequence immediately before the requested window. Publication can advance past frames already queued by the source; retaining their first diff --git a/crates/cellule-runtime/docs/failover-and-followers.md b/crates/cellule-runtime/docs/failover-and-followers.md index a90234b7..809d9c3f 100644 --- a/crates/cellule-runtime/docs/failover-and-followers.md +++ b/crates/cellule-runtime/docs/failover-and-followers.md @@ -1640,6 +1640,16 @@ uploading or attaching a manifest grants no transfer authority. The public lineage and Cell CAS, including quiet bindings. It stages bounded catalog cohorts and selects their complete terminal inventory with one recovery-claim node CAS before log seal. Claims are rechecked after I/O; exact lost replies reconcile. +The managed path renews the same claim generation while verification and +materialization run, using the existing 30-second lifetime. Renewal shares the +terminal node CAS lock and ends with the attempt, including cancellation. A dead +claimant, changed proof or expired claim still rejects. Terminal inventory +verification runs outside that lock; sealing checks its exact immutable head +against fresh authority before the final CAS. +A retry after a committed seal adopts the exact recorded result without a new +claim. It checks current controls, the complete original manifest and each +referenced overlay, then rechecks canonical closure and the live claimant. A +stale control or missing payload rejects; adoption writes no recovery state. Partial-root retries reuse the original digest-verified manifest and match every remaining tail's scope, predecessor, complete physical/logical range and bundle digest. A materialized root ahead of the old catalog is only a verified base; diff --git a/crates/cellule-runtime/docs/write-performance-design.md b/crates/cellule-runtime/docs/write-performance-design.md index 8ad08df0..9d8b0bc4 100644 --- a/crates/cellule-runtime/docs/write-performance-design.md +++ b/crates/cellule-runtime/docs/write-performance-design.md @@ -1,16 +1,25 @@ # Node write and read performance design -The latest completed production mixed-load diagnostic reaches 1,249.90 Fleet -writes/s and 12,580.70 reads/s across 2,000 Cells on an 8-vCPU/16-GiB owner. -Write request p99 is 240.5 ms; read request p99 is 48.1 ms. Queued offers are -dropped and root debt grows. A checkpoint-overlap experiment completes fewer -requests and grows more debt in its complete pair, so production retains the -checkpoint serial guard. All 402,167 ACKs across three completed cases pass -independent warm/cold state and original retry audits. The final baseline repeat -fails with VM filesystem I/O errors and is excluded. See the -[current managed-pipeline measurements](../../../docs/managed-bundle-pipeline.md#current-application-measurement). -These 60-second tmpfs diagnostics do not qualify capacity or performance parity; -the acceptance gates below remain unchanged. +A fresh five-minute public-application run at 2,000 Cells sustains 500 offered +Fleet writes/s with zero errors, drops or admission refusals under the unchanged +64-MiB retained-work budget. A fresh repeat after joining authority activation +also has zero errors/drops: all 167,001 ACKs pass warm/cold readback and exact +retries; drain takes 39.25 seconds. Scheduled write p99 is 16.0 ms, and root age +stays around 42–49 seconds. These are development diagnostics with tmpfs local +state; they do not qualify the 2K-write/20K-read target or +physical-media durability. See [fresh publication-progress measurements](../../../docs/publication-progress-measurements.md) +for source identity, actual resources, phase observations and retained failures. +The acceptance gates below remain unchanged. + +The fresh 60-second mixed diagnostic completes 971.25 writes/s and 18,529.95 +reads/s at 2K/20K offered load. Writes have 53,037 errors and 8,662 drops; reads +have zero returned errors and 88,152 drops. Publication backlog reaches the +foreground threshold, root age grows 57.7→88.7 seconds, and checkpoints force +serial publication in 1,019 of 1,022 rounds. A read-only window reaches 15,806.40/s +with zero returned errors and 251,484 drops under sampled competing compiler +load. All 90,562 ACKs pass warm/cold reads and exact retries; drain takes 38.77 +seconds. See the same fresh report for actual host/storage differences and phase +attribution. The full mixed/read targets remain unqualified. The following measurements remain historical observations. diff --git a/crates/cellule-runtime/src/cell/actor/handle.rs b/crates/cellule-runtime/src/cell/actor/handle.rs index cec21e31..ad6462c0 100644 --- a/crates/cellule-runtime/src/cell/actor/handle.rs +++ b/crates/cellule-runtime/src/cell/actor/handle.rs @@ -84,7 +84,9 @@ impl WorkAdmission { // own this exact cut without racing a second global admission. Keep // the input and actual reply charged until their command is dropped. let publication = if publication_bytes != 0 && publication_bytes <= unused { - Some(self._node_bytes.split_retained(publication_bytes)?) + let mut publication = self._node_bytes.split_retained(publication_bytes)?; + publication.set_stage(crate::fleet::resource::RetentionStage::Capture)?; + Some(publication) } else { None }; @@ -587,6 +589,21 @@ impl CellHandle { kind: AdmissionKind, operation_bytes: usize, max_result_bytes: usize, + ) -> crate::Result { + let result = self.reserve_work_kind_inner(kind, operation_bytes, max_result_bytes); + if let Err(error) = &result { + self.inner + .telemetry + .admission_refused(crate::fleet::telemetry::AdmissionRefusal::from_error(error)); + } + result + } + + fn reserve_work_kind_inner( + &self, + kind: AdmissionKind, + operation_bytes: usize, + max_result_bytes: usize, ) -> crate::Result { if self.inner.shutting_down.load(Ordering::Acquire) { return Err(Error::RuntimeClosed); @@ -622,7 +639,7 @@ impl CellHandle { let retained = snapshot.used.retained_bytes(); let limit = snapshot.limit.retained_bytes(); let disk = self.inner.replica_host.local_disk_budget(); - if retained >= limit.saturating_sub(limit / 4) + if retained >= crate::fleet::resource::retained_admission_threshold(limit) || disk.used() >= disk.capacity().saturating_sub(disk.capacity() / 4) { return Err(Error::Capacity("publication backlog")); @@ -643,7 +660,10 @@ impl CellHandle { _node_bytes: self .inner .resources - .try_reserve(ResourceCost::zero().with_retained_bytes(reservation_bytes)) + .try_reserve_in( + crate::fleet::resource::RetentionStage::Request, + ResourceCost::zero().with_retained_bytes(reservation_bytes), + ) .map_err(|error| match error { Error::Capacity(_) => Error::Capacity("node retained bytes"), error => error, diff --git a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs index cb49d4b7..14a5adf2 100644 --- a/crates/cellule-runtime/src/cell/actor/materialization/mod.rs +++ b/crates/cellule-runtime/src/cell/actor/materialization/mod.rs @@ -8,6 +8,16 @@ pub(super) const CHECKPOINT_COMMANDS: u64 = 215; const MAX_ROOT_AGE: std::time::Duration = std::time::Duration::from_secs(45); const MAX_MATERIALIZERS: usize = 8; +fn reserve_materialization( + resources: &crate::fleet::resource::ResourceLedger, + bytes: usize, +) -> crate::Result { + resources.try_reserve_materialization(ResourceCost::zero().with_retained_bytes(bytes)) +} + +#[cfg(test)] +mod tests; + pub(super) fn blocks_commands(active: &ActiveCell) -> bool { active.root_debt.as_ref().is_some_and(|debt| { debt.selected.proof.locator_count() >= CHECKPOINT_COMMANDS as usize @@ -203,6 +213,19 @@ pub(super) fn dispatch( if available == 0 { return; } + // Every root needs at least this original I/O body and prefix metadata. + // Avoid sorting the entire dirty fleet on each SQL event when no parallel + // job can fit. Exact admission below still arbitrates concurrent releases. + let minimum = crate::node::bundle::MAX_BUNDLE_BYTES as usize + + crate::node::bundle::MaterializedBundlePrefix::maximum_retained_bytes(); + if running != 0 + && pool + .resource_ledger() + .materialization_headroom() + .is_ok_and(|(bytes, working)| working && bytes <= minimum) + { + return; + } let now = std::time::Instant::now(); let mut ready: Vec<_> = cells .iter() @@ -233,10 +256,7 @@ pub(super) fn dispatch( ) .ok_or(Error::Capacity("checkpoint metadata admission")) }) - .and_then(|bytes| { - pool.resource_ledger() - .try_reserve(ResourceCost::zero().with_retained_bytes(bytes)) - }); + .and_then(|bytes| reserve_materialization(&pool.resource_ledger(), bytes)); let Ok(reservation) = reservation else { continue; }; @@ -281,10 +301,12 @@ pub(super) fn dispatch( drop(reservation); // Checkpoint this exact cut. Later selected captures can retire // independently; a rebased proof waits for the native witness. - debt.durability.checkpoint_materialized(publisher.authority(), root).await?; + pool.telemetry_handle().publication_phase(crate::fleet::telemetry::NodePublicationPhase::CheckpointWait, + debt.durability.checkpoint_materialized(publisher.authority(), root)).await?; // Transfer pre-admitted metadata to the worker. No admission // can fail after the canonical root and checkpoint are joined. - pool.bind_bundle_materialized(cell, root, Arc::clone(&debt.selected), retained).await + pool.telemetry_handle().publication_phase(crate::fleet::telemetry::NodePublicationPhase::WorkerBinding, + pool.bind_bundle_materialized(cell, root, Arc::clone(&debt.selected), retained)).await }.await; joined.cancel(); publisher.record_publication_timing(crate::fleet::telemetry::PublicationTiming { diff --git a/crates/cellule-runtime/src/cell/actor/materialization/tests.rs b/crates/cellule-runtime/src/cell/actor/materialization/tests.rs new file mode 100644 index 00000000..6a924b81 --- /dev/null +++ b/crates/cellule-runtime/src/cell/actor/materialization/tests.rs @@ -0,0 +1,99 @@ +use super::*; +use crate::fleet::resource::{ResourceCost, ResourceLedger, RetentionStage}; + +#[test] +fn checkpoint_metadata_does_not_hold_the_only_cleanup_admission() { + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(64 << 20)); + let publisher = resources + .try_reserve_in( + RetentionStage::Publisher, + ResourceCost::zero().with_retained_bytes(22 << 20), + ) + .unwrap(); + let selection = resources + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(10 << 20), + ) + .unwrap(); + let mut working = reserve_materialization(&resources, 10 << 20).unwrap(); + assert_eq!(resources.snapshot().unwrap().working_materializations, 1); + let checkpoint = working + .split_retained(crate::node::bundle::MaterializedBundlePrefix::maximum_retained_bytes()) + .unwrap(); + // The first actor is still materializing until its exact checkpoint joins, + // but its original root I/O and working buffers have already exited. + drop(working); + assert_eq!(resources.snapshot().unwrap().working_materializations, 0); + let next = reserve_materialization(&resources, 10 << 20) + .expect("a joined root I/O must release the single cleanup opportunity"); + assert!(reserve_materialization(&resources, 5 << 20).is_err()); + let commands = resources + .try_reserve_in( + RetentionStage::Request, + ResourceCost::zero().with_retained_bytes(4 << 20), + ) + .unwrap(); + let snapshot = resources.snapshot().unwrap(); + assert!( + snapshot.used.retained_bytes() + < crate::fleet::resource::retained_admission_threshold(snapshot.limit.retained_bytes()) + ); + drop(commands); + drop(next); + assert_eq!(resources.snapshot().unwrap().working_materializations, 0); + drop(checkpoint); + drop(selection); + drop(publisher); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); +} + +#[test] +fn working_roots_leave_room_for_foreground_and_existing_debt_can_progress() { + let resources = ResourceLedger::new(ResourceCost::zero().with_retained_bytes(64 << 20)); + let publisher = resources + .try_reserve_in( + RetentionStage::Publisher, + ResourceCost::zero().with_retained_bytes(22 << 20), + ) + .unwrap(); + let mut selected = resources + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(10 << 20), + ) + .unwrap(); + let root = reserve_materialization(&resources, 5 << 20).unwrap(); + assert!( + reserve_materialization(&resources, 5 << 20).is_err(), + "parallel root buffers consumed foreground burst headroom" + ); + let commands = resources + .try_reserve_in( + RetentionStage::Request, + ResourceCost::zero().with_retained_bytes(8 << 20), + ) + .unwrap(); + let snapshot = resources.snapshot().unwrap(); + assert!( + snapshot.used.retained_bytes() + < crate::fleet::resource::retained_admission_threshold(snapshot.limit.retained_bytes()) + ); + drop(commands); + drop(root); + // Already-retained debt must not wait forever for the foreground margin. + let more_debt = resources + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(16 << 20), + ) + .unwrap(); + let cleanup = reserve_materialization(&resources, 5 << 20).unwrap(); + assert!(reserve_materialization(&resources, 5 << 20).is_err()); + drop(more_debt); + selected.shrink_retained(0).unwrap(); + drop(cleanup); + drop(publisher); + drop(selected); + assert_eq!(resources.snapshot().unwrap().used.retained_bytes(), 0); +} diff --git a/crates/cellule-runtime/src/cell/actor/requests.rs b/crates/cellule-runtime/src/cell/actor/requests.rs index fd3fae6d..78d5e2c7 100644 --- a/crates/cellule-runtime/src/cell/actor/requests.rs +++ b/crates/cellule-runtime/src/cell/actor/requests.rs @@ -324,7 +324,10 @@ pub(super) fn reserve_pending_publication( // Larger cuts still need their full charged admission. Returning unused // result capacity first makes it available without changing the node limit. pool.resource_ledger() - .try_reserve(ResourceCost::zero().with_retained_bytes(bytes)) + .try_reserve_in( + crate::fleet::resource::RetentionStage::Capture, + ResourceCost::zero().with_retained_bytes(bytes), + ) .map_err(|error| match error { Error::Capacity(_) => Error::Capacity("pending publication bytes"), error => error, diff --git a/crates/cellule-runtime/src/cell/actor/runtime.rs b/crates/cellule-runtime/src/cell/actor/runtime.rs index 849b4a25..a869d7d3 100644 --- a/crates/cellule-runtime/src/cell/actor/runtime.rs +++ b/crates/cellule-runtime/src/cell/actor/runtime.rs @@ -62,6 +62,13 @@ fn permille(used: u64, limit: u64) -> crate::Result { } impl CellRuntime { + /// Diagnostic retained ownership from one locked resource snapshot. + /// The returned stage counts reconcile exactly to its retained total; + /// they grant no admission or durability proof. + pub fn retained_work(&self) -> crate::Result { + self.inner.resources.snapshot() + } + pub(crate) fn session(&self) -> SessionId { self.inner.session } diff --git a/crates/cellule-runtime/src/control/mod.rs b/crates/cellule-runtime/src/control/mod.rs index a84680e9..34a336bc 100644 --- a/crates/cellule-runtime/src/control/mod.rs +++ b/crates/cellule-runtime/src/control/mod.rs @@ -217,6 +217,29 @@ pub enum Transition { } impl Control { + /// Heap storage retained by an already decoded control. Inline fields are + /// charged by the containing proof; codec buffers have working admission. + pub(crate) fn retained_heap_bytes(&self) -> usize { + // Keep this exhaustive: adding a persisted field requires deciding + // whether its allocation belongs to retained proof admission. + let Self { + cell: _, + incarnation: _, + epoch: _, + revision: _, + progress: _, + state: _, + owner, + root: _, + recovery: _, + bundle_binding: _, + code: _, + schema: _, + next_due_ms: _, + } = self; + owner.as_ref().map_or(0, |owner| owner.endpoint.capacity()) + } + /// Returns this record's incarnation/epoch identity, without granting authority. #[must_use] pub const fn owner_fence(&self) -> OwnerFence { diff --git a/crates/cellule-runtime/src/fleet/resource.rs b/crates/cellule-runtime/src/fleet/resource.rs index b921020e..739a3a54 100644 --- a/crates/cellule-runtime/src/fleet/resource.rs +++ b/crates/cellule-runtime/src/fleet/resource.rs @@ -23,6 +23,16 @@ pub(crate) const READ_REPLICA_NATIVE_BYTES: usize = 12 * 1024 * 1024; pub(crate) const READ_REPLICA_FILE_DESCRIPTORS: usize = 4; pub(crate) const HYDRATION_JOB_CAPACITY: usize = 2; +pub(crate) const fn retained_admission_threshold(limit: usize) -> usize { + limit.saturating_sub(limit / 4) +} + +const fn publication_working_threshold(limit: usize) -> usize { + // Keep the original quarter for publication progress, and another eighth + // below foreground shedding for bursts of admitted SQL/result buffers. + retained_admission_threshold(limit).saturating_sub(limit / 8) +} + /// Bounded resources owned by one runtime admission token. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct ResourceCost { @@ -310,6 +320,50 @@ impl ResourceCost { } } +/// Diagnostic ownership of retained work; stages do not change admission. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RetentionStage { + /// Reservations outside the instrumented publication/request paths. + Other, + /// Accepted operation and response bytes. + Request, + /// Captured data awaiting publication and local cleanup. + Capture, + /// The original publisher's working buffers and metadata cache. + Publisher, + /// Pre-admitted successor buffers and receipt allowance. + Pipeline, + /// Selected dependency proofs retained by original consumers. + Selection, + /// Root reconstruction and joined checkpoint work. + Materialization, +} + +impl RetentionStage { + /// All finite ownership classes, in snapshot order. + pub const ALL: [Self; 7] = [ + Self::Other, + Self::Request, + Self::Capture, + Self::Publisher, + Self::Pipeline, + Self::Selection, + Self::Materialization, + ]; + /// Stable diagnostic label, without Cell or request identity. + pub const fn label(self) -> &'static str { + match self { + Self::Other => "other", + Self::Request => "request", + Self::Capture => "capture", + Self::Publisher => "publisher", + Self::Pipeline => "pipeline", + Self::Selection => "selection", + Self::Materialization => "materialization", + } + } +} + /// Point-in-time usage and limits for one resource ledger. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ResourceSnapshot { @@ -317,6 +371,20 @@ pub struct ResourceSnapshot { pub used: ResourceCost, /// Cost the ledger admits. pub limit: ResourceCost, + /// Retained ownership, indexed by [`RetentionStage::ALL`]. Its sum equals + /// `used.retained_bytes()` in this same locked snapshot. + pub retained_by_stage: [usize; 7], + /// Admitted root I/O lifetimes whose working buffers have not yet exited. + /// Joined roots waiting for their exact checkpoint retain bytes but no + /// working admission. + pub working_materializations: usize, +} + +#[derive(Clone, Copy)] +enum WorkingAdmission { + Unrestricted, + Optional, + Materialization, } #[derive(Clone)] @@ -342,6 +410,8 @@ impl ResourceLedger { snapshot: Mutex::new(ResourceSnapshot { used: ResourceCost::zero(), limit, + retained_by_stage: [0; 7], + working_materializations: 0, }), released: tokio::sync::Notify::new(), }), @@ -353,6 +423,49 @@ impl ResourceLedger { } pub(crate) fn try_reserve(&self, cost: ResourceCost) -> Result { + self.try_reserve_in(RetentionStage::Other, cost) + } + + pub(crate) fn try_reserve_in( + &self, + stage: RetentionStage, + cost: ResourceCost, + ) -> Result { + self.try_reserve_inner(stage, cost, WorkingAdmission::Unrestricted) + } + + /// Optional overlap must leave the foreground's existing progress reserve. + /// Check and admission share the lock so concurrent receipts cannot defeat it. + pub(crate) fn try_reserve_pipeline(&self, cost: ResourceCost) -> Result { + self.try_reserve_inner(RetentionStage::Pipeline, cost, WorkingAdmission::Optional) + } + + pub(crate) fn try_reserve_materialization( + &self, + cost: ResourceCost, + ) -> Result { + self.try_reserve_inner( + RetentionStage::Materialization, + cost, + WorkingAdmission::Materialization, + ) + } + + pub(crate) fn materialization_headroom(&self) -> Result<(usize, bool)> { + let state = self.snapshot()?; + Ok(( + publication_working_threshold(state.limit.retained_bytes()) + .saturating_sub(state.used.retained_bytes()), + state.working_materializations != 0, + )) + } + + fn try_reserve_inner( + &self, + stage: RetentionStage, + cost: ResourceCost, + admission: WorkingAdmission, + ) -> Result { let mut state = self .state .snapshot @@ -365,14 +478,47 @@ impl ResourceLedger { if !next.fits_within(state.limit) { return Err(Error::Capacity("resource ledger")); } + let working = matches!(admission, WorkingAdmission::Materialization); + // One root I/O may reduce debt beyond the working threshold. Its + // original buffers release this opportunity before checkpoint waiting; + // retained prefix metadata cannot serialize unrelated root I/O. + let preserve_headroom = match admission { + WorkingAdmission::Unrestricted => false, + WorkingAdmission::Optional => true, + WorkingAdmission::Materialization => state.working_materializations != 0, + }; + if preserve_headroom + && next.retained_bytes() >= publication_working_threshold(state.limit.retained_bytes()) + { + return Err(Error::Capacity(match stage { + RetentionStage::Materialization => "root materialization headroom", + _ => "bundle pipeline headroom", + })); + } + let working_materializations = state + .working_materializations + .checked_add(usize::from(working)) + .ok_or(Error::Capacity("materialization admission arithmetic"))?; state.used = next; + state.working_materializations = working_materializations; + state.retained_by_stage[stage as usize] += cost.retained_bytes; Ok(ResourceReservation { ledger: self.clone(), cost, + stage, + working, }) } pub(crate) async fn reserve(&self, cost: ResourceCost) -> Result { + self.reserve_in(RetentionStage::Other, cost).await + } + + pub(crate) async fn reserve_in( + &self, + stage: RetentionStage, + cost: ResourceCost, + ) -> Result { if !cost.fits_within(self.snapshot()?.limit) { return Err(Error::Capacity("resource ledger")); } @@ -380,7 +526,7 @@ impl ResourceLedger { // Register before checking capacity so a release between the // failed attempt and await cannot leave the waiter asleep. let released = self.state.released.notified(); - match self.try_reserve(cost) { + match self.try_reserve_in(stage, cost) { Ok(reservation) => return Ok(reservation), Err(Error::Capacity(_)) => released.await, Err(error) => return Err(error), @@ -476,11 +622,16 @@ impl ResourceLedger { Ok(()) } - fn release(&self, cost: ResourceCost) { + fn release(&self, stage: RetentionStage, cost: ResourceCost, working: bool) { if let Ok(mut state) = self.state.snapshot.lock() && let Some(used) = state.used.checked_sub(cost) + && let Some(working_materializations) = state + .working_materializations + .checked_sub(usize::from(working)) { state.used = used; + state.working_materializations = working_materializations; + state.retained_by_stage[stage as usize] -= cost.retained_bytes; } } } @@ -490,9 +641,26 @@ impl ResourceLedger { pub(crate) struct ResourceReservation { ledger: ResourceLedger, cost: ResourceCost, + stage: RetentionStage, + working: bool, } impl ResourceReservation { + /// Reclassifies the original admitted lifetime without changing capacity. + pub(crate) fn set_stage(&mut self, stage: RetentionStage) -> Result<()> { + if stage != self.stage { + let mut state = self + .ledger + .state + .snapshot + .lock() + .map_err(|_| Error::Control("resource ledger lock poisoned"))?; + state.retained_by_stage[self.stage as usize] -= self.cost.retained_bytes; + state.retained_by_stage[stage as usize] += self.cost.retained_bytes; + self.stage = stage; + } + Ok(()) + } pub(crate) const fn retained_bytes(&self) -> usize { self.cost.retained_bytes } @@ -509,6 +677,8 @@ impl ResourceReservation { Ok(Self { ledger: self.ledger.clone(), cost: ResourceCost::zero().with_retained_bytes(bytes), + stage: self.stage, + working: false, }) } @@ -520,8 +690,11 @@ impl ResourceReservation { .checked_sub(bytes) .ok_or(Error::Capacity("capture cleanup cannot grow admission"))?; self.cost.retained_bytes = bytes; - self.ledger - .release(ResourceCost::zero().with_retained_bytes(released)); + self.ledger.release( + self.stage, + ResourceCost::zero().with_retained_bytes(released), + false, + ); self.ledger.state.released.notify_waiters(); Ok(()) } @@ -529,7 +702,7 @@ impl ResourceReservation { impl Drop for ResourceReservation { fn drop(&mut self) { - self.ledger.release(self.cost); + self.ledger.release(self.stage, self.cost, self.working); self.ledger.state.released.notify_waiters(); } } @@ -648,6 +821,48 @@ impl cellule_ltx::HostResourceAdmission for LedgerHostResourceAdmission { mod tests { use super::*; + #[test] + fn stage_transfer_reconciles_without_admitting_more_work() { + let limit = ResourceCost::zero().with_retained_bytes(128); + let ledger = ResourceLedger::new(limit); + let mut request = ledger + .try_reserve_in(RetentionStage::Request, limit) + .unwrap(); + let mut capture = request.split_retained(96).unwrap(); + capture.set_stage(RetentionStage::Capture).unwrap(); + let snapshot = ledger.snapshot().unwrap(); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Request as usize], + 32 + ); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Capture as usize], + 96 + ); + assert_eq!( + snapshot.retained_by_stage.iter().sum::(), + snapshot.used.retained_bytes() + ); + assert!( + ledger + .try_reserve(ResourceCost::zero().with_retained_bytes(1)) + .is_err() + ); + capture.shrink_retained(16).unwrap(); + capture.set_stage(RetentionStage::Selection).unwrap(); + drop(request); + let snapshot = ledger.snapshot().unwrap(); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Selection as usize], + 16 + ); + assert_eq!(snapshot.retained_by_stage.iter().sum::(), 16); + drop(capture); + let snapshot = ledger.snapshot().unwrap(); + assert_eq!(snapshot.used, ResourceCost::zero()); + assert_eq!(snapshot.retained_by_stage, [0; 7]); + } + #[test] fn retained_transfer_preserves_usage_until_each_owner_releases() { let cost = ResourceCost::active_cell().with_retained_bytes(128); diff --git a/crates/cellule-runtime/src/fleet/telemetry.rs b/crates/cellule-runtime/src/fleet/telemetry.rs index 6e965db4..ae7a2342 100644 --- a/crates/cellule-runtime/src/fleet/telemetry.rs +++ b/crates/cellule-runtime/src/fleet/telemetry.rs @@ -183,6 +183,172 @@ pub struct SharedPublicationTiming { pub succeeded: bool, } +/// A bounded node-publication or materialization phase. Durations are +/// nested where documented; overlapping phases must not be summed as latency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NodePublicationPhase { + /// catalog wait observation. + CatalogWait, + /// preparation observation. + Preparation, + /// upload observation. + Upload, + /// verification observation. + Verification, + /// selection observation. + Selection, + /// Authority CAS inside a selection round (nested duration). + AuthorityCas, + /// receipt wait observation. + ReceiptWait, + /// materialization observation. + Materialization, + /// checkpoint observation. + Checkpoint, + /// Queueing and completion of an original materialized-root notification. + CheckpointWait, + /// worker binding observation. + WorkerBinding, +} +impl NodePublicationPhase { + /// Finite diagnostic values in index order. + pub const ALL: [Self; 11] = [ + Self::CatalogWait, + Self::Preparation, + Self::Upload, + Self::Verification, + Self::Selection, + Self::AuthorityCas, + Self::ReceiptWait, + Self::Materialization, + Self::Checkpoint, + Self::CheckpointWait, + Self::WorkerBinding, + ]; + /// Diagnostic label with no request or Cell identity. + pub const fn label(self) -> &'static str { + match self { + Self::CatalogWait => "catalog_wait", + Self::Preparation => "preparation", + Self::Upload => "upload", + Self::Verification => "verification", + Self::Selection => "selection", + Self::AuthorityCas => "authority_cas", + Self::ReceiptWait => "receipt_wait", + Self::Materialization => "materialization", + Self::Checkpoint => "checkpoint", + Self::CheckpointWait => "checkpoint_wait", + Self::WorkerBinding => "worker_binding", + } + } +} +/// Why the original actor refused work before dispatch. +/// A command refusal may still enter the separately bounded retry-resolution lane. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdmissionRefusal { + /// closed observation. + Closed, + /// fenced observation. + Fenced, + /// draining observation. + Draining, + /// publication backlog observation. + PublicationBacklog, + /// mailbox requests observation. + MailboxRequests, + /// mailbox bytes observation. + MailboxBytes, + /// node bytes observation. + NodeBytes, + /// invalid size observation. + InvalidSize, + /// other observation. + Other, +} +impl AdmissionRefusal { + /// Finite diagnostic values in index order. + pub const ALL: [Self; 9] = [ + Self::Closed, + Self::Fenced, + Self::Draining, + Self::PublicationBacklog, + Self::MailboxRequests, + Self::MailboxBytes, + Self::NodeBytes, + Self::InvalidSize, + Self::Other, + ]; + /// Diagnostic label with no request or Cell identity. + pub const fn label(self) -> &'static str { + match self { + Self::Closed => "closed", + Self::Fenced => "fenced", + Self::Draining => "draining", + Self::PublicationBacklog => "publication_backlog", + Self::MailboxRequests => "mailbox_requests", + Self::MailboxBytes => "mailbox_bytes", + Self::NodeBytes => "node_bytes", + Self::InvalidSize => "invalid_size", + Self::Other => "other", + } + } +} + +impl AdmissionRefusal { + pub(crate) fn from_error(error: &crate::Error) -> Self { + use crate::Error; + match error { + Error::Shared(source) => Self::from_error(source), + Error::RuntimeClosed => Self::Closed, + Error::Fenced => Self::Fenced, + Error::CellDraining => Self::Draining, + Error::Capacity("publication backlog") => Self::PublicationBacklog, + Error::Capacity("Cell mailbox requests") => Self::MailboxRequests, + Error::Capacity("Cell mailbox bytes") => Self::MailboxBytes, + Error::Capacity("node retained bytes") => Self::NodeBytes, + Error::Capacity("operation or result bytes" | "mailbox bytes") => Self::InvalidSize, + _ => Self::Other, + } + } +} + +/// Terminal phase outcome; cancellation records elapsed work without implying success. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PublicationPhaseOutcome { + /// The original phase completed successfully. + Succeeded, + /// The original phase returned an error. + Failed, + /// Its future exited without a terminal result. + Cancelled, +} + +/// A frozen native cohort; bytes exclude the catalog/framing overhead. +#[derive(Clone, Copy, Debug)] +pub struct NodePublicationCohort { + /// Complete captured assignments. + pub captures: usize, + /// Native frames in those assignments. + pub frames: usize, + /// Encoded native bytes retained by the captures. + pub bytes: usize, + /// Original materialized-root notifications included in this cohort. + pub checkpoints: usize, +} + +/// Why a ready predecessor did not overlap a successor. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PublicationPipelineFallback { + /// Materialized checkpoints require serial catalog preparation. + Checkpoints, + /// The original ledger could not pre-admit successor buffers and receipts. + Capacity, + /// No successor arrived before predecessor selection completed. + NoSuccessor, + /// Overlap would consume the existing foreground publication headroom. + Headroom, +} + /// Native follower timing for one dispatched append batch. /// One follower append attempt, measured on the blocking worker. /// @@ -492,6 +658,24 @@ pub trait CellTelemetry: Send + Sync { /// record the objects they did upload. fn publication_cost(&self, _objects: u64, _bytes: u64) {} + /// Begins one original phase; pair with its terminal observation to expose + /// in-flight work. Sinks must remain finite and nonblocking. + fn node_publication_started(&self, _phase: NodePublicationPhase) {} + /// Completes one phase, including failed and cancelled work. + fn node_publication_phase( + &self, + _phase: NodePublicationPhase, + _elapsed: Duration, + _outcome: PublicationPhaseOutcome, + ) { + } + /// Observes a frozen managed bundle's density. + fn node_publication_cohort(&self, _cohort: NodePublicationCohort) {} + /// Observes an ordered pipeline's serial fallback reason. + fn node_publication_fallback(&self, _reason: PublicationPipelineFallback) {} + /// Records the specific pre-dispatch refusal; it grants no outcome proof. + fn admission_refused(&self, _reason: AdmissionRefusal) {} + /// Records shared cohort work without Cell identity labels. fn shared_publication(&self, _timing: SharedPublicationTiming) {} @@ -699,6 +883,44 @@ impl CellTelemetryHandle { } } + pub(crate) fn publication_phase( + &self, + phase: NodePublicationPhase, + work: impl std::future::Future>, + ) -> impl std::future::Future> { + // Box at construction, before the observation future can be embedded + // in another async frame. Publication work contains large bounded + // envelopes; inlining it through telemetry can exhaust a worker stack. + let work = Box::pin(work); + let handle = self.clone(); + async move { + let mut observation = PublicationObservation::new(handle, phase); + let result = work.await; + observation.outcome = if result.is_ok() { + PublicationPhaseOutcome::Succeeded + } else { + PublicationPhaseOutcome::Failed + }; + result + } + } + + pub(crate) fn node_publication_cohort(&self, cohort: NodePublicationCohort) { + if let Some(sink) = self.inner.get() { + sink.node_publication_cohort(cohort); + } + } + pub(crate) fn node_publication_fallback(&self, reason: PublicationPipelineFallback) { + if let Some(sink) = self.inner.get() { + sink.node_publication_fallback(reason); + } + } + pub(crate) fn admission_refused(&self, reason: AdmissionRefusal) { + if let Some(sink) = self.inner.get() { + sink.admission_refused(reason); + } + } + pub(crate) fn shared_publication(&self, timing: SharedPublicationTiming) { if let Some(telemetry) = self.inner.get() { telemetry.shared_publication(timing); @@ -917,3 +1139,105 @@ mod tests { ); } } + +// This guard follows the polled original future, including cancellation. It +// owns no authority or work reservation and never changes the source error. +struct PublicationObservation { + sink: Option>, + phase: NodePublicationPhase, + started: std::time::Instant, + outcome: PublicationPhaseOutcome, +} +impl PublicationObservation { + fn new(telemetry: CellTelemetryHandle, phase: NodePublicationPhase) -> Self { + let sink = telemetry.inner.get().cloned(); + if let Some(sink) = &sink { + sink.node_publication_started(phase); + } + Self { + sink, + phase, + started: std::time::Instant::now(), + outcome: PublicationPhaseOutcome::Cancelled, + } + } +} +impl Drop for PublicationObservation { + fn drop(&mut self) { + if let Some(sink) = &self.sink { + sink.node_publication_phase(self.phase, self.started.elapsed(), self.outcome); + } + } +} + +#[cfg(test)] +mod publication_tests { + use super::*; + use std::sync::Mutex; + + #[derive(Default)] + struct Sink { + events: Mutex>, + } + impl CellTelemetry for Sink { + fn node_publication_phase( + &self, + phase: NodePublicationPhase, + _: Duration, + outcome: PublicationPhaseOutcome, + ) { + self.events.lock().unwrap().push((phase, outcome)); + } + } + + #[tokio::test] + async fn phase_observation_keeps_large_work_out_of_its_caller_frame() { + let handle = CellTelemetryHandle::default(); + let bytes = [0x5a_u8; 128 << 10]; + let work = async move { + tokio::task::yield_now().await; + Ok(bytes.iter().map(|byte| u64::from(*byte)).sum::()) + }; + let observed = handle.publication_phase(NodePublicationPhase::Materialization, work); + assert!( + std::mem::size_of_val(&observed) <= 1024, + "phase observation embeds {} bytes of accepted work in its caller", + std::mem::size_of_val(&observed), + ); + assert_eq!(observed.await.unwrap(), 0x5a * (128 << 10)); + } + + #[tokio::test] + async fn original_phase_reports_failure_and_cancellation_without_replacing_source() { + let sink = Arc::new(Sink::default()); + let handle = CellTelemetryHandle::from_sink(sink.clone()); + let result: crate::Result<()> = handle + .publication_phase(NodePublicationPhase::Upload, async { + Err(crate::Error::Capacity("original source")) + }) + .await; + assert!(matches!( + result, + Err(crate::Error::Capacity("original source")) + )); + let mut pending = Box::pin(handle.publication_phase( + NodePublicationPhase::ReceiptWait, + std::future::pending::>(), + )); + assert!(futures_util::poll!(pending.as_mut()).is_pending()); + drop(pending); + assert_eq!( + *sink.events.lock().unwrap(), + vec![ + ( + NodePublicationPhase::Upload, + PublicationPhaseOutcome::Failed + ), + ( + NodePublicationPhase::ReceiptWait, + PublicationPhaseOutcome::Cancelled + ), + ] + ); + } +} diff --git a/crates/cellule-runtime/src/follower/grant/tests.rs b/crates/cellule-runtime/src/follower/grant/tests.rs index b71e0ac1..0dd8fcf5 100644 --- a/crates/cellule-runtime/src/follower/grant/tests.rs +++ b/crates/cellule-runtime/src/follower/grant/tests.rs @@ -146,6 +146,34 @@ impl Fixture { } } +#[tokio::test] +async fn receiver_clock_ahead_preserves_the_exact_signed_append_window() { + let mut f = Fixture::new().await; + let grant = f.grant(1).await.unwrap(); + // A sender can receive a freshly issued grant before its own wall clock + // reaches the receiver's issue time. Native authorization still uses the + // receiver's original monotonic horizon and terminal lease. + let decoded = NodeAppendGrant::verify(&grant.encode(), &f.receiver, NOW - 1).unwrap(); + assert_eq!(decoded.digest(), grant.digest()); + assert_eq!(decoded.issued_at_ms(), NOW); + assert_eq!(decoded.expires_at_ms(), NOW + APPEND_GRANT_LIFETIME_MS); + assert!(!decoded.valid_at(NOW - 1)); + assert!(NodeAppendGrant::verify(&grant.encode(), &f.receiver, NOW - 5 * 60_000 - 1,).is_err()); + assert!( + NodeAppendGrant::verify(&grant.encode(), &f.receiver, decoded.expires_at_ms()).is_err() + ); + f.backend.reset(); + let frame = f.frame(1); + assert_eq!( + f.append(&decoded, vec![frame]) + .await + .unwrap() + .durable_through, + 1 + ); + assert!(f.backend.requests().is_empty()); +} + #[tokio::test] async fn signed_grant_binds_every_byte_and_amortizes_directory_reads() { let mut f = Fixture::new().await; diff --git a/crates/cellule-runtime/src/node/append_grant/mod.rs b/crates/cellule-runtime/src/node/append_grant/mod.rs index bf5c514b..f8ff01bd 100644 --- a/crates/cellule-runtime/src/node/append_grant/mod.rs +++ b/crates/cellule-runtime/src/node/append_grant/mod.rs @@ -24,6 +24,8 @@ pub struct NodeAppendGrant { impl NodeAppendGrant { /// Verifies the exact receiver signature and the caller's original boot pin. + /// Receiver issue times use the same bounded clock skew as advertisements; + /// this cannot extend the receiver's registered monotonic authorization. pub fn verify(encoded: &[u8], receiver: &NodeAdvertisement, now_ms: i64) -> Result { if encoded.len() != BODY_BYTES + 64 || &encoded[..8] != MAGIC { return Err(Error::PeerAuthorization("invalid append grant encoding")); @@ -46,7 +48,11 @@ impl NodeAppendGrant { || grant.receiver_certificate() != receiver.certificate || grant.receiver_key() != receiver.public_key || grant.expires_at_ms() > receiver.expires_at_ms - || !grant.valid_at(now_ms) + || now_ms < 0 + || grant.issued_at_ms() > now_ms.saturating_add(MAX_CLOCK_SKEW_MS) + // This is the sender's clock, while issuance and native proof + // release use the receiver's clock and original monotonic horizon. + || !grant.valid_at(now_ms.max(grant.issued_at_ms())) || grant.last_sequence().checked_sub(grant.first_sequence()) != Some(APPEND_GRANT_SEQUENCES - 1) || grant.first_sequence() == 0 @@ -111,7 +117,8 @@ impl NodeAppendGrant { pub fn expires_at_ms(&self) -> i64 { self.number(368) as i64 } - /// Whether wall-clock policy still permits this signed grant. + /// Whether the receiver's wall clock still permits this signed grant. + /// Senders use [`Self::verify`] for bounded differences between clocks. #[must_use] pub fn valid_at(&self, now_ms: i64) -> bool { self.issued_at_ms() >= 0 diff --git a/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs b/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs index 090e44dc..a2b9a86f 100644 --- a/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/index/preparation/mod.rs @@ -38,8 +38,10 @@ const BODY_BYTES: usize = PREPARATION_BYTES impl BundlePreparation { pub(crate) fn reserve(resources: &ResourceLedger) -> Result { - let reservation = - resources.try_reserve(ResourceCost::zero().with_retained_bytes(PREPARATION_BYTES))?; + let reservation = resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Publisher, + ResourceCost::zero().with_retained_bytes(PREPARATION_BYTES), + )?; Ok(Self { scope: None, entries: std::iter::repeat_with(|| None) diff --git a/crates/cellule-runtime/src/node/bundle/mod.rs b/crates/cellule-runtime/src/node/bundle/mod.rs index e08cee6b..c40fa51a 100644 --- a/crates/cellule-runtime/src/node/bundle/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/mod.rs @@ -196,6 +196,44 @@ impl StagedNodeBundle { pub const fn head(&self) -> NodeBundleHead { self.head } + + fn retained_metadata_bytes(&self) -> Result { + if self.catalog.index.is_some() { + return Err(Error::Node("staged bundle retains preparation index")); + } + let assignments = self + .assignments + .capacity() + .checked_mul(std::mem::size_of::()) + .ok_or(Error::Capacity("staged bundle metadata"))?; + let fixed = self + .catalog + .bindings + .capacity() + .checked_mul(std::mem::size_of::()) + .and_then(|bytes| bytes.checked_add(assignments)) + .and_then(|bytes| bytes.checked_add(std::mem::size_of::() + 256)) + .ok_or(Error::Capacity("staged bundle metadata"))?; + self.catalog + .bindings + .iter() + .try_fold(fixed, |bytes, binding| { + binding + .locators + .capacity() + .checked_mul(std::mem::size_of::()) + .and_then(|locators| bytes.checked_add(locators)) + .and_then(|bytes| bytes.checked_add(binding.control.retained_heap_bytes())) + .and_then(|bytes| bytes.checked_add(256)) + .ok_or(Error::Capacity("staged bundle metadata")) + }) + } + + pub(crate) fn retained_work_bytes(&self) -> Result { + self.retained_metadata_bytes()? + .checked_add(self.body.len()) + .ok_or(Error::Capacity("staged bundle memory")) + } } /// Selected, dependency-verified coverage of one exact Cell writer. @@ -292,7 +330,10 @@ impl BundleCoverageProof { } pub(crate) fn retained_metadata_bytes(&self) -> Result { - let control = self.binding.control.encode()?.len(); + // This proof retains one decoded control, never serialized copies. + // Charge its endpoint allocation by capacity and its inline fields + // through Self below. Encoding scratch belongs to publication work. + let control_heap = self.binding.control.retained_heap_bytes(); self.binding .locators .capacity() @@ -304,7 +345,7 @@ impl BundleCoverageProof { .map_or(0, |live| live.retained_metadata_bytes()), ) }) - .and_then(|bytes| bytes.checked_add(control.checked_mul(4)?)) + .and_then(|bytes| bytes.checked_add(control_heap)) .and_then(|bytes| bytes.checked_add(std::mem::size_of::() + 256)) .ok_or(Error::Capacity("selected bundle metadata")) } diff --git a/crates/cellule-runtime/src/node/bundle/proof.rs b/crates/cellule-runtime/src/node/bundle/proof.rs index 4f75fcb6..a45dec4a 100644 --- a/crates/cellule-runtime/src/node/bundle/proof.rs +++ b/crates/cellule-runtime/src/node/bundle/proof.rs @@ -3,6 +3,7 @@ use super::*; use crate::control::authority::{CellAuthority, VersionedControl}; use crate::node::NodeDirectory; use crate::node::directory::NodeRecord; +use futures_util::{StreamExt, stream::FuturesOrdered}; impl NodeDirectory { /// Reopens one authority-pinned selected Cell suffix from canonical origin. @@ -150,23 +151,65 @@ async fn verify_binding_into( ) -> Result<()> { verify_base(layout, binding, limits).await?; let mut chain = BindingChain::new(binding)?; - for locator in &binding.locators { - let object = locator - .object - .ok_or(Error::Node("bundle locator is unresolved"))?; - let end = locator - .offset - .checked_add(locator.bytes) - .ok_or(Error::Node("bundle locator overflow"))?; - let bytes = - origin::read_range(layout, session, epoch, object, locator.offset..end, origin).await?; - let frame = checked_frame(session, epoch, binding, locator, bytes, limits)?; - chain.accept(FrameStep::from_frame(&frame))?; - if let Some(frames) = &mut frames { - frames.push(frame); + let mut pending = binding.locators.iter(); + let mut reads = FuturesOrdered::new(); + for locator in pending.by_ref().take(verification::READ_CONCURRENCY) { + reads.push_back(read_frame( + layout, session, epoch, binding, locator, limits, origin, + )); + } + // The original suffix admission already covers all retained frame bodies. + // Overlap bounded GETs, consume facts in exact locator order, and join the + // accepted window after failure without dispatching any later reads. + let mut failure = None; + while let Some(result) = reads.next().await { + if failure.is_some() { + continue; + } + let result = result.and_then(|frame| { + chain.accept(FrameStep::from_frame(&frame))?; + if let Some(frames) = &mut frames { + frames.push(frame); + } + Ok(()) + }); + if let Err(error) = result { + failure = Some(error); + } else if let Some(locator) = pending.next() { + reads.push_back(read_frame( + layout, session, epoch, binding, locator, limits, origin, + )); } } - chain.finish(binding) + match failure { + Some(error) => Err(error), + None => chain.finish(binding), + } +} + +#[allow( + clippy::too_many_arguments, + reason = "one exact authenticated locator and origin scope" +)] +async fn read_frame( + layout: &cellule_ltx::CellStorageLayout, + session: SessionId, + epoch: u64, + binding: &Binding, + locator: &Locator, + limits: cellule_ltx::Limits, + origin: Option<&origin::OriginBundle>, +) -> Result { + let object = locator + .object + .ok_or(Error::Node("bundle locator is unresolved"))?; + let end = locator + .offset + .checked_add(locator.bytes) + .ok_or(Error::Node("bundle locator overflow"))?; + let bytes = + origin::read_range(layout, session, epoch, object, locator.offset..end, origin).await?; + checked_frame(session, epoch, binding, locator, bytes, limits) } /// Checked frame facts retained only within one verification operation. They diff --git a/crates/cellule-runtime/src/node/bundle/recovery/drain.rs b/crates/cellule-runtime/src/node/bundle/recovery/drain.rs index 4f0f4414..b429571d 100644 --- a/crates/cellule-runtime/src/node/bundle/recovery/drain.rs +++ b/crates/cellule-runtime/src/node/bundle/recovery/drain.rs @@ -1,6 +1,6 @@ //! Terminal catalog selection owned by a verified complete recovery attempt. use super::*; -use crate::node::log_recovery::RecoveryCell; +use crate::node::log_recovery::{RecoveryCell, RecoveryClaimLease}; use crate::recovery::manifest::RecoveryManifestStore; use std::collections::BTreeMap; @@ -30,6 +30,7 @@ impl BundleRecoveryDrain { cells: &[RecoveryCell], durable_through: u64, limits: cellule_ltx::Limits, + claim: Option<&RecoveryClaimLease<'_>>, ) -> Result> { let Some(original) = fenced.bundle_head() else { return Ok(None); @@ -42,7 +43,15 @@ impl BundleRecoveryDrain { let mut scopes = BTreeMap::new(); let mut closed = BTreeMap::new(); let mut unpinned = BTreeMap::new(); - for binding in fenced_bindings(layout, fenced).await? { + let bindings = if let Some(claim) = claim { + // Renewal changes expiry, while the fenced immutable head stays + // fixed. Check the lease's exact current proof before reading it. + claim.verify().await?; + index::binding_inventory(layout, fenced.session(), original).await? + } else { + fenced_bindings(layout, fenced).await? + }; + for binding in bindings { if binding.phase == BindingPhase::Closed { if binding.terminal != Some(( @@ -216,9 +225,8 @@ impl BundleRecoveryDrain { directory: &crate::node::NodeDirectory, manifests: &RecoveryManifestStore, fenced: &FencedNodeSession, - now_ms: i64, - started: std::time::Instant, - ) -> Result<(FencedNodeSession, Vec)> { + claim: &RecoveryClaimLease<'_>, + ) -> Result> { if fenced.bundle_head() != Some(self.original) || directory.layout.node_path(fenced.session().as_bytes()) != manifests.layout().node_path(fenced.session().as_bytes()) @@ -229,7 +237,7 @@ impl BundleRecoveryDrain { } let mut controls = Vec::with_capacity(self.scopes.len()); for scope in self.scopes.values() { - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + claim.verify().await?; let authority = &scope.cell.authority; let mut current = authority .load(scope.binding.control.cell) @@ -259,7 +267,7 @@ impl BundleRecoveryDrain { let successor = current .value() .publish_recovery(&prepared, current.value().next_due_ms)?; - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + claim.verify().await?; current = match authority .transition( ¤t, @@ -349,15 +357,14 @@ impl BundleRecoveryDrain { catalog.selected_through = self.durable_through; let prepared = directory.upload_catalog(Some(head), catalog, &[]).await?; staged = prepared.head; - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + claim.verify().await?; } // Intermediate cohort objects are immutable staging, never authority. // Select the fully closed inventory with one original-claim node CAS. store::ensure_session_drained(manifests.layout(), fenced.session(), Some(staged)).await?; - let (mut current, token) = - check_claim(directory, fenced, logical_now(now_ms, started)?).await?; + let (mut proof, mut current, token) = claim.lock_verified().await?; if staged == self.original { - return Ok((fenced.clone(), controls)); + return Ok(controls); } current.bundle = Some(staged); let path = manifests.layout().node_path(fenced.session().as_bytes()); @@ -370,11 +377,15 @@ impl BundleRecoveryDrain { { // Lost reply only reconciles this exact immutable head, original // claim generation, expiry and recovering log. - check_claim(directory, &fresh, logical_now(now_ms, started)?) - .await - .map_err(|_| Error::from(source))?; + let actual = directory.load_record_at(&path).await?; + if !matches!(actual, Some((NodeRecord::Tombstone(current), _)) if current.fenced().is_ok_and(|current| current == fresh)) + || claim.now()? >= fresh.claim_expires_at_ms() + { + return Err(source.into()); + } } - Ok((fresh, controls)) + *proof = fresh; + Ok(controls) } } @@ -387,26 +398,6 @@ fn same_scope(original: &Control, current: &Control) -> bool { && original.schema == current.schema } -async fn check_claim( - directory: &crate::node::NodeDirectory, - fenced: &FencedNodeSession, - now_ms: i64, -) -> Result<(crate::node::directory::NodeTombstone, cellule_store::ETag)> { - directory - .load(fenced.claimant(), now_ms) - .await? - .ok_or(Error::Fenced)?; - let path = directory.layout.node_path(fenced.session().as_bytes()); - let Some((NodeRecord::Tombstone(current), token)) = directory.load_record_at(&path).await? - else { - return Err(Error::Fenced); - }; - if current.fenced()? != *fenced || now_ms >= fenced.claim_expires_at_ms() { - return Err(Error::Fenced); - } - Ok((*current, token)) -} - pub(crate) fn logical_now(now_ms: i64, started: std::time::Instant) -> Result { now_ms .checked_add( diff --git a/crates/cellule-runtime/src/node/bundle/selection.rs b/crates/cellule-runtime/src/node/bundle/selection.rs index 465b10be..63142ec0 100644 --- a/crates/cellule-runtime/src/node/bundle/selection.rs +++ b/crates/cellule-runtime/src/node/bundle/selection.rs @@ -126,7 +126,8 @@ impl NodeDirectory { } // Cloned locator arrays cannot exceed the validated history bound. // Collected live assignments grow geometrically within the 64-frame - // cohort. Control-owned strings/maps retain their existing 4x charge. + // cohort. Preserve the conservative codec/endpoint preflight allowance; + // transfer only concrete retained allocations after verification. let origin = self.layout.node_path(&[0; 16]).to_string(); let each = MAX_LOCATORS * std::mem::size_of::() + MAX_FRAMES * std::mem::size_of::() @@ -381,48 +382,61 @@ impl NodeDirectory { if prepared.assignments.is_empty() { return Err(Error::Node("native bundle has no complete assignments")); } - let origin = origin::OriginBundle::load(&self.layout, prepared).await?; - let bindings = origin.selected_bindings(prepared)?; - let mut fresh = Vec::with_capacity(bindings.len()); - for binding in &bindings { - if !extension::verify( - &self.layout, - prepared, - lease, - binding, - prefixes, - limits, - &origin, - )? { - fresh.push(binding); - } - } - verification::verify_cohort( - &self.layout, - prepared.catalog.session, - prepared.catalog.epoch, - &fresh, - limits, - &origin, - ) - .await?; - let mut proofs = Vec::new(); - for binding in bindings { - let pin = binding - .control - .bundle_binding - .ok_or(Error::Node("selected binding lacks pin"))?; - proofs.push(BundleCoverageProof { - pin, - binding, - head: prepared.head, - session: prepared.catalog.session, - live: None, - }); - } + let mut proofs = self + .telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::Verification, + async { + let origin = origin::OriginBundle::load(&self.layout, prepared).await?; + let bindings = origin.selected_bindings(prepared)?; + let mut fresh = Vec::with_capacity(bindings.len()); + for binding in &bindings { + if !extension::verify( + &self.layout, + prepared, + lease, + binding, + prefixes, + limits, + &origin, + )? { + fresh.push(binding); + } + } + verification::verify_cohort( + &self.layout, + prepared.catalog.session, + prepared.catalog.epoch, + &fresh, + limits, + &origin, + ) + .await?; + let mut proofs = Vec::new(); + for binding in bindings { + let pin = binding + .control + .bundle_binding + .ok_or(Error::Node("selected binding lacks pin"))?; + proofs.push(BundleCoverageProof { + pin, + binding, + head: prepared.head, + session: prepared.catalog.session, + live: None, + }); + } + Ok(proofs) + }, + ) + .await?; lease.check()?; let selected = self - .select_native_catalog(observed, prepared, now_ms) + .telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::AuthorityCas, + self.select_native_catalog(observed, prepared, now_ms), + ) .await?; // Expiry during verification must not revive the original writer. lease.check()?; diff --git a/crates/cellule-runtime/src/node/bundle/store.rs b/crates/cellule-runtime/src/node/bundle/store.rs index 4c7451ac..8a88e092 100644 --- a/crates/cellule-runtime/src/node/bundle/store.rs +++ b/crates/cellule-runtime/src/node/bundle/store.rs @@ -33,13 +33,47 @@ impl NodeDirectory { digest, selected_through: catalog.selected_through, }; - Ok(std::sync::Arc::new(StagedNodeBundle { + // Encoding and self-verification consumed the preparation snapshot. + // Successor assembly reads its immutable bytes; live selection needs + // only the new native bindings. Keep neither sibling rows nor their + // duplicate loaded histories across upload/selection lifetimes. + catalog.index = None; + let rows = std::mem::take(&mut catalog.bindings); + let count = rows + .iter() + .filter(|binding| { + binding + .locators + .iter() + .any(|locator| locator.object.is_none()) + }) + .count(); + let mut selected = Vec::with_capacity(count); + for binding in rows { + if binding + .locators + .iter() + .any(|locator| locator.object.is_none()) + { + selected.push(binding); + } + } + catalog.bindings = selected; + let staged = StagedNodeBundle { original, catalog, body, head, assignments, - })) + }; + // A retained proposal is at most one encoded object plus one bounded + // native metadata allocation. Shared preparation/verification scratch + // can account for that concrete lifetime independently of temporary + // decoder/encoder allocations, which have already joined above. + if staged.retained_metadata_bytes()? > MAX_BUNDLE_BYTES as usize { + return Err(Error::Capacity("staged bundle metadata")); + } + Ok(std::sync::Arc::new(staged)) } /// Uploads already-encoded immutable bytes, without selecting authority or diff --git a/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs new file mode 100644 index 00000000..f66b1fec --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/checkpoint_pipeline.rs @@ -0,0 +1,428 @@ +//! A checkpoint-bearing predecessor can overlap a native successor safely. +use super::actor::{Authority, transport}; +use super::composed::HeldSelection; +use super::pipeline::ManagedStore; +use super::receipt_pressure::submission; +use super::*; +use crate::cell::worker::SqlWorkerPool; +use crate::node::durability::NodeDurability; +use crate::node::log_shipper::NodeLogShipper; +use futures_util::poll; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +#[tokio::test] +async fn checkpoint_predecessor_overlaps_successor_and_cold_restores_both_exact_histories() { + checkpoint_case(128 << 20, Outcome::Complete).await; +} + +#[tokio::test] +async fn checkpoint_predecessor_overlaps_with_original_64_mib_budget() { + checkpoint_case(64 << 20, Outcome::Complete).await; +} + +#[tokio::test] +async fn both_cohorts_compose_checkpoints_and_credit_their_own_ordered_selections() { + checkpoint_case(64 << 20, Outcome::SuccessorCheckpoint).await; +} + +#[tokio::test] +async fn failed_checkpoint_predecessor_joins_accepted_successor_and_preserves_upload_source() { + checkpoint_case(64 << 20, Outcome::FailedUpload).await; +} + +#[tokio::test] +async fn fenced_checkpoint_predecessor_joins_uploads_after_shutdown_waiter_cancellation() { + checkpoint_case(64 << 20, Outcome::Fenced).await; +} + +#[tokio::test] +async fn checkpoint_base_missing_after_upload_prevents_both_selections_and_joins_successor() { + checkpoint_case(64 << 20, Outcome::MissingRoot).await; +} + +#[derive(Clone, Copy)] +enum Outcome { + Complete, + SuccessorCheckpoint, + FailedUpload, + Fenced, + MissingRoot, +} + +async fn checkpoint_case(retained: usize, outcome: Outcome) { + let store = Arc::new(ManagedStore::default()); + let mut f = Fixture::with_store(store.clone()).await; + super::coverage::enroll(&mut f).await; + let mut cells = [f.cell(4).await, f.cell(5).await]; + let original = Arc::new(Authority { + directory: f.directory.clone(), + observed: tokio::sync::Mutex::new(f.node.clone()), + }); + let authority = Arc::new(HeldSelection { + original: original.clone(), + calls: AtomicUsize::new(0), + combined: AtomicUsize::new(0), + largest_combined: AtomicUsize::new(0), + hold_after: 2, + entered: tokio::sync::Notify::new(), + resume: tokio::sync::Notify::new(), + }); + let peers = transport(&f, true); + let shipper = NodeLogShipper::new(f.gate.clone(), peers.clone(), Limits::default()).unwrap(); + let durability = Arc::new(NodeDurability::new( + f.gate.clone(), + shipper, + original, + peers, + f.lease.clone(), + )); + let pool = SqlWorkerPool::new(1, 1).unwrap(); + pool.configure_retained_capacity(retained).unwrap(); + durability + .attach_selection_resources(pool.resource_ledger()) + .unwrap(); + durability + .start_bundle_publication(authority.clone()) + .unwrap(); + + let seed = durability + .submit_capture(submission(&mut cells[0], 2)) + .await + .unwrap(); + let seed_selected = seed.selection.as_ref().unwrap().selected().await.unwrap(); + let root = f + .publisher(&cells[0]) + .materialize_bundle(&seed_selected.proof) + .await + .unwrap(); + // Joining the original catalog lock proves the seed round has returned; + // a ready receipt alone can precede the round's final cleanup. + drop(authority.original.observed.lock().await); + store.armed.store(true, Ordering::Release); + let filler = durability + .submit_capture(submission(&mut cells[1], 2)) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), store.entered.notified()) + .await + .unwrap(); + // Fill the prior round's successor before queueing the checkpoint. Without + // this barrier the checkpoint can become that successor, and this test + // would measure the intentional two-cohort bound instead of its guard. + let prior_successor = durability + .submit_capture(submission(&mut cells[1], 3)) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), store.second_done.notified()) + .await + .unwrap(); + // Consume the prior successor's entry permit before resetting its PUT + // counter. Otherwise it can certify the next cohort before that upload + // enters, and fencing legitimately prevents admission instead of testing + // that shutdown joins an accepted PUT. + tokio::time::timeout(Duration::from_secs(3), store.second_entered.notified()) + .await + .unwrap(); + store.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(3), authority.entered.notified()) + .await + .unwrap(); + let successor_checkpoint = matches!(outcome, Outcome::SuccessorCheckpoint); + let second_base = if successor_checkpoint { + let selected = filler.selection.as_ref().unwrap().selected().await.unwrap(); + let root = f + .publisher(&cells[1]) + .materialize_bundle(&selected.proof) + .await + .unwrap(); + Some((root, selected)) + } else { + None + }; + let first = durability + .submit_capture(submission(&mut cells[0], 3)) + .await + .unwrap(); + let mut checkpoint = Box::pin(tokio::task::unconstrained( + durability.checkpoint_materialized(cells[0].authority.clone(), root, seed_selected.clone()), + )); + assert!(poll!(checkpoint.as_mut()).is_pending()); + + // Queue the original callback and native work behind the prior selection; + // the held PUT therefore belongs to their combined catalog proposal. + // Both prior PUTs joined before resetting this task-local fault counter. + store.puts.store(0, Ordering::Release); + authority.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(3), store.entered.notified()) + .await + .unwrap(); + let fault = matches!( + outcome, + Outcome::FailedUpload | Outcome::Fenced | Outcome::MissingRoot + ); + store + .hold_second + .store(fault || successor_checkpoint, Ordering::Release); + store + .fail_first + .store(matches!(outcome, Outcome::FailedUpload), Ordering::Release); + let mut second_checkpoint = second_base.as_ref().map(|(root, selected)| { + Box::pin(tokio::task::unconstrained( + durability.checkpoint_materialized(cells[1].authority.clone(), *root, selected.clone()), + )) + }); + if let Some(callback) = &mut second_checkpoint { + assert!(poll!(callback.as_mut()).is_pending()); + } + let second = durability + .submit_capture(submission(&mut cells[1], 4)) + .await + .unwrap(); + let successor_entered = if fault || successor_checkpoint { + &store.second_entered + } else { + &store.second_done + }; + let overlap = tokio::time::timeout(Duration::from_secs(2), successor_entered.notified()) + .await + .is_ok(); + assert_eq!( + store.puts.load(Ordering::Acquire), + 2, + "current successor PUT has not entered" + ); + assert!(!first.selection.as_ref().unwrap().is_ready()); + assert!(!second.selection.as_ref().unwrap().is_ready()); + assert!( + poll!(checkpoint.as_mut()).is_pending(), + "upload supplies no checkpoint credit" + ); + assert_eq!( + durability.progress().unwrap().tiered_through, + prior_successor.assignment.ticket().last_sequence() + ); + if fault { + let prior_head = f + .directory + .load(SessionId::from_bytes([1; 16]), NOW) + .await + .unwrap() + .unwrap() + .advertisement() + .bundle_head(); + if matches!(outcome, Outcome::Fenced) { + f.lease.fence(); + } + let root_path = f.layout.incarnation_object_path( + &root.cell, + &root.incarnation, + &root.digest, + cellule_ltx::CellObjectKind::Root, + ); + if matches!(outcome, Outcome::MissingRoot) { + f.count.block_body_reads_for(&root_path); + } + let mut joined = Box::pin(durability.shutdown()); + assert!(poll!(joined.as_mut()).is_pending()); + store.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(3), f.lease.wait_fenced()) + .await + .unwrap(); + let checkpoint_error = tokio::time::timeout(Duration::from_secs(3), checkpoint.as_mut()) + .await + .unwrap() + .unwrap_err(); + if matches!(outcome, Outcome::FailedUpload) { + assert!(format!("{checkpoint_error:?}").contains("first bundle PUT failed")); + } + assert!( + poll!(joined.as_mut()).is_pending(), + "accepted successor PUT was abandoned" + ); + assert_eq!( + durability.progress().unwrap().tiered_through, + prior_successor.assignment.ticket().last_sequence() + ); + assert_eq!( + f.directory + .load(SessionId::from_bytes([1; 16]), NOW) + .await + .unwrap() + .unwrap() + .advertisement() + .bundle_head(), + prior_head + ); + drop(joined); + store.second_resume.notify_one(); + let error = tokio::time::timeout(Duration::from_secs(3), durability.shutdown()) + .await + .unwrap() + .unwrap_err(); + if matches!(outcome, Outcome::FailedUpload) { + assert!(format!("{error:?}").contains("first bundle PUT failed")); + } + if matches!(outcome, Outcome::MissingRoot) { + f.count.unblock_body_reads_for(&root_path); + } + drop(checkpoint); + drop(seed_selected); + drop(seed); + drop(filler); + drop(prior_successor); + drop(first); + drop(second); + assert_eq!( + pool.resource_ledger() + .snapshot() + .unwrap() + .used + .retained_bytes(), + 0 + ); + pool.shutdown().await.unwrap(); + assert!(overlap, "fault did not exercise both accepted uploads"); + return; + } + store.resume.notify_one(); + let (checkpoint_result, first_selected, second_selected) = if successor_checkpoint { + let (checkpoint_result, first_selected) = + tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!( + checkpoint.as_mut(), + first.selection.as_ref().unwrap().selected() + ) + }) + .await + .unwrap(); + checkpoint_result.unwrap(); + assert!(poll!(second_checkpoint.as_mut().unwrap().as_mut()).is_pending()); + assert!(!second.selection.as_ref().unwrap().is_ready()); + assert_eq!( + durability.progress().unwrap().tiered_through, + first.assignment.ticket().last_sequence() + ); + store.second_resume.notify_one(); + let (checkpoint_result, second_selected) = + tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!( + second_checkpoint.as_mut().unwrap(), + second.selection.as_ref().unwrap().selected() + ) + }) + .await + .unwrap(); + (checkpoint_result, first_selected, second_selected) + } else { + tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!( + checkpoint.as_mut(), + first.selection.as_ref().unwrap().selected(), + second.selection.as_ref().unwrap().selected() + ) + }) + .await + .unwrap() + }; + checkpoint_result.unwrap(); + let selected = [first_selected.unwrap(), second_selected.unwrap()]; + assert_eq!( + authority.combined.load(Ordering::Acquire), + if successor_checkpoint { 2 } else { 1 } + ); + assert_eq!(selected[0].proof.base().unwrap(), root); + if let Some((root, _)) = &second_base { + assert_eq!(selected[1].proof.base().unwrap(), *root); + } + + for (index, (cell, selected)) in cells.iter_mut().zip(&selected).enumerate() { + let overlay = selected + .proof + .recovery_overlay(&f.layout, Limits::default()) + .await + .unwrap(); + let restored = cell + .replica + .prepare_recovered_overlay(&overlay, 1) + .await + .unwrap(); + let destination = f + .scratch + .path() + .join(format!("checkpoint-pipeline-{index}.sqlite")); + cell.replica + .open_root(&restored.root()) + .await + .unwrap() + .restore(&destination) + .await + .unwrap(); + let database = rusqlite::Connection::open(destination).unwrap(); + let outcomes: Vec<(String, String)> = database + .prepare("SELECT request, result FROM outcomes ORDER BY request") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap(); + let mut expected = vec![ + ("request-2".into(), "result-2".into()), + ("request-3".into(), "result-3".into()), + ]; + if index == 1 { + expected.push(("request-4".into(), "result-4".into())); + } + expected.push(("seed".into(), "original".into())); + assert_eq!(outcomes, expected); + cell.control = cell + .authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap(); + let root = f + .publisher(cell) + .materialize_bundle(&selected.proof) + .await + .unwrap(); + durability + .checkpoint_materialized(cell.authority.clone(), root, selected.clone()) + .await + .unwrap(); + cell.control = cell + .authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap(); + durability + .close_bundle_cell(&cell.authority, &cell.control) + .await + .unwrap(); + } + durability.shutdown().await.unwrap(); + drop(selected); + drop(seed_selected); + drop(seed); + drop(filler); + drop(prior_successor); + drop(first); + drop(second); + drop(checkpoint); + drop(second_checkpoint); + drop(second_base); + assert_eq!( + pool.resource_ledger() + .snapshot() + .unwrap() + .used + .retained_bytes(), + 0 + ); + pool.shutdown().await.unwrap(); + assert!( + overlap, + "checkpoint-bearing predecessor serialized immutable successor work" + ); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/composed.rs b/crates/cellule-runtime/src/node/bundle/tests/composed.rs index 7438091b..566de604 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/composed.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/composed.rs @@ -11,14 +11,14 @@ use crate::node::log_shipper::NodeLogShipper; use futures_util::{future::BoxFuture, poll}; use std::sync::atomic::{AtomicUsize, Ordering}; -struct HeldSelection { - original: Arc, - calls: AtomicUsize, - combined: AtomicUsize, - largest_combined: AtomicUsize, - hold_after: usize, - entered: tokio::sync::Notify, - resume: tokio::sync::Notify, +pub(super) struct HeldSelection { + pub(super) original: Arc, + pub(super) calls: AtomicUsize, + pub(super) combined: AtomicUsize, + pub(super) largest_combined: AtomicUsize, + pub(super) hold_after: usize, + pub(super) entered: tokio::sync::Notify, + pub(super) resume: tokio::sync::Notify, } impl NodeBundleAuthority for HeldSelection { fn bind<'a>( diff --git a/crates/cellule-runtime/src/node/bundle/tests/faults.rs b/crates/cellule-runtime/src/node/bundle/tests/faults.rs index fadc2927..2ba8fe40 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/faults.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/faults.rs @@ -122,6 +122,16 @@ impl ObjectStore for ReplyFault { } async fn get_opts(&self, path: &Path, opts: GetOptions) -> object_store::Result { let mode = self.mode.load(Ordering::SeqCst); + if mode == 17 + && path.as_ref().ends_with(".root") + && self + .mode + .compare_exchange(17, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + // Exercise a real I/O wait beyond the production claim lifetime. + tokio::time::sleep(std::time::Duration::from_secs(33)).await; + } if mode == 16 && path.as_ref().ends_with("/control.json") { self.control_started.fetch_add(1, Ordering::SeqCst); self.node_started.notify_one(); diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs b/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs index 7a45f480..03f9a2f5 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/encoding.rs @@ -42,6 +42,23 @@ async fn dense_multi_cell_encoding_preserves_the_original_native_order_and_bytes frames.extend(capture); assignments.push(assignment); } + let keys = cells + .iter() + .map(|cell| { + ( + *f.layout.application_id(), + *cell.control.value().cell.as_bytes(), + ) + }) + .collect(); + let mut original_catalog = crate::node::bundle::store::load_catalog_cells( + &f.layout, + f.node.advertisement().session(), + f.node.advertisement().bundle_head().unwrap(), + &keys, + ) + .await + .unwrap(); let proposal = f .directory .prepare_node_bundle(&f.node, &frames, &assignments, now) @@ -52,12 +69,28 @@ async fn dense_multi_cell_encoding_preserves_the_original_native_order_and_bytes // roots include generated identities, so a stored digest is not a // stable cross-run wire fixture. Both encoders must agree byte for byte // over the same 64-Cell cohort and its unrelated catalog siblings. - let mut original_catalog = proposal.catalog.clone(); + // Staging now discards preparation snapshots and unrelated live rows + // after encoding. Compare against the complete original input catalog, + // preserving every sibling and its predecessor shard/history references. + original_catalog.selected_through = proposal.catalog.selected_through; + original_catalog.predecessor = proposal.catalog.predecessor; + for binding in &proposal.catalog.bindings { + *original_catalog + .binding_mut(binding.control.bundle_binding.unwrap().digest) + .unwrap() = binding.clone(); + } let (original_body, original_digest) = catalog_index::encode_original(&mut original_catalog, &frames).unwrap(); assert_eq!(proposal.body, original_body); assert_eq!(proposal.head.digest, original_digest); - assert_eq!(proposal.catalog, original_catalog); + for binding in &proposal.catalog.bindings { + assert_eq!( + binding, + original_catalog + .binding_mut(binding.control.bundle_binding.unwrap().digest) + .unwrap() + ); + } let mut next_offset = None; for frame in &frames { let locators: Vec<_> = proposal diff --git a/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs index bcd4bea6..6b33bf47 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/index/pipeline.rs @@ -67,6 +67,17 @@ async fn successor_stages_before_upload_but_requires_ordered_selection_and_full_ first.head.epoch, first.head.digest.as_bytes(), ); + assert!( + first.catalog.index.is_none(), + "encoded proposal retained duplicate preparation histories" + ); + assert_eq!( + first.catalog.bindings.len(), + 64, + "staged metadata retains unrelated inventory rows" + ); + assert_eq!(first.catalog.bindings.capacity(), 64); + assert!(first.retained_work_bytes().unwrap() <= 2 * MAX_BUNDLE_BYTES as usize); let mut second_frames = Vec::new(); let mut second_assignments = Vec::new(); for cell in &mut cells { diff --git a/crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs b/crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs new file mode 100644 index 00000000..27d0b430 --- /dev/null +++ b/crates/cellule-runtime/src/node/bundle/tests/materialization_reads.rs @@ -0,0 +1,249 @@ +use super::*; +use futures_util::stream::BoxStream; +use object_store::ObjectStore; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Notify; + +#[derive(Debug, Default)] +struct HeldFrameStore { + inner: InMemory, + armed: AtomicBool, + reads: AtomicUsize, + fail_second: AtomicBool, + failed_path: std::sync::Mutex>, + paths: std::sync::Mutex>, + first: Notify, + second: Notify, + third: Notify, + resume: Notify, + resume_third: Notify, +} +impl std::fmt::Display for HeldFrameStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("HeldFrameStore") + } +} +#[async_trait::async_trait] +impl ObjectStore for HeldFrameStore { + async fn put_opts( + &self, + path: &Path, + payload: object_store::PutPayload, + options: object_store::PutOptions, + ) -> object_store::Result { + self.inner.put_opts(path, payload, options).await + } + async fn put_multipart_opts( + &self, + path: &Path, + options: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(path, options).await + } + async fn get_opts( + &self, + path: &Path, + options: object_store::GetOptions, + ) -> object_store::Result { + if self.armed.load(Ordering::Acquire) + && options.range.is_some() + && path.as_ref().ends_with(".nodebundle") + { + self.paths.lock().unwrap().insert(path.to_string()); + if self.fail_second.load(Ordering::Acquire) + && self.failed_path.lock().unwrap().as_deref() == Some(path.as_ref()) + { + return Err(object_store::Error::Generic { + store: "HeldFrameStore", + source: std::io::Error::other("deliberate historical read failure").into(), + }); + } + match self.reads.fetch_add(1, Ordering::AcqRel) { + 0 => { + self.first.notify_one(); + self.resume.notified().await; + } + 1 => { + self.second.notify_one(); + if self.fail_second.load(Ordering::Acquire) { + *self.failed_path.lock().unwrap() = Some(path.to_string()); + return Err(object_store::Error::Generic { + store: "HeldFrameStore", + source: std::io::Error::other("deliberate historical read failure") + .into(), + }); + } + } + 2 if self.fail_second.load(Ordering::Acquire) => { + self.third.notify_one(); + self.resume_third.notified().await; + } + _ => {} + } + } + self.inner.get_opts(path, options).await + } + fn delete_stream( + &self, + paths: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(paths) + } + fn list( + &self, + prefix: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +#[tokio::test] +async fn failed_historical_window_joins_accepted_reads_and_publishes_no_root() { + let store = Arc::new(HeldFrameStore::default()); + let mut f = Fixture::with_store(store.clone()).await; + let mut cell = f.cell(4).await; + let mut proof = None; + for commit in 2..=11 { + let (_, frames, assigned) = f.append(&mut cell, commit); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (node, mut proofs) = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + f.node = node; + proof = proofs.pop(); + } + let before = cell.control.value().encode().unwrap(); + let proof = proof.unwrap(); + let mut publisher = f.publisher(&cell); + store.fail_second.store(true, Ordering::Release); + store.armed.store(true, Ordering::Release); + let materialized = publisher.materialize_bundle(&proof); + tokio::pin!(materialized); + tokio::select! { + _ = store.third.notified() => {}, + _ = &mut materialized => panic!("accepted historical reads must remain joined"), + _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => panic!("historical window was not dispatched"), + } + store.resume.notify_one(); + tokio::select! { + _ = &mut materialized => panic!("failure escaped while an accepted read remained held"), + _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => {}, + } + // Transport retries of the same missing frame are not another dispatched + // locator. The accepted window and its one earlier refill remain bounded. + let dispatched = store.paths.lock().unwrap().len(); + assert!(dispatched <= verification::READ_CONCURRENCY + 1); + store.resume_third.notify_one(); + let error = materialized.await.unwrap_err(); + assert_eq!(store.paths.lock().unwrap().len(), dispatched); + let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut found = false; + while let Some(error) = source { + found |= error + .to_string() + .contains("deliberate historical read failure"); + source = error.source(); + } + assert!(found, "original read failure was replaced: {error:?}"); + assert_eq!( + cell.authority + .load(cell.control.value().cell) + .await + .unwrap() + .unwrap() + .value() + .encode() + .unwrap(), + before + ); +} + +#[tokio::test] +async fn materialization_overlaps_historical_reads_before_exact_cold_restore() { + let store = Arc::new(HeldFrameStore::default()); + let mut f = Fixture::with_store(store.clone()).await; + let mut cell = f.cell(4).await; + let mut proof = None; + for commit in [2, 3] { + let (_, frames, assigned) = f.append(&mut cell, commit); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (node, mut proofs) = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + f.node = node; + proof = proofs.pop(); + } + let proof = proof.unwrap(); + assert_eq!(proof.locator_count(), 2); + let mut publisher = f.publisher(&cell); + store.armed.store(true, Ordering::Release); + let materialized = publisher.materialize_bundle(&proof); + tokio::pin!(materialized); + tokio::select! { + _ = store.first.notified() => {}, + _ = &mut materialized => panic!("first historical read must remain held"), + } + let overlapped = tokio::select! { + _ = store.second.notified() => true, + _ = tokio::time::sleep(std::time::Duration::from_millis(250)) => false, + _ = &mut materialized => panic!("root cannot publish before its first frame"), + }; + assert!( + overlapped, + "historical reads remained serial under the original suffix admission" + ); + store.resume.notify_one(); + let root = materialized.await.unwrap(); + assert_eq!(root.commit_sequence, 3); + let restored = f.scratch.path().join("overlapped-restore.sqlite"); + cell.replica + .open_root(&root) + .await + .unwrap() + .restore(&restored) + .await + .unwrap(); + let db = rusqlite::Connection::open(restored).unwrap(); + let mut stmt = db + .prepare("SELECT request, result FROM outcomes ORDER BY request") + .unwrap(); + let rows = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + rows, + vec![ + ("request-2".into(), "result-2".into()), + ("request-3".into(), "result-3".into()), + ("seed".into(), "original".into()) + ] + ); +} diff --git a/crates/cellule-runtime/src/node/bundle/tests/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/mod.rs index c931b40d..60f8664f 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/mod.rs @@ -17,12 +17,14 @@ const NOW: i64 = 1_000_000; const EPOCH: u64 = 2; mod actor; mod checkpoint_credit; +mod checkpoint_pipeline; mod composed; mod coverage; mod faults; mod index; mod lifecycle; mod managed; +mod materialization_reads; mod pipeline; mod publication_hooks; mod ranges; diff --git a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs index c1646a7f..dfca904b 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/pipeline.rs @@ -12,17 +12,17 @@ use futures_util::stream::BoxStream; use object_store::ObjectStore; use std::sync::atomic::{AtomicBool, Ordering}; #[derive(Debug, Default)] -struct ManagedStore { +pub(super) struct ManagedStore { inner: InMemory, - armed: AtomicBool, - puts: std::sync::atomic::AtomicUsize, - hold_second: AtomicBool, - second_entered: tokio::sync::Notify, - second_done: tokio::sync::Notify, - second_resume: tokio::sync::Notify, - fail_first: AtomicBool, - entered: tokio::sync::Notify, - resume: tokio::sync::Notify, + pub(super) armed: AtomicBool, + pub(super) puts: std::sync::atomic::AtomicUsize, + pub(super) hold_second: AtomicBool, + pub(super) second_entered: tokio::sync::Notify, + pub(super) second_done: tokio::sync::Notify, + pub(super) second_resume: tokio::sync::Notify, + pub(super) fail_first: AtomicBool, + pub(super) entered: tokio::sync::Notify, + pub(super) resume: tokio::sync::Notify, } impl std::fmt::Display for ManagedStore { @@ -324,6 +324,68 @@ async fn tight_original_budget_uses_one_cohort_and_returns_all_credit() { case.finish([first, second]).await; } +#[tokio::test] +async fn optional_overlap_preserves_foreground_headroom_with_live_selected_debt() { + use crate::fleet::resource::{ResourceCost, RetentionStage}; + let mut case = Case::new(64 << 20, false, false).await; + // Model other Cells' original selected receipts while this cohort's PUT + // is stalled. The optional successor must not make fresh SQL shed itself. + let selected_debt = case + .pool + .resource_ledger() + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(14 << 20), + ) + .unwrap(); + let first = case.first().await; + let second = case.second().await; + let overlapped = tokio::time::timeout( + Duration::from_millis(250), + case.store.second_done.notified(), + ) + .await + .is_ok(); + assert!( + !overlapped, + "optional overlap consumed foreground progress headroom" + ); + let snapshot = case.pool.resource_ledger().snapshot().unwrap(); + assert_eq!( + snapshot.retained_by_stage[RetentionStage::Pipeline as usize], + 0 + ); + assert!(snapshot.used.retained_bytes() < snapshot.limit.retained_bytes() * 3 / 4); + case.store.resume.notify_one(); + drop(selected_debt); + case.finish([first, second]).await; +} + +#[tokio::test] +async fn compact_pipeline_overlaps_with_selected_debt_without_consuming_foreground_headroom() { + use crate::fleet::resource::{ResourceCost, RetentionStage}; + let mut case = Case::new(64 << 20, false, false).await; + let selected_debt = case + .pool + .resource_ledger() + .try_reserve_in( + RetentionStage::Selection, + ResourceCost::zero().with_retained_bytes(8 << 20), + ) + .unwrap(); + let first = case.first().await; + let second = case.second().await; + tokio::time::timeout(Duration::from_secs(2), case.store.second_done.notified()) + .await + .unwrap(); + let snapshot = case.pool.resource_ledger().snapshot().unwrap(); + assert!(snapshot.used.retained_bytes() < snapshot.limit.retained_bytes() * 3 / 4); + assert_eq!(case.durability.progress().unwrap().tiered_through, 0); + case.store.resume.notify_one(); + drop(selected_debt); + case.finish([first, second]).await; +} + #[tokio::test] async fn fenced_join_waits_for_both_accepted_uploads_after_caller_cancellation() { failed_join(false).await; diff --git a/crates/cellule-runtime/src/node/bundle/tests/ranges.rs b/crates/cellule-runtime/src/node/bundle/tests/ranges.rs index 1f38b6a8..3afdb445 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/ranges.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/ranges.rs @@ -1,6 +1,45 @@ use super::*; use object_store::ObjectStoreExt as _; +#[tokio::test] +async fn retained_proof_charges_endpoint_capacity_without_serialized_control_copies() { + let mut f = Fixture::new().await; + let mut cell = f.cell(4).await; + let (_, frames, assigned) = f.append(&mut cell, 2); + let prepared = f + .directory + .prepare_node_bundle(&f.node, &frames, &[assigned], NOW) + .await + .unwrap(); + let (_, mut proofs) = f + .directory + .select_node_bundle(&f.node, &prepared, &f.lease, Limits::default(), NOW) + .await + .unwrap(); + let mut proof = proofs.pop().unwrap(); + let original = proof.retained_metadata_bytes().unwrap(); + let owner = proof.binding.control.owner.as_mut().unwrap(); + let original_capacity = owner.endpoint.capacity(); + // Identical authority bytes can retain a larger allocation. Accounting + // must observe capacity, rather than infer storage from serialized length. + let encoded = owner.endpoint.clone(); + owner.endpoint.reserve_exact(8192); + let extra = owner.endpoint.capacity() - original_capacity; + assert_eq!(owner.endpoint, encoded); + assert_eq!(proof.retained_metadata_bytes().unwrap(), original + extra); + + // One small selected proof should not retain transient codec buffers. + // This is a footprint ceiling, independent of the accounting formula. + assert!( + original < 2048, + "selected proof retains serialized control scratch" + ); + assert!( + proof.retained_metadata_bytes().unwrap() + <= f.directory.bundle_receipt_memory_bound(1).unwrap() + ); +} + #[tokio::test] async fn missing_base_chunk_cannot_mint_a_selected_reconstruction_proof() { let mut f = Fixture::new().await; diff --git a/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs b/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs index 495fb8ea..e90608ec 100644 --- a/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs +++ b/crates/cellule-runtime/src/node/bundle/tests/recovery/mod.rs @@ -125,6 +125,7 @@ async fn fence(f: &Fixture) -> crate::node::FencedNodeSession { enum Scenario { Pruned, Complete, + SlowRecovery, PartialRoot, LostRootReply, LostCatalogReply, @@ -141,6 +142,7 @@ impl Scenario { matches!( self, Self::Complete + | Self::SlowRecovery | Self::PartialRoot | Self::LostRootReply | Self::LostCatalogReply @@ -173,6 +175,11 @@ async fn recovery_materializes_full_fleet_suffix_and_closes_quiet_binding_before verify_recovery(Scenario::Complete).await; } +#[tokio::test] +async fn live_claimant_renews_recovery_during_root_verification_beyond_claim_lifetime() { + verify_recovery(Scenario::SlowRecovery).await; +} + #[tokio::test] async fn recovery_retries_after_one_root_cas_without_replacing_the_original_manifest() { verify_recovery(Scenario::PartialRoot).await; @@ -617,10 +624,38 @@ async fn verify_recovery(scenario: Scenario) { cells = discovered.cells; coordinator = RecoveryCoordinator::new(resumed, manifests.clone()); } - let completed = coordinator - .recover_and_seal(&f.directory, completion_fence, cells, NOW + 31_002) - .await - .unwrap(); + let completed = if matches!(scenario, Scenario::SlowRecovery) { + faults.mode.store(17, std::sync::atomic::Ordering::SeqCst); + let now = NOW + 31_002; + let started = std::time::Instant::now(); + let mut claimant = f + .directory + .load(completion_fence.claimant(), now) + .await + .unwrap() + .unwrap(); + let recovery = coordinator.recover_and_seal(&f.directory, completion_fence, cells, now); + tokio::pin!(recovery); + loop { + tokio::select! { + result = &mut recovery => break result.unwrap(), + () = tokio::time::sleep(std::time::Duration::from_secs(5)) => { + let current_now = now + i64::try_from(started.elapsed().as_millis()).unwrap(); + let mut next = claimant.advertisement().clone(); + next.issued_at_ms = current_now; + next.expires_at_ms = current_now + 30_000; + next.progress += 1; + next.signature = SigningKey::from_bytes(&[10; 32]).sign(&next.signing_bytes().unwrap()).to_bytes(); + claimant = f.directory.refresh(&claimant, next, current_now).await.unwrap(); + } + } + } + } else { + coordinator + .recover_and_seal(&f.directory, completion_fence, cells, NOW + 31_002) + .await + .unwrap() + }; assert_eq!( completed.controls.len(), if matches!(scenario, Scenario::SealInterrupted) { diff --git a/crates/cellule-runtime/src/node/directory/mod.rs b/crates/cellule-runtime/src/node/directory/mod.rs index f13e8ede..806ec409 100644 --- a/crates/cellule-runtime/src/node/directory/mod.rs +++ b/crates/cellule-runtime/src/node/directory/mod.rs @@ -40,6 +40,7 @@ pub struct NodeDirectory { // maintenance continue to read their canonical records directly. reader_membership: Arc>>, inventory_scope: [u8; 16], + pub(super) telemetry: crate::fleet::telemetry::CellTelemetryHandle, } /// Request verifier bound to one mTLS-authenticated enrollment observation. @@ -106,9 +107,21 @@ impl NodeDirectory { recovery_scan: Arc::new(RwLock::new(None)), reader_membership: Arc::new(RwLock::new(None)), inventory_scope: rand::random(), + telemetry: crate::fleet::telemetry::CellTelemetryHandle::default(), } } + /// Uses the application's original runtime sink for bundle verification and + /// selection diagnostics. Observations grant no availability or authority. + #[must_use] + pub fn with_telemetry( + mut self, + telemetry: crate::fleet::telemetry::CellTelemetryHandle, + ) -> Self { + self.telemetry = telemetry; + self + } + /// Returns the fleet this directory is scoped to. #[must_use] pub const fn fleet(&self) -> Digest { @@ -573,10 +586,7 @@ impl NodeTombstone { } pub(super) fn renew(mut self, fenced: &FencedNodeSession, now_ms: i64) -> Result { - if self.session != fenced.session - || self.claimant != Some(fenced.claimant) - || self.claim_generation != fenced.claim_generation - { + if self.fenced()? != *fenced { return Err(Error::Fenced); } let current_expiry = self diff --git a/crates/cellule-runtime/src/node/directory/recovery.rs b/crates/cellule-runtime/src/node/directory/recovery.rs index c642dbf2..01b919a7 100644 --- a/crates/cellule-runtime/src/node/directory/recovery.rs +++ b/crates/cellule-runtime/src/node/directory/recovery.rs @@ -6,6 +6,14 @@ use super::*; +/// Verified immutable closure, bound to this store and exact failed-session head. +pub(crate) struct PreparedRecoverySeal { + session: SessionId, + head: Option, + path: object_store::path::Path, + store_identity: u64, +} + impl NodeDirectory { /// Reads live scheduler membership and primes the advisory recovery scan. /// @@ -501,34 +509,72 @@ impl NodeDirectory { }; let renewed = (*current).renew(fenced, now_ms)?; let proof = renewed.fenced()?; - self.layout + if let Err(source) = self + .layout .store() .update(&path, Bytes::from(renewed.encode()?), token) - .await?; + .await + { + // A lost reply can credit only this exact renewal, never another + // claimant, generation, log or terminal catalog selection. + let actual = self.load_record_at(&path).await?; + if !matches!(actual, Some((NodeRecord::Tombstone(current), _)) if current.fenced().is_ok_and(|current| current == proof)) + { + return Err(source.into()); + } + } Ok(proof) } + pub(crate) async fn prepare_recovery_seal( + &self, + fenced: &FencedNodeSession, + ) -> Result { + crate::node::bundle::store::ensure_session_drained( + &self.layout, + fenced.session(), + fenced.bundle_head(), + ) + .await?; + Ok(PreparedRecoverySeal { + session: fenced.session(), + head: fenced.bundle_head(), + path: self.layout.node_path(fenced.session().as_bytes()), + store_identity: self.layout.immutable_cache_identity(), + }) + } + /// Seals an exact recovery claim after every affected Cell pins its overlay. pub(crate) async fn seal_recovery( &self, fenced: &FencedNodeSession, recovery_manifest: Option, now_ms: i64, + ) -> Result { + let prepared = self.prepare_recovery_seal(fenced).await?; + self.seal_recovery_prepared(fenced, recovery_manifest, now_ms, &prepared) + .await + } + + pub(crate) async fn seal_recovery_prepared( + &self, + fenced: &FencedNodeSession, + recovery_manifest: Option, + now_ms: i64, + prepared: &PreparedRecoverySeal, ) -> Result { let path = self.layout.node_path(fenced.session.as_bytes()); let Some((NodeRecord::Tombstone(current), token)) = self.load_record_at(&path).await? else { return Err(Error::Fenced); }; - // A recovery manifest is not terminal bundle closure. Keep the whole - // failed boot fenced until every original binding is materialized and - // checkpointed; lower-level callers cannot bypass the coordinator. - crate::node::bundle::store::ensure_session_drained( - &self.layout, - current.session, - current.bundle, - ) - .await?; + if prepared.session != current.session + || prepared.head != current.bundle + || prepared.path != path + || prepared.store_identity != self.layout.immutable_cache_identity() + { + return Err(Error::Fenced); + } if let Some(log) = ¤t.log && matches!(log.phase(), NodeLogPhase::Sealed | NodeLogPhase::Retired) && log.recovery_manifest() == recovery_manifest @@ -538,6 +584,9 @@ impl NodeDirectory { log: log.clone(), }); } + if current.fenced()? != *fenced { + return Err(Error::Fenced); + } let sealed = (*current).seal(fenced, recovery_manifest, now_ms)?; let log = sealed .log diff --git a/crates/cellule-runtime/src/node/durability/activation.rs b/crates/cellule-runtime/src/node/durability/activation.rs new file mode 100644 index 00000000..8a9e9768 --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/activation.rs @@ -0,0 +1,106 @@ +//! One epoch-owned activation callback, independent of proof waiters. +use std::sync::{Arc, Mutex}; + +use futures_util::FutureExt; +use futures_util::future::{BoxFuture, Shared}; + +use super::NodeLogAuthority; +use crate::node::lease::NodeLeaseGuard; +use crate::node::log::DurabilityGate; +use crate::{Error, Result}; + +type Attempt = std::result::Result<(), Arc>; +type Completion = Shared>>>; + +#[derive(Default)] +pub(super) struct Activation { + state: Mutex, +} + +#[derive(Default)] +struct State { + completion: Option, + joining: bool, +} + +impl Activation { + pub(super) async fn wait( + &self, + epoch: u64, + gate: &DurabilityGate, + authority: &Arc, + lease: &NodeLeaseGuard, + ) -> Result<()> { + let completion = { + let mut state = self + .state + .lock() + .map_err(|_| Error::Node("node activation lock poisoned"))?; + if !state.joining + && state + .completion + .as_ref() + .is_some_and(|completion| matches!(completion.peek(), Some(Ok(Err(_))))) + { + // A completed failed attempt can be retried, but never overlap + // another callback or restart after shutdown begins joining. + state.completion = None; + } + if let Some(completion) = &state.completion { + completion.clone() + } else { + if state.joining { + return Err(Error::RuntimeClosed); + } + let runtime = tokio::runtime::Handle::try_current().map_err(Error::RuntimeStart)?; + let gate = gate.clone(); + let authority = Arc::clone(authority); + let lease = lease.clone(); + // Dropping a proof waiter (including an Object proof winning) + // cannot drop an accepted CAS before the adapter joins its reply + // and updates the version used by later authority mutations. + let worker = runtime.spawn(async move { + lease.check()?; + tokio::select! { + result = authority.activate(epoch) => result?, + () = lease.wait_fenced() => return Err(Error::Fenced), + } + lease.check()?; + gate.activate_fleet() + }); + let completion = async move { + worker + .await + .map(|result| result.map_err(Arc::new)) + .map_err(|error| Arc::new(Error::FollowerWorkerJoin(error))) + } + .boxed() + .shared(); + state.completion = Some(completion.clone()); + completion + } + }; + completion + .await + .map_err(Error::Shared)? + .map_err(Error::Shared) + } + + pub(super) async fn join(&self) -> Result<()> { + let completion = { + let mut state = self + .state + .lock() + .map_err(|_| Error::Node("node activation lock poisoned"))?; + state.joining = true; + state.completion.clone() + }; + if let Some(completion) = completion { + // Callback failure grants no Fleet proof. Shutdown may still close + // through complete Object coverage, which it checks separately. + // Worker failure remains a source error: its callback did not join. + let _attempt = completion.await.map_err(Error::Shared)?; + } + Ok(()) + } +} diff --git a/crates/cellule-runtime/src/node/durability/mod.rs b/crates/cellule-runtime/src/node/durability/mod.rs index 6cde8051..a5d99730 100644 --- a/crates/cellule-runtime/src/node/durability/mod.rs +++ b/crates/cellule-runtime/src/node/durability/mod.rs @@ -15,6 +15,7 @@ use crate::node::log_shipper::{NodeLogShipper, NodeLogSubmission}; use crate::node::log_transport::NodeLogTransport; use crate::{Error, Result}; +mod activation; mod object_coverage; use object_coverage::ObjectCoverage; mod publication; @@ -195,7 +196,7 @@ pub struct NodeDurability { authority: Arc, transport: Arc, node_lease: NodeLeaseGuard, - activated: OnceCell<()>, + activation: activation::Activation, object_coverage: ObjectCoverage, shutdown: Mutex<()>, retirement: std::sync::Mutex>>, @@ -245,7 +246,7 @@ impl NodeDurability { authority, transport, node_lease, - activated: OnceCell::new(), + activation: activation::Activation::default(), object_coverage: ObjectCoverage::default(), shutdown: Mutex::new(()), retirement: std::sync::Mutex::new(None), @@ -367,7 +368,8 @@ impl NodeDurability { .ok_or(Error::PendingPublication)?; // Reserve before cloning locator metadata; the original cohort remains // separately charged until its other original consumers release it. - let memory = resources.try_reserve( + let memory = resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Selection, crate::fleet::resource::ResourceCost::zero() .with_retained_bytes(selected.proof.retained_metadata_bytes()?), )?; @@ -486,18 +488,7 @@ impl NodeDurability { result = self.gate.wait_followers(ticket) => result?, () = self.node_lease.wait_fenced() => return Err(Error::Fenced), } - self.activated - .get_or_try_init(|| async { - self.node_lease.check()?; - tokio::select! { - result = self.authority.activate(ticket.log_epoch()) => result?, - () = self.node_lease.wait_fenced() => return Err(Error::Fenced), - } - self.node_lease.check()?; - self.gate.activate_fleet()?; - Ok::<(), Error>(()) - }) - .await?; + self.activate_fleet(ticket.log_epoch()).await?; let proof = tokio::select! { result = self.gate.prove(ticket) => result?, () = self.node_lease.wait_fenced() => return Err(Error::Fenced), @@ -516,18 +507,7 @@ impl NodeDurability { self.node_lease.check()?; let activate = async { self.gate.wait_followers(ticket).await?; - self.activated - .get_or_try_init(|| async { - self.node_lease.check()?; - tokio::select! { - result = self.authority.activate(ticket.log_epoch()) => result?, - () = self.node_lease.wait_fenced() => return Err(Error::Fenced), - } - self.node_lease.check()?; - self.gate.activate_fleet()?; - Ok::<(), Error>(()) - }) - .await?; + self.activate_fleet(ticket.log_epoch()).await?; Ok::<(), Error>(()) }; tokio::pin!(activate); @@ -543,6 +523,15 @@ impl NodeDurability { Ok(proof) } + async fn activate_fleet(&self, epoch: u64) -> Result<()> { + let activation = self + .activation + .wait(epoch, &self.gate, &self.authority, &self.node_lease) + .await; + self.node_lease.check()?; + activation + } + /// Records an already-published object root and persists its contiguous watermark. /// /// Callers must complete the exact Cell root CAS before invoking this method. @@ -678,12 +667,20 @@ impl NodeDurability { return Ok(proof); } let shipping = self.shipper.shutdown().await; - if let Some(publisher) = self.publisher.get() { + let publication = if let Some(publisher) = self.publisher.get() { // Join both tasks even when fencing stopped the shipper. Preserve // the producer's original cause rather than its secondary fence. - publisher.join().await?; - } + publisher.join().await + } else { + Ok(()) + }; + // Activation may outlive its original waiter. Join it before coverage + // and closure use the authority adapter's acknowledged version, even + // when shipping or publication has already failed. + let activation = self.activation.join().await; + publication?; shipping?; + activation?; self.object_coverage .flush(&self.gate, self.authority.as_ref(), &self.node_lease, &[]) .await?; diff --git a/crates/cellule-runtime/src/node/durability/publication/mod.rs b/crates/cellule-runtime/src/node/durability/publication/mod.rs index fa21ed82..513172e5 100644 --- a/crates/cellule-runtime/src/node/durability/publication/mod.rs +++ b/crates/cellule-runtime/src/node/durability/publication/mod.rs @@ -71,7 +71,8 @@ impl Publisher { .selection_resources .get() .ok_or(Error::PendingPublication)?; - resources.try_reserve( + resources.try_reserve_in( + crate::fleet::resource::RetentionStage::Publisher, crate::fleet::resource::ResourceCost::zero() // The fifth buffer covers the fresh cohort origin read. After // matching the proposal, its allocation is released for 2 MiB @@ -149,10 +150,14 @@ impl Publisher { // Cell departure is allowed only after this original callback joins, // not after enqueue. Otherwise closing/detaching its pin could overtake // a still-running catalog checkpoint from the accepted root task. - tokio::select! { - result = completion => result.map_err(|_| Error::RuntimeClosed)?.map_err(Error::Shared), - () = self.lease.wait_fenced() => self.terminal_error(), + // Admission transferred the obligation to the original producer. + // Fencing cannot abandon that callback or replace its source error + // while another accepted upload still delays the worker's termination. + match completion.await { + Ok(result) => result.map_err(Error::Shared)?, + Err(_) => return self.terminal_error(), } + self.lease.check() } fn terminal_error(&self) -> Result<()> { @@ -221,6 +226,12 @@ async fn run( mut preparation: crate::node::bundle::BundlePreparation, lease: &NodeLeaseGuard, ) -> Result<()> { + let telemetry = durability + .upgrade() + .ok_or(Error::RuntimeClosed)? + .shipper + .telemetry() + .clone(); let mut prefixes = prefixes::Prefixes::new()?; let mut carry = None; let mut checkpoints_open = true; @@ -234,14 +245,14 @@ async fn run( } else if let Some(capture) = feed.try_recv() { Some(capture) } else if let Some(first) = first_checkpoint.take() { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; continue; } else { tokio::select! { capture = feed.recv() => capture, checkpoint = checkpoints.recv(), if checkpoints_open => { if let Some(first) = checkpoint { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } else { checkpoints_open = false; } continue; @@ -251,7 +262,7 @@ async fn run( }; let Some(first) = capture else { if let Some(first) = first_checkpoint.take() { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } break; }; @@ -264,6 +275,7 @@ async fn run( first_checkpoint, true, lease, + &telemetry, ) .await?; let original = durability.upgrade().ok_or(Error::RuntimeClosed)?; @@ -285,7 +297,13 @@ async fn run( }; let cohort = receipts::SelectedCaptures::new(&original, &captures, proofs)?; let memory = { - let admission = cohort.resources(&original)?.reserve(cohort.cost()); + let admission = telemetry.publication_phase( + crate::fleet::telemetry::NodePublicationPhase::ReceiptWait, + cohort.resources(&original)?.reserve_in( + crate::fleet::resource::RetentionStage::Selection, + cohort.cost(), + ), + ); tokio::pin!(admission); loop { tokio::select! { @@ -300,7 +318,7 @@ async fn run( // Root tasks retain their admission until this callback // joins. Servicing it while receipt credit is exhausted // releases credit without repeating the durable CAS. - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } else { checkpoints_open = false; } } } @@ -315,7 +333,7 @@ async fn run( } checkpoints.close(); while let Some(first) = checkpoints.recv().await { - checkpoint_cohort(&authority, &mut checkpoints, first).await?; + checkpoint_cohort(&authority, &mut checkpoints, first, &telemetry).await?; } Ok(()) } @@ -331,10 +349,14 @@ async fn checkpoint_cohort( authority: &Arc, receiver: &mut mpsc::Receiver, first: CheckpointRequest, + telemetry: &crate::fleet::telemetry::CellTelemetryHandle, ) -> Result<()> { let cohort = checkpoints::Cohort::gather(receiver, Some(first), MAX_CAPTURES)?; - let result = authority - .checkpoint(cohort.values()) + let result = telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::Checkpoint, + authority.checkpoint(cohort.values()), + ) .await .map_err(Arc::new); cohort.complete(result.clone()); @@ -359,6 +381,7 @@ async fn assemble( mut first_checkpoint: Option, include_checkpoints: bool, lease: &NodeLeaseGuard, + telemetry: &crate::fleet::telemetry::CellTelemetryHandle, ) -> Result { let mut captures = vec![first]; let mut frames = captures[0].frames().len(); @@ -371,7 +394,7 @@ async fn assemble( if frames == MAX_FRAMES && let Some(first) = first_checkpoint.take() { - checkpoint_cohort(authority, checkpoints, first).await?; + checkpoint_cohort(authority, checkpoints, first, telemetry).await?; } // Original root tasks hold credit until their callback joins. Reserve // the entire ready cohort before native assembly can occupy its rows; @@ -411,6 +434,12 @@ async fn assemble( ready_checkpoints.extend(checkpoints, MAX_FRAMES - frames)?; } + telemetry.node_publication_cohort(crate::fleet::telemetry::NodePublicationCohort { + captures: captures.len(), + frames, + bytes, + checkpoints: ready_checkpoints.notification_count(), + }); Ok(CaptureCohort { captures, ready_checkpoints, diff --git a/crates/cellule-runtime/src/node/durability/publication/pipeline.rs b/crates/cellule-runtime/src/node/durability/publication/pipeline.rs index 881a8257..02f0a48b 100644 --- a/crates/cellule-runtime/src/node/durability/publication/pipeline.rs +++ b/crates/cellule-runtime/src/node/durability/publication/pipeline.rs @@ -1,6 +1,9 @@ //! One lazy successor: pre-admitted overlap, ordered proof, joined I/O. use super::*; use crate::fleet::resource::{ResourceCost, ResourceReservation}; +use crate::fleet::telemetry::{ + NodePublicationPhase as Phase, PublicationPipelineFallback as Fallback, +}; use crate::node::bundle::PreparedNodeBundle; use std::sync::OnceLock; @@ -23,18 +26,20 @@ fn reserve( original: &NodeDurability, authority: &dyn NodeBundlePublicationAuthority, captures: usize, + staged: &crate::node::bundle::StagedNodeBundle, ) -> Result { let first = authority.receipt_memory_bound(captures)?; let second = authority.receipt_memory_bound(MAX_CAPTURES)?; + let working = staged.retained_work_bytes()?; let bytes = first .checked_add(second) - .and_then(|bytes| bytes.checked_add((5 * crate::node::bundle::MAX_BUNDLE_BYTES) as usize)) + .and_then(|bytes| bytes.checked_add(working)) .ok_or(Error::Capacity("bundle pipeline memory"))?; let reserved = original .selection_resources .get() .ok_or(Error::PendingPublication)? - .try_reserve(ResourceCost::zero().with_retained_bytes(bytes))?; + .try_reserve_pipeline(ResourceCost::zero().with_retained_bytes(bytes))?; Ok(Memory { first, second, @@ -72,10 +77,13 @@ pub(super) async fn publish( progress: &watch::Sender, lease: &NodeLeaseGuard, ) -> Result> { + let telemetry = original.shipper.telemetry(); let first_capture_count = first.captures.len(); - let may_overlap = first.ready_checkpoints.values().is_empty(); let failure = OnceLock::new(); - let round = match authority.begin_round().await { + let round = match telemetry + .publication_phase(Phase::CatalogWait, authority.begin_round()) + .await + { Ok(round) => round, Err(error) => { let error = Arc::new(error); @@ -83,12 +91,15 @@ pub(super) async fn publish( return Err(Error::Shared(error)); } }; - let staged = match round - .stage( - None, - &first.captures, - first.ready_checkpoints.values(), - Some(preparation), + let staged = match telemetry + .publication_phase( + Phase::Preparation, + round.stage( + None, + &first.captures, + first.ready_checkpoints.values(), + Some(preparation), + ), ) .await { @@ -101,17 +112,31 @@ pub(super) async fn publish( }; let (finished, first_finished) = oneshot::channel(); let (receipt, first_receipt) = oneshot::channel(); + // Preserve the original five-buffer working bound by serializing heavy + // preparation and verification. Uploads still overlap. Extra admission + // owns the first compact proposal while the successor uses that scratch. + // During first verification, its three scratch buffers and the at-most + // two-buffer successor fit the original five; the first proposal remains + // covered by this exact additional reservation throughout both cohorts. + let scratch = tokio::sync::Mutex::new(()); let (first_result, successor) = tokio::join!( async { let result = async { let publication = async { - let uploaded = round.upload(Arc::clone(&staged)).await?; + let uploaded = telemetry + .publication_phase(Phase::Upload, round.upload(Arc::clone(&staged))) + .await?; let prior = prefixes.for_captures(&first.captures); let borrowed = prior .iter() .map(|selected| &selected.proof) .collect::>(); - round.select(&uploaded, &borrowed, lease).await + telemetry + .publication_phase(Phase::Selection, async { + let _scratch = scratch.lock().await; + round.select(&uploaded, &borrowed, lease).await + }) + .await } .await; let publication = recorded(publication, &failure, lease).map_err(Arc::new); @@ -146,10 +171,6 @@ pub(super) async fn publish( }, async { let result = async { - if !may_overlap { - let _ = receipt.send(None); - return Ok(None); - } let next = if let Some(next) = carry.take().or_else(|| feed.try_recv()) { Some(next) } else { @@ -161,25 +182,37 @@ pub(super) async fn publish( } }; let Some(next) = next else { + telemetry.node_publication_fallback(Fallback::NoSuccessor); let _ = receipt.send(None); return Ok(None); }; - let mut memory = match reserve(original, authority.as_ref(), first_capture_count) { - Ok(memory) => memory, - Err(Error::Capacity(_)) => { - *carry = Some(next); - let _ = receipt.send(None); - return Ok(None); - } - Err(error) => return Err(error), - }; + let mut memory = + match reserve(original, authority.as_ref(), first_capture_count, &staged) { + Ok(memory) => memory, + Err(Error::Capacity("bundle pipeline headroom")) => { + telemetry.node_publication_fallback(Fallback::Headroom); + *carry = Some(next); + let _ = receipt.send(None); + return Ok(None); + } + Err(Error::Capacity(_)) => { + telemetry.node_publication_fallback(Fallback::Capacity); + *carry = Some(next); + let _ = receipt.send(None); + return Ok(None); + } + Err(error) => return Err(error), + }; // This is a nonblocking admission under ordering. It either // prepaid both receipt lifetimes or returns to the serial path. let first_memory = memory.reserved.split_retained(memory.first)?; let second_memory = memory.reserved.split_retained(memory.second)?; let _ = receipt.send(Some(first_memory)); - // No standalone checkpoint is dispatched while this round owns - // ordering; the bounded gather includes only available row slots. + // The staged predecessor already carries its verified new + // bases. A successor inherits them and may compose further + // checkpoints in its available rows; neither callback gains + // credit until that cohort's ordered authoritative selection. + // No standalone checkpoint is dispatched under round ordering. let second = assemble( authority, feed, @@ -187,21 +220,29 @@ pub(super) async fn publish( checkpoints, next, None, - false, + true, lease, + telemetry, ) .await?; let upload = async { - let next = round - .stage( - Some(&staged), - &second.captures, - second.ready_checkpoints.values(), - Some(preparation), - ) + let next = telemetry + .publication_phase(Phase::Preparation, async { + let _scratch = scratch.lock().await; + round + .stage( + Some(&staged), + &second.captures, + second.ready_checkpoints.values(), + Some(preparation), + ) + .await + }) .await?; lease.check()?; - round.upload(next).await + telemetry + .publication_phase(Phase::Upload, round.upload(next)) + .await } .await; match recorded(upload, &failure, lease) { @@ -245,8 +286,11 @@ pub(super) async fn publish( .iter() .map(|selected| &selected.proof) .collect::>(); - let result = round - .select(&successor.uploaded, &borrowed, lease) + let result = telemetry + .publication_phase( + Phase::Selection, + round.select(&successor.uploaded, &borrowed, lease), + ) .await .map_err(Arc::new); drop(borrowed); diff --git a/crates/cellule-runtime/src/node/durability/receipts.rs b/crates/cellule-runtime/src/node/durability/receipts.rs index 09d9bc83..9218309c 100644 --- a/crates/cellule-runtime/src/node/durability/receipts.rs +++ b/crates/cellule-runtime/src/node/durability/receipts.rs @@ -84,6 +84,7 @@ impl<'a> SelectedCaptures<'a> { // Transfer a single cohort admission before waking any sibling. Waiting // for credit may have joined older checkpoints, so revalidate the // original lease/gate against these exact live proofs after the wait. + memory.set_stage(crate::fleet::resource::RetentionStage::Selection)?; let memories = self .bytes .into_iter() diff --git a/crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs b/crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs new file mode 100644 index 00000000..c9453696 --- /dev/null +++ b/crates/cellule-runtime/src/node/durability/tests/activation_lifetime.rs @@ -0,0 +1,213 @@ +use super::*; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +#[derive(Default)] +struct CommittedActivation { + state: tokio::sync::Mutex<()>, + remote: AtomicBool, + local: AtomicBool, + calls: AtomicUsize, + committed: Notify, + resume: Notify, + joined: Notify, +} + +impl NodeLogAuthority for CommittedActivation { + fn activate<'a>(&'a self, _epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let _state = self.state.lock().await; + self.calls.fetch_add(1, Ordering::AcqRel); + // The provider has accepted the CAS, but its response and the + // adapter's acknowledged version have not yet joined. + self.remote.store(true, Ordering::Release); + self.committed.notify_one(); + self.resume.notified().await; + self.local.store(true, Ordering::Release); + self.joined.notify_one(); + Ok(()) + }) + } + + fn advance_coverage<'a>(&'a self, _epoch: u64, _through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let _state = self.state.lock().await; + if self.remote.load(Ordering::Acquire) != self.local.load(Ordering::Acquire) { + return Err(Error::Node( + "coverage CAS used a cancelled activation version", + )); + } + Ok(()) + }) + } + + fn close<'a>( + &'a self, + _retirement: &'a crate::node::log::NodeLogRetirementObservation, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn cancelled_proof_waiter_keeps_accepted_activation_joined() { + for fleet_only in [true, false] { + cancelled_waiter(fleet_only).await; + } +} + +fn runtime(authority: Arc) -> Arc { + let gate = DurabilityGate::new(session(1), node(1), 2, [node(2)]).unwrap(); + let transport: Arc = Arc::new(ImmediateTransport); + let shipper = NodeLogShipper::new( + gate.clone(), + Arc::clone(&transport), + cellule_ltx::Limits::default(), + ) + .unwrap(); + Arc::new(NodeDurability::new( + gate, + shipper, + authority, + transport, + lease(), + )) +} + +async fn cancelled_waiter(fleet_only: bool) { + let authority = Arc::new(CommittedActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + let waiter = tokio::spawn({ + let durability = durability.clone(); + async move { + if fleet_only { + durability.prove_fleet(ticket).await + } else { + durability.prove(ticket).await + } + } + }); + tokio::time::timeout( + std::time::Duration::from_secs(1), + authority.committed.notified(), + ) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + authority.resume.notify_one(); + tokio::time::timeout( + std::time::Duration::from_millis(250), + authority.joined.notified(), + ) + .await + .expect("accepted authority I/O was cancelled with its proof waiter"); + assert!(authority.local.load(Ordering::Acquire)); + assert_eq!( + durability.prove_fleet(ticket).await.unwrap().source(), + DurabilitySource::Fleet + ); + assert_eq!(authority.calls.load(Ordering::Acquire), 1); + durability.prove_object(ticket).await.unwrap(); + durability.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn shutdown_joins_cancelled_waiters_activation_before_coverage() { + let authority = Arc::new(CommittedActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + let waiter = tokio::spawn({ + let durability = durability.clone(); + async move { durability.prove(ticket).await } + }); + tokio::time::timeout( + std::time::Duration::from_secs(1), + authority.committed.notified(), + ) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let mut shutdown = tokio::spawn({ + let durability = durability.clone(); + async move { durability.shutdown().await } + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), &mut shutdown) + .await + .is_err() + ); + authority.resume.notify_one(); + // Joining the callback does not invent bucket coverage for the Fleet tail. + assert!(matches!( + shutdown.await.unwrap(), + Err(Error::PendingPublication) + )); + assert!(authority.local.load(Ordering::Acquire)); + durability.prove_object(ticket).await.unwrap(); + durability.shutdown().await.unwrap(); + assert_eq!(authority.calls.load(Ordering::Acquire), 1); +} + +#[derive(Default)] +struct RetryActivation(AtomicUsize); + +impl NodeLogAuthority for RetryActivation { + fn activate<'a>(&'a self, _epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async { + if self.0.fetch_add(1, Ordering::AcqRel) == 0 { + Err(Error::Node("injected activation failure")) + } else { + Ok(()) + } + }) + } + + fn advance_coverage<'a>(&'a self, _epoch: u64, _through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } + + fn close<'a>( + &'a self, + _retirement: &'a crate::node::log::NodeLogRetirementObservation, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async { Ok(()) }) + } +} + +#[tokio::test] +async fn completed_activation_failure_can_retry_without_reissuing_the_ticket() { + let authority = Arc::new(RetryActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + let error = durability.prove_fleet(ticket).await.unwrap_err(); + assert!( + matches!(error, Error::Shared(source) if matches!(source.as_ref(), Error::Node("injected activation failure"))) + ); + assert_eq!( + durability.prove_fleet(ticket).await.unwrap().source(), + DurabilitySource::Fleet + ); + assert_eq!(authority.0.load(Ordering::Acquire), 2); + durability.prove_object(ticket).await.unwrap(); + durability.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn failed_activation_can_drain_through_complete_object_coverage() { + let authority = Arc::new(RetryActivation::default()); + let durability = runtime(authority.clone()); + let (_directory, cuts) = capture(); + let ticket = durability.submit(submission(&cuts)).await.unwrap(); + durability.prove_fleet(ticket).await.unwrap_err(); + assert_eq!( + durability.prove_object(ticket).await.unwrap().source(), + DurabilitySource::Object + ); + durability.shutdown().await.unwrap(); + assert_eq!(authority.0.load(Ordering::Acquire), 1); +} diff --git a/crates/cellule-runtime/src/node/durability/tests/mod.rs b/crates/cellule-runtime/src/node/durability/tests/mod.rs index bfec28d1..fca3a397 100644 --- a/crates/cellule-runtime/src/node/durability/tests/mod.rs +++ b/crates/cellule-runtime/src/node/durability/tests/mod.rs @@ -11,6 +11,7 @@ use crate::node::log_transport::{ AppendRequest, NodeLogTransport, RetireRequest, SealRequest, TailRequest, }; +mod activation_lifetime; mod closure; #[derive(Default)] diff --git a/crates/cellule-runtime/src/node/log_recovery/claim.rs b/crates/cellule-runtime/src/node/log_recovery/claim.rs new file mode 100644 index 00000000..9f112ff3 --- /dev/null +++ b/crates/cellule-runtime/src/node/log_recovery/claim.rs @@ -0,0 +1,126 @@ +//! One cancellation-scoped renewal loop for a managed recovery attempt. +use super::*; +use crate::node::directory::{NodeRecord, NodeTombstone}; +use tokio::sync::{Mutex, MutexGuard}; + +pub(crate) struct RecoveryClaimLease<'a> { + directory: &'a NodeDirectory, + current: Mutex, + now_ms: i64, + started: std::time::Instant, +} + +impl<'a> RecoveryClaimLease<'a> { + pub(crate) async fn new( + directory: &'a NodeDirectory, + fenced: FencedNodeSession, + now_ms: i64, + ) -> Result { + let lease = Self { + directory, + current: Mutex::new(fenced), + now_ms, + started: std::time::Instant::now(), + }; + lease.verify().await?; + Ok(lease) + } + + pub(crate) fn now(&self) -> Result { + crate::node::bundle::recovery::logical_now(self.now_ms, self.started) + } + + pub(in crate::node) async fn lock_verified( + &self, + ) -> Result<( + MutexGuard<'_, FencedNodeSession>, + NodeTombstone, + cellule_store::ETag, + )> { + let current = self.current.lock().await; + let (record, token) = self.check(¤t).await?; + Ok((current, record, token)) + } + + pub(crate) async fn verify(&self) -> Result { + let (current, _, _) = self.lock_verified().await?; + Ok(current.clone()) + } + + async fn check( + &self, + fenced: &FencedNodeSession, + ) -> Result<(NodeTombstone, cellule_store::ETag)> { + self.directory + .load(fenced.claimant(), self.now()?) + .await? + .ok_or(Error::Fenced)?; + let path = self.directory.layout.node_path(fenced.session().as_bytes()); + let Some((NodeRecord::Tombstone(current), token)) = + self.directory.load_record_at(&path).await? + else { + return Err(Error::Fenced); + }; + if current.fenced()? != *fenced || self.now()? >= fenced.claim_expires_at_ms() { + return Err(Error::Fenced); + } + Ok((*current, token)) + } + + async fn keep_alive(&self) -> Result<()> { + loop { + let expiry = self.current.lock().await.claim_expires_at_ms(); + let lifetime = crate::node::log_state::RECOVERY_CLAIM_LIFETIME_MS; + let delay = expiry + .saturating_sub(self.now()?) + .saturating_sub(lifetime * 2 / 3); + tokio::time::sleep(std::time::Duration::from_millis( + u64::try_from(delay.max(1)) + .map_err(|_| Error::Capacity("recovery renewal delay"))?, + )) + .await; + // Renewal and terminal node CAS share this lock. Immutable data + // verification and Cell materialization never hold it. + let (mut current, _, _) = self.lock_verified().await?; + *current = self + .directory + .refresh_recovery_claim(¤t, self.now()?) + .await?; + self.check(¤t).await?; + } + } + + pub(crate) async fn run( + &self, + work: impl std::future::Future>, + ) -> Result { + // No detached heartbeat survives success, failure or cancellation. + tokio::select! { + biased; + result = work => result, + result = self.keep_alive() => { + result?; + Err(Error::Node("recovery renewal loop stopped")) + } + } + } + + pub(crate) async fn seal(&self, manifest: Option) -> Result { + let snapshot = self.verify().await?; + let prepared = self.directory.prepare_recovery_seal(&snapshot).await?; + // The immutable terminal inventory was checked outside the renewal + // lock. Its exact head is checked again by the final authority CAS. + let (current, _, _) = self.lock_verified().await?; + let sealed = self + .directory + .seal_recovery_prepared(¤t, manifest, self.now()?, &prepared) + .await?; + let takeover = NodeTakeoverProof::after_recovery(¤t, &sealed)?; + Ok(CompletedNodeRecoverySeal { sealed, takeover }) + } +} + +pub(crate) struct CompletedNodeRecoverySeal { + pub(crate) sealed: SealedNodeLog, + pub(crate) takeover: NodeTakeoverProof, +} diff --git a/crates/cellule-runtime/src/node/log_recovery/completed.rs b/crates/cellule-runtime/src/node/log_recovery/completed.rs new file mode 100644 index 00000000..4aed55b4 --- /dev/null +++ b/crates/cellule-runtime/src/node/log_recovery/completed.rs @@ -0,0 +1,92 @@ +//! Read-only adoption of an already committed recovery result. +use super::*; + +impl RecoveryCoordinator { + pub(super) async fn resume_completed( + &self, + directory: &NodeDirectory, + fenced: &FencedNodeSession, + cells: &[RecoveryCell], + now_ms: i64, + ) -> Result> { + let started = std::time::Instant::now(); + let recovered = match directory + .recovered_session(fenced.node(), fenced.session(), fenced.claimant(), now_ms) + .await + { + Ok(recovered) => recovered, + Err(Error::Control("node session log recovery is incomplete")) => return Ok(None), + Err(source) => return Err(source), + }; + let sealed = SealedNodeLog { + session: fenced.session(), + log: recovered.log().cloned().ok_or(Error::Fenced)?, + }; + let takeover = NodeTakeoverProof::after_recovery(fenced, &sealed)?; + let mut controls = Vec::new(); + for cell in cells { + let current = cell + .authority + .load(cell.observed.value().cell) + .await? + .ok_or(Error::Fenced)?; + if current.value() != cell.observed.value() + || *cell.application.as_bytes() == [0; 16] + || current.value().ltx_root().is_none() + || current.value().owner.as_ref().map(|owner| owner.session) + != Some(fenced.session()) + { + return Err(Error::Fenced); + } + if current.value().recovery.is_some() { + controls.push(current); + } + } + let manifest = pinned_manifest(&controls, fenced, self.recovery.log_epoch)?; + if sealed.log().recovery_manifest() != manifest { + return Err(Error::Fenced); + } + if let Some(manifest) = manifest { + let inventory = self + .manifests + .load_manifest(fenced.session(), self.recovery.log_epoch, manifest) + .await?; + if inventory.cells().len() != controls.len() { + return Err(Error::Fenced); + } + for pin in inventory.cells() { + cells + .iter() + .find(|cell| { + cell.application == pin.application + && cell.observed.value().cell == pin.cell + && cell.observed.value().incarnation == pin.incarnation + && cell.observed.value().epoch == pin.cell_epoch + && cell.observed.value().recovery.as_ref() == Some(&pin.recovery) + }) + .ok_or(Error::Fenced)?; + // A terminal pointer does not prove its immutable payload is + // still available. Reopen the exact original overlay. + self.manifests + .for_application(pin.application) + .load_overlay(pin.cell, pin.incarnation, &pin.recovery) + .await?; + } + } + let now = crate::node::bundle::recovery::logical_now(now_ms, started)?; + let current = directory + .recovered_session(fenced.node(), fenced.session(), fenced.claimant(), now) + .await?; + if current != recovered { + return Err(Error::Fenced); + } + // Sealing clears the transient recovery claim. A repeated successful + // call adopts only the exact terminal result; it renews no old claim + // and performs no Cell, manifest or node authority writes. + Ok(Some(CompletedNodeRecovery { + sealed, + controls, + takeover, + })) + } +} diff --git a/crates/cellule-runtime/src/node/log_recovery/mod.rs b/crates/cellule-runtime/src/node/log_recovery/mod.rs index e2046937..a40da591 100644 --- a/crates/cellule-runtime/src/node/log_recovery/mod.rs +++ b/crates/cellule-runtime/src/node/log_recovery/mod.rs @@ -25,12 +25,15 @@ const MAX_RECOVERY_CATALOG_HEAD_READS: usize = 32; const MAX_RECOVERY_PAGE_BYTES: u64 = 1 << 20; const MAX_RECOVERY_PAGE_FRAMES: usize = 4_096; +mod claim; +mod completed; pub mod retirement; mod tail; mod witness; use tail::*; +pub(crate) use claim::RecoveryClaimLease; use witness::*; /// Bounded work counters emitted for one node-log recovery attempt. @@ -537,7 +540,10 @@ impl RecoveryCoordinator { cells: Vec, sealed: SealedSession, ) -> Result { - Ok(self.recover_attempt(fenced, cells, sealed).await?.result) + Ok(self + .recover_attempt(fenced, cells, sealed, None) + .await? + .result) } async fn recover_attempt( @@ -545,6 +551,7 @@ impl RecoveryCoordinator { fenced: FencedNodeSession, cells: Vec, sealed: SealedSession, + claim: Option<&RecoveryClaimLease<'_>>, ) -> Result { self.recovery.validate_fence(&fenced)?; let mut bases = Vec::with_capacity(cells.len()); @@ -583,6 +590,7 @@ impl RecoveryCoordinator { &cells, sealed.durable_through, self.recovery.limits, + claim, ) .await?; if let Some(drain) = &drain { @@ -741,27 +749,38 @@ impl RecoveryCoordinator { cells: Vec, now_ms: i64, ) -> Result { - let started = std::time::Instant::now(); self.recovery.validate_fence(&fenced)?; - let witness = self.recovery.ensure_sealed_bounded().await?; - let attempt = self.recover_attempt(fenced.clone(), cells, witness).await?; - let Some(drain) = attempt.drain else { - return self - .finish(directory, fenced, attempt.result.controls, now_ms) - .await; - }; - let manifest = pinned_manifest(&attempt.result.controls, &fenced, self.recovery.log_epoch)?; - let (fresh, controls) = drain - .complete(directory, &self.manifests, &fenced, now_ms, started) - .await?; - let now = crate::node::bundle::recovery::logical_now(now_ms, started)?; - let sealed = directory.seal_recovery(&fresh, manifest, now).await?; - let takeover = NodeTakeoverProof::after_recovery(&fenced, &sealed)?; - Ok(CompletedNodeRecovery { - sealed, - controls, - takeover, - }) + if let Some(completed) = self + .resume_completed(directory, &fenced, &cells, now_ms) + .await? + { + return Ok(completed); + } + let claim = RecoveryClaimLease::new(directory, fenced.clone(), now_ms).await?; + claim + .run(async { + let witness = self.recovery.ensure_sealed_bounded().await?; + let attempt = self + .recover_attempt(fenced.clone(), cells, witness, Some(&claim)) + .await?; + let manifest = + pinned_manifest(&attempt.result.controls, &fenced, self.recovery.log_epoch)?; + let controls = match attempt.drain { + Some(drain) => { + drain + .complete(directory, &self.manifests, &fenced, &claim) + .await? + } + None => attempt.result.controls, + }; + let completed = claim.seal(manifest).await?; + Ok(CompletedNodeRecovery { + sealed: completed.sealed, + controls, + takeover: completed.takeover, + }) + }) + .await } /// Seals one still-current claim after all recovered controls were pinned. diff --git a/crates/cellule-runtime/src/node/log_shipper/mod.rs b/crates/cellule-runtime/src/node/log_shipper/mod.rs index eabafc02..348a6369 100644 --- a/crates/cellule-runtime/src/node/log_shipper/mod.rs +++ b/crates/cellule-runtime/src/node/log_shipper/mod.rs @@ -213,6 +213,10 @@ pub struct NodeLogShipper { } impl NodeLogShipper { + pub(crate) fn telemetry(&self) -> &crate::fleet::telemetry::CellTelemetryHandle { + &self.telemetry + } + /// Starts one shipper for the exact ensemble owned by `gate`. pub fn new( gate: DurabilityGate, diff --git a/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs b/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs index 7433265a..04af7cfb 100644 --- a/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs +++ b/crates/cellule-runtime/src/node/log_shipper/replication/mod.rs @@ -211,6 +211,20 @@ async fn run_member( ) .await .map_err(Arc::new); + if let Err(error) = &result { + tracing::warn!( + leader_session = ?leader, + log_epoch = epoch, + member = ?member, + first_sequence = requests.first().map(|request| request.first), + last_sequence = last, + covered_through = covered, + frame_count, + bytes, + error = ?error, + "follower group append failed" + ); + } // One canonical follower append covers the delivered group and its // single fsync. Every original request receives the same verified watermark. for request in requests { diff --git a/crates/cellule-runtime/src/node/tests/candidates.rs b/crates/cellule-runtime/src/node/tests/candidates.rs index 4b786df8..ccc97e0e 100644 --- a/crates/cellule-runtime/src/node/tests/candidates.rs +++ b/crates/cellule-runtime/src/node/tests/candidates.rs @@ -249,6 +249,12 @@ async fn expired_enrolled_log_becomes_a_renewable_recovery_claim() { .unwrap(); assert_eq!(renewed.claim_generation(), fenced.claim_generation()); assert!(renewed.claim_expires_at_ms() > fenced.claim_expires_at_ms()); + assert!(matches!( + directory + .refresh_recovery_claim(&fenced, NOW_MS + 15_001) + .await, + Err(Error::Fenced) + )); let sealed = directory .seal_recovery(&renewed, None, NOW_MS + 15_001) .await diff --git a/crates/cellule-runtime/src/publication/mod.rs b/crates/cellule-runtime/src/publication/mod.rs index ff412d0b..d8a7611f 100644 --- a/crates/cellule-runtime/src/publication/mod.rs +++ b/crates/cellule-runtime/src/publication/mod.rs @@ -117,27 +117,38 @@ impl CellPublisher { proof: &crate::node::bundle::BundleCoverageProof, next_due_ms: Option, ) -> Result { - self.check_node_lease()?; - if self.observed.value().bundle_binding != Some(proof.binding()) { - return Err(Error::Fenced); - } - let base = self.observed.value().ltx_root().ok_or(Error::Fenced)?; - if base.commit_sequence == proof.commit_sequence() && base.position == proof.position() { - return Ok(base); - } - let overlay = proof - .recovery_overlay_from(self.authority.layout(), self.replica.limits(), base) - .await?; - self.check_node_lease()?; - let (replica, confirmation) = lineage::replica(self.replica.clone(), &self.authority); - let prepared = replica - .prepare_recovered_overlay(&overlay, self.observed.value().schema) + let telemetry = self.telemetry.clone(); + telemetry + .publication_phase( + crate::fleet::telemetry::NodePublicationPhase::Materialization, + async { + self.check_node_lease()?; + if self.observed.value().bundle_binding != Some(proof.binding()) { + return Err(Error::Fenced); + } + let base = self.observed.value().ltx_root().ok_or(Error::Fenced)?; + if base.commit_sequence == proof.commit_sequence() + && base.position == proof.position() + { + return Ok(base); + } + let overlay = proof + .recovery_overlay_from(self.authority.layout(), self.replica.limits(), base) + .await?; + self.check_node_lease()?; + let (replica, confirmation) = + lineage::replica(self.replica.clone(), &self.authority); + let prepared = replica + .prepare_recovered_overlay(&overlay, self.observed.value().schema) + .await + .map_err(lineage::error)?; + self.lineage_confirmed = *confirmation + .lock() + .map_err(|_| Error::Peer("root lineage confirmation lock poisoned"))?; + self.publish_prepared(&prepared, next_due_ms).await + }, + ) .await - .map_err(lineage::error)?; - self.lineage_confirmed = *confirmation - .lock() - .map_err(|_| Error::Peer("root lineage confirmation lock poisoned"))?; - self.publish_prepared(&prepared, next_due_ms).await } pub(crate) fn with_shared_publication( diff --git a/crates/cellule-runtime/tests/runtime/lifecycle.rs b/crates/cellule-runtime/tests/runtime/lifecycle.rs index 3c65cd21..3e6999a2 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle.rs @@ -61,7 +61,7 @@ pub mod residency; #[derive(Debug)] pub(super) struct PausingStore { - inner: Arc, + inner: Arc, armed: AtomicBool, update_armed: AtomicBool, failing: AtomicBool, @@ -87,7 +87,7 @@ pub(super) struct PausingStore { } impl PausingStore { - pub(super) fn new(inner: Arc) -> Self { + pub(super) fn new(inner: Arc) -> Self { Self { inner, armed: AtomicBool::new(false), diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs index 7acc881a..a3d59675 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership.rs @@ -2,7 +2,9 @@ use super::*; +mod follower_process; mod lease; +mod partition_proxy; mod prefix; mod recovery; mod sparse_process; diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs new file mode 100644 index 00000000..1488445e --- /dev/null +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs @@ -0,0 +1,682 @@ +//! Device-backed process loss of a two-follower ACK before root publication. +//! Native followers share the killed process; this is storage/recovery +//! qualification, not authentication or remote-peer transport qualification. +use super::*; +use cellule_runtime::node::log_transport::LocalFollowerTransport; +use cellule_runtime::node::{NodeAdvertisement, NodeCapacity, NodeDirectory, NodeFailureDomain}; +use std::path::{Path as FsPath, PathBuf}; +use std::process::{Child, Command, Stdio}; + +const CHILD: &str = "runtime::lifecycle::ownership::follower_process::rustfs_follower_tail_child"; +const CHILD_ROOT: &str = "CELLULE_FOLLOWER_TAIL_CHILD_ROOT"; +const PARTITION: &str = "CELLULE_FOLLOWER_TAIL_PARTITION"; +const LEADER: SessionId = SessionId::from_bytes([151; 16]); +const SUCCESSOR: SessionId = SessionId::from_bytes([154; 16]); +const MEMBERS: [NodeId; 2] = [NodeId::from_bytes([152; 16]), NodeId::from_bytes([153; 16])]; + +struct KillOnDrop(Child); +impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn fixture( + store: Store, + partition: bool, +) -> (CellTarget, CellStorageLayout, CellReplica, CellAuthority) { + let target = CellTarget::new( + TenantId::from_bytes([61; 16]), + ApplicationId::from_bytes([62; 16]), + NamespaceId::from_bytes([63; 16]), + b"follower-process-tail", + ) + .unwrap(); + let prefix = std::env::var("CELLULE_TEST_PREFIX").unwrap(); + let layout = CellStorageLayout::new( + store, + Path::from(format!( + "{prefix}/follower-process-tail/{}", + if partition { "partition" } else { "kill" } + )), + [62; 16], + ); + let replica = CellReplica::new( + layout.clone(), + *target.cell_id().as_bytes(), + [64; 16], + Limits::default(), + ) + .unwrap(); + let authority = CellAuthority::new(layout.clone()); + (target, layout, replica, authority) +} + +fn directory(layout: &CellStorageLayout) -> NodeDirectory { + NodeDirectory::new( + layout.clone(), + Digest::from_bytes([90; 32]), + Digest::from_bytes([91; 32]), + Digest::from_bytes([92; 32]), + ) +} + +async fn enroll(directory: &NodeDirectory) { + let mut leader = None; + for session in [ + LEADER, + SessionId::from_bytes([152; 16]), + SessionId::from_bytes([153; 16]), + SUCCESSOR, + ] { + let expires = if session == LEADER { 10_001 } else { 20_000 }; + let ad = NodeAdvertisement::sign( + NodeId::from_bytes(*session.as_bytes()), + session, + "https://process-fixture.internal:8081".into(), + Digest::from_bytes([90; 32]), + Digest::from_bytes([94; 32]), + Digest::from_bytes([91; 32]), + Digest::from_bytes([92; 32]), + &ed25519_dalek::SigningKey::from_bytes( + session.as_bytes().repeat(2).as_slice().try_into().unwrap(), + ), + 1, + 1, + expires, + vec![Digest::from_bytes([95; 32])], + vec![1], + NodeFailureDomain::default(), + NodeCapacity { + free_memory_bytes: 1 << 20, + free_disk_bytes: 1 << 30, + follower_free_bytes: if MEMBERS.contains(&NodeId::from_bytes(*session.as_bytes())) { + 1 << 30 + } else { + 0 + }, + follower_retained_bytes: 0, + job_credits: 1, + log_protocol: cellule_runtime::node::NODE_LOG_PROTOCOL_VERSION, + }, + ) + .unwrap(); + let observed = directory.create(ad, 1).await.unwrap(); + if session == LEADER { + leader = Some(observed); + } + } + let enrolled = directory + .recruit_log(&leader.unwrap(), 1, 2, 4, 2) + .await + .unwrap(); + assert_eq!(enrolled.advertisement().log().unwrap().members(), MEMBERS); + directory.activate_log(&enrolled, 3).await.unwrap(); +} + +struct NativeAuthority(NodeDirectory); +impl NodeLogAuthority for NativeAuthority { + fn activate<'a>( + &'a self, + epoch: u64, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result<()>> { + Box::pin(async move { + let current = self + .0 + .load_if_live(LEADER, 4) + .await? + .ok_or(cellule_runtime::Error::Fenced)?; + let log = current + .advertisement() + .log() + .ok_or(cellule_runtime::Error::Fenced)?; + assert_eq!(log.epoch(), epoch); + assert!(log.active()); + assert_eq!(log.members(), MEMBERS); + Ok(()) + }) + } + fn advance_coverage<'a>( + &'a self, + _epoch: u64, + _through: u64, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result<()>> { + // This cut must retain an unpublished tail. Any credit here fails the test. + Box::pin(async { + Err(cellule_runtime::Error::Node( + "unexpected bucket credit before killed owner", + )) + }) + } + fn close<'a>( + &'a self, + _retirement: &'a cellule_runtime::node::log::NodeLogRetirementObservation, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result<()>> { + Box::pin(async { + Err(cellule_runtime::Error::Node( + "killed owner cannot close its log", + )) + }) + } +} + +struct NativeTransport(Vec<(NodeId, LocalFollowerTransport)>); +impl NativeTransport { + fn open(root: &FsPath) -> Self { + Self( + MEMBERS + .into_iter() + .enumerate() + .map(|(index, member)| { + let store = cellule_runtime::FollowerStore::open( + root.join(format!("follower-{index}")), + Limits::default(), + DiskBudget::new(1 << 30), + ) + .unwrap(); + (member, LocalFollowerTransport::new(member, store)) + }) + .collect(), + ) + } + fn peer(&self, member: NodeId) -> &LocalFollowerTransport { + &self.0.iter().find(|(id, _)| *id == member).unwrap().1 + } +} +impl NodeLogTransport for NativeTransport { + fn append<'a>( + &'a self, + member: NodeId, + request: AppendRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result> { + self.peer(member).append(member, request) + } + fn seal<'a>( + &'a self, + member: NodeId, + request: SealRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result> { + self.peer(member).seal(member, request) + } + fn retire<'a>( + &'a self, + member: NodeId, + request: RetireRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result> { + self.peer(member).retire(member, request) + } + fn tail<'a>( + &'a self, + member: NodeId, + request: TailRequest, + ) -> futures_util::future::BoxFuture<'a, cellule_runtime::Result>> { + self.peer(member).tail(member, request) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rustfs_follower_tail_child() { + let Ok(root) = std::env::var(CHILD_ROOT) else { + return; + }; + let root = PathBuf::from(root); + let partition = std::env::var_os(PARTITION).is_some(); + let remote = sparse_process::rustfs_store(); + let pausing = Arc::new(PausingStore::new(remote.inner().clone())); + let (target, layout, replica, authority) = fixture(Store::new(pausing.clone()), partition); + let catalog = cellule_runtime::cell::catalog::CellCatalog::new(layout.clone(), target.tenant()); + let proof = catalog.lookup(target.cell_id()).await.unwrap().unwrap(); + let observed = authority.load(target.cell_id()).await.unwrap().unwrap(); + let runtime = CellRuntime::new_with_replica_host_requiring_node_lease( + SqlWorkerPool::new(1, 10).unwrap(), + 64 << 20, + LEADER, + ReplicaHost::default(), + ) + .unwrap(); + let lease = NodeLeaseGuard::new(0, 60_000).unwrap(); + runtime.install_node_lease(lease.clone()).unwrap(); + let handle = runtime + .acquire_idle_restored( + proof, + replica, + authority.clone(), + observed, + root.join("owner/active.sqlite"), + Owner { + session: LEADER, + endpoint: "https://process-fixture.internal:8081".into(), + }, + ) + .await + .unwrap(); + if partition { + let first = handle + .execute( + mutation_identity_window(155, 10, 10_000), + Digest::from_bytes([156; 32]), + 20, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success( + b"published-before-partition".to_vec(), + )) + }, + ) + .await + .unwrap(); + assert_eq!(first.commit_sequence(), 1); + assert_eq!( + authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap() + .value() + .ltx_root() + .unwrap() + .commit_sequence, + 1 + ); + } + let transport: Arc = Arc::new(NativeTransport::open(&root)); + let gate = + DurabilityGate::new(LEADER, NodeId::from_bytes(*LEADER.as_bytes()), 1, MEMBERS).unwrap(); + let shipper = NodeLogShipper::new(gate.clone(), transport.clone(), Limits::default()).unwrap(); + runtime + .install_node_durability( + target.application(), + Arc::new(NodeDurability::new( + gate, + shipper, + Arc::new(NativeAuthority(directory(&layout))), + transport, + lease, + )), + ) + .unwrap(); + let responses = Arc::new(super::super::durability::RecordingResponses::default()); + runtime.install_telemetry(responses.clone()).unwrap(); + runtime.active_catalog_entries().await.unwrap(); + if partition { + std::fs::write( + root.join("ready-to-partition"), + b"published-before-partition:1", + ) + .unwrap(); + while !root.join("partition-started").exists() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let uncertain = tokio::time::timeout( + std::time::Duration::from_secs(5), + handle.execute( + mutation_identity_window(157, 10, 10_000), + Digest::from_bytes([158; 32]), + 21, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(b"uncertain-partition".to_vec())) + }, + ), + ) + .await; + assert!( + !matches!(uncertain, Ok(Ok(_))), + "partition exposed unproved success: {uncertain:?}" + ); + assert!(responses.0.lock().unwrap().is_empty()); + // Timeout is uncertain: it does not certify that SQLite never committed. + std::fs::write(root.join("partition-result"), format!("{uncertain:?}")).unwrap(); + std::fs::write(root.join("acknowledged"), b"published-before-partition:1").unwrap(); + std::future::pending::<()>().await; + } + // Hold the first data PUT, before either bytes or root credit are published. + pausing.arm(); + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(10), + handle.execute( + mutation_identity_window(155, 10, 10_000), + Digest::from_bytes([156; 32]), + 20, + 1_024, + 1_024, + |tx| { + tx.execute("UPDATE counter SET value = value + 1", [])?; + Ok(HandlerOutcome::Success(b"fleet-tail".to_vec())) + }, + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + outcome, + StoredOutcome::Success { + result: b"fleet-tail".to_vec(), + commit_sequence: 1 + } + ); + assert_eq!( + responses.0.lock().unwrap().as_slice(), + &[cellule_runtime::fleet::telemetry::CommandResponseSource::Fleet] + ); + pausing.wait_until_blocked().await; + assert_eq!( + authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap() + .value() + .ltx_root() + .unwrap() + .commit_sequence, + 0 + ); + std::fs::write(root.join("acknowledged"), b"fleet-tail:1").unwrap(); + std::future::pending::<()>().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires isolated RustFS, explicit credentials and CELLULE_TEST_DEVICE_PATH on a device filesystem"] +async fn rustfs_killed_follower_tail_owner_recovers_every_ack_and_exact_retry() { + run_cut(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires isolated RustFS, explicit credentials and CELLULE_TEST_DEVICE_PATH on a device filesystem"] +async fn rustfs_partitioned_owner_exposes_no_unproved_success_and_recovers_earlier_ack() { + run_cut(true).await; +} + +async fn run_cut(partition: bool) { + let device = PathBuf::from(std::env::var("CELLULE_TEST_DEVICE_PATH").unwrap()); + let scratch = tempfile::tempdir_in(device).unwrap(); + let root = scratch.path(); + std::fs::create_dir(root.join("owner")).unwrap(); + let (target, layout, replica, authority) = fixture(sparse_process::rustfs_store(), partition); + let catalog = cellule_runtime::cell::catalog::CellCatalog::new(layout.clone(), target.tenant()); + let proof = catalog + .provision( + CatalogEntry::new( + &target, + CatalogRole::Application, + Digest::from_bytes([65; 32]), + 1, + ) + .unwrap(), + ) + .await + .unwrap(); + let initial = authority + .create_initial( + &proof, + IncarnationId::from_bytes([64; 16]), + Owner { + session: LEADER, + endpoint: "https://process-fixture.internal:8081".into(), + }, + ) + .await + .unwrap(); + let bootstrap_runtime = + CellRuntime::new(SqlWorkerPool::new(1, 10).unwrap(), 64 << 20, LEADER).unwrap(); + let bootstrap = bootstrap_runtime + .bootstrap( + proof.clone(), + replica.clone(), + authority.clone(), + initial, + root.join("bootstrap.sqlite"), + |tx| { + tx.execute_batch( + "CREATE TABLE counter(value INTEGER NOT NULL); INSERT INTO counter VALUES(0)", + )?; + Ok(()) + }, + ) + .await + .unwrap(); + bootstrap.drain().await.unwrap(); + bootstrap_runtime.shutdown().await.unwrap(); + let directory = directory(&layout); + // The exact ensemble is authoritative before the child can acknowledge. + enroll(&directory).await; + let proxy = if partition { + Some(super::partition_proxy::PartitionProxy::start().await) + } else { + None + }; + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", CHILD, "--nocapture"]) + .env(CHILD_ROOT, root) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + if let Some(proxy) = &proxy { + command + .env(PARTITION, "1") + .env("CELLULE_TEST_ENDPOINT", &proxy.endpoint); + } + let mut child = KillOnDrop(command.spawn().unwrap()); + tokio::time::timeout(std::time::Duration::from_secs(20), async { + loop { + assert!( + child.0.try_wait().unwrap().is_none(), + "child exited before its ACK cut" + ); + if root.join("acknowledged").exists() { + break; + } + if partition + && root.join("ready-to-partition").exists() + && !root.join("partition-started").exists() + { + assert_eq!( + std::fs::read(root.join("ready-to-partition")).unwrap(), + b"published-before-partition:1" + ); + proxy.as_ref().unwrap().partition(); + std::fs::write(root.join("partition-started"), b"network-cut").unwrap(); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!( + std::fs::read(root.join("acknowledged")).unwrap(), + if partition { + b"published-before-partition:1".as_slice() + } else { + b"fleet-tail:1".as_slice() + } + ); + child.0.kill().unwrap(); + assert!(!child.0.wait().unwrap().success()); + if let Some(proxy) = proxy { + assert!( + proxy.rejected() > 0, + "network partition was never exercised" + ); + eprintln!( + "partition={} rejected_connections={} response={}", + proxy.endpoint, + proxy.rejected(), + std::fs::read_to_string(root.join("partition-result")).unwrap() + ); + proxy.shutdown().await; + } + std::fs::remove_dir_all(root.join("owner")).unwrap(); + assert_eq!( + authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap() + .value() + .ltx_root() + .unwrap() + .commit_sequence, + u64::from(partition) + ); + let transport: Arc = Arc::new(NativeTransport::open(root)); + let fenced = directory + .claim_expired(LEADER, SUCCESSOR, 10_001) + .await + .unwrap(); + for member in MEMBERS { + assert_eq!( + transport + .seal( + member, + SealRequest { + leader_session: LEADER, + log_epoch: 1 + } + ) + .await + .unwrap(), + FollowerReceipt { + base_sequence: 1, + durable_through: 1 + } + ); + let frames = transport + .tail( + member, + TailRequest { + leader_session: LEADER, + log_epoch: 1, + first_sequence: 1, + }, + ) + .await + .unwrap(); + assert_eq!(frames.len(), 1); + let frame = cellule_ltx::inspect_node_frame(frames[0].clone(), Limits::default()).unwrap(); + assert_eq!(frame.scope().leader_session, *LEADER.as_bytes()); + assert_eq!(frame.scope().log_epoch, 1); + assert_eq!(frame.scope().node_sequence, 1); + assert_eq!(frame.scope().commit_sequence, if partition { 2 } else { 1 }); + assert_eq!(frame.scope().cell, *target.cell_id().as_bytes()); + } + let recovery = cellule_runtime::node::log_recovery::NodeLogRecovery::from_fenced( + transport, + &fenced, + Limits::default(), + ) + .unwrap(); + let manifests = cellule_runtime::recovery::manifest::RecoveryManifestStore::new( + layout.clone(), + Limits::default(), + ); + let coordinator = + cellule_runtime::node::log_recovery::RecoveryCoordinator::new(recovery, manifests.clone()); + let inventory = + cellule_runtime::node::log_recovery::recoverable_cells(&catalog, &authority, LEADER, 10) + .await + .unwrap(); + let completed = coordinator + .recover_and_seal(&directory, fenced, inventory, 10_002) + .await + .unwrap(); + assert_eq!(completed.controls.len(), 1); + let successor = + CellRuntime::new(SqlWorkerPool::new(1, 10).unwrap(), 64 << 20, SUCCESSOR).unwrap(); + let restored = successor + .takeover_restored( + proof, + replica, + authority.clone(), + completed.controls.into_iter().next().unwrap(), + completed.takeover, + manifests, + root.join("successor.sqlite"), + Owner { + session: SUCCESSOR, + endpoint: "https://successor.internal:8081".into(), + }, + ) + .await + .unwrap(); + let retry = restored + .execute( + mutation_identity_window(155, 10, 10_000), + Digest::from_bytes([156; 32]), + 30, + 1_024, + 1_024, + |_| { + Err(cellule_runtime::Error::Control( + "acknowledged follower-tail retry executed twice", + )) + }, + ) + .await + .unwrap(); + assert_eq!( + retry, + StoredOutcome::Success { + result: if partition { + b"published-before-partition".to_vec() + } else { + b"fleet-tail".to_vec() + }, + commit_sequence: 1 + } + ); + let bytes = restored + .query(64, 64, |connection| { + let value: i64 = + connection.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }) + .await + .unwrap(); + assert_eq!( + i64::from_be_bytes(bytes.try_into().unwrap()), + if partition { 2 } else { 1 } + ); + if partition { + let resolved = restored + .execute( + mutation_identity_window(157, 10, 10_000), + Digest::from_bytes([158; 32]), + 31, + 1_024, + 1_024, + |_| { + Err(cellule_runtime::Error::Control( + "recovered uncertain partition retry executed twice", + )) + }, + ) + .await + .unwrap(); + assert_eq!( + resolved, + StoredOutcome::Success { + result: b"uncertain-partition".to_vec(), + commit_sequence: 2 + } + ); + let bytes = restored + .query(64, 64, |connection| { + let value: i64 = + connection.query_row("SELECT value FROM counter", [], |row| row.get(0))?; + Ok(value.to_be_bytes().to_vec()) + }) + .await + .unwrap(); + assert_eq!(i64::from_be_bytes(bytes.try_into().unwrap()), 2); + } + restored.drain().await.unwrap(); + successor.shutdown().await.unwrap(); + scratch.close().unwrap(); +} diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs new file mode 100644 index 00000000..b08df5c9 --- /dev/null +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/partition_proxy.rs @@ -0,0 +1,92 @@ +//! Test-owned TCP partition of one child's real RustFS transport. +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::watch; +use tokio::task::{JoinHandle, JoinSet}; + +pub(super) struct PartitionProxy { + pub(super) endpoint: String, + mode: watch::Sender, + rejected: Arc, + joined: Option>, +} +impl PartitionProxy { + pub(super) async fn start() -> Self { + let endpoint = std::env::var("CELLULE_TEST_ENDPOINT").unwrap(); + // This isolated qualification uses plain HTTP to local RustFS. The + // proxy forwards the original signed bytes, including the Host header. + let destination = endpoint + .strip_prefix("http://") + .unwrap() + .trim_end_matches('/') + .to_owned(); + assert!(!destination.contains('/')); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let (mode, mut changed) = watch::channel(0_u8); + let rejected = Arc::new(AtomicUsize::new(0)); + let counted = rejected.clone(); + let joined = tokio::spawn(async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + biased; + _ = changed.changed() => { + if *changed.borrow() == 2 { break; } + } + Some(result) = connections.join_next(), if !connections.is_empty() => { result.unwrap(); } + incoming = listener.accept() => { + let (mut downstream, _) = incoming.unwrap(); + if *changed.borrow() != 0 { + counted.fetch_add(1, Ordering::SeqCst); + continue; + } + let mut cut = changed.clone(); + let destination = destination.clone(); + let counted = counted.clone(); + connections.spawn(async move { + let mut upstream = TcpStream::connect(destination).await.unwrap(); + tokio::select! { + biased; + _ = cut.changed() => { counted.fetch_add(1, Ordering::SeqCst); } + _ = tokio::io::copy_bidirectional(&mut downstream, &mut upstream) => {} + } + }); + } + } + } + connections.abort_all(); + while let Some(result) = connections.join_next().await { + if let Err(error) = result { + assert!(error.is_cancelled()); + } + } + }); + Self { + endpoint, + mode, + rejected, + joined: Some(joined), + } + } + pub(super) fn partition(&self) { + self.mode.send(1).unwrap(); + } + pub(super) fn rejected(&self) -> usize { + self.rejected.load(Ordering::SeqCst) + } + pub(super) async fn shutdown(mut self) { + self.mode.send(2).unwrap(); + self.joined.take().unwrap().await.unwrap(); + } +} +impl Drop for PartitionProxy { + fn drop(&mut self) { + if let Some(joined) = &self.joined { + joined.abort(); + } + } +} diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs index ee377e0e..b4c2c535 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/recovery.rs @@ -263,6 +263,23 @@ async fn recover_retained_tail(rooted: bool, fail_at: u8, interrupted: bool) { completed.sealed.log().phase(), cellule_runtime::node::log_state::NodeLogPhase::Sealed ); + // A terminal recovery retry may adopt the exact attached result, but it + // cannot certify the stale pre-recovery control under that same seal. + assert!(matches!( + resumed + .recover_and_seal( + &directory, + fenced.clone(), + vec![cellule_runtime::node::log_recovery::RecoveryCell { + application: fixture.target.application(), + authority: authority.clone(), + observed: stale.clone(), + }], + 10_003, + ) + .await, + Err(cellule_runtime::Error::Fenced) + )); let repeated = resumed .recover_and_seal( &directory, diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs index 861aa92f..b0744bf5 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/ownership/sparse_process.rs @@ -8,7 +8,7 @@ const CHILD_PATH: &str = "runtime::lifecycle::ownership::sparse_process::rustfs_ const ACTIVE_PATH_ENV: &str = "CELLULE_LTX_SPARSE_OWNER_ACTIVE"; const READY_PATH_ENV: &str = "CELLULE_LTX_SPARSE_OWNER_READY"; -fn rustfs_store() -> Store { +pub(super) fn rustfs_store() -> Store { let required = |name| std::env::var(name).unwrap_or_else(|_| panic!("missing {name}")); build_explicit_store( &required("CELLULE_TEST_BUCKET"), @@ -258,6 +258,31 @@ async fn rustfs_process_killed_sparse_owner_restores_published_root_and_continue ) .await .unwrap(); + // The killed owner acknowledged this exact mutation. Recovery must retain + // both its state and request outcome: an original retry must never execute + // the mutation again merely because the local writer files disappeared. + let retried = successor + .execute( + mutation_identity_window(67, 10, 10_000), + Digest::from_bytes([68; 32]), + 21, + 1_024, + 1_024, + |_| { + Err(cellule_runtime::Error::Control( + "acknowledged retry executed after owner kill", + )) + }, + ) + .await + .unwrap(); + assert_eq!( + retried, + StoredOutcome::Success { + result: b"published".to_vec(), + commit_sequence: 1, + } + ); let outcome = successor .execute( mutation_identity_window(71, 10, 10_000), diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs b/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs index f3968acb..42ab2212 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/receiver.rs @@ -230,10 +230,16 @@ async fn prepared_credit_transfers_into_exact_root_sql_and_one_fenced_writer() { .is_none() ); retire(&receiver, &prepared).await; - assert_eq!(receiver.stats().retained_bytes(), 0); + assert!(matches!( + prepared.state(), + Err(cellule_runtime::Error::RuntimeClosed) + )); assert_eq!(receiver.stats().active_cells(), 1); activated.drain().await.unwrap(); receiver.shutdown().await.unwrap(); + // Resolve completion and native background work retain their own bytes + // while serving. The joined shutdown is the boundary for total reclamation. + assert_eq!(receiver.stats().retained_bytes(), 0); assert_eq!(receiver.stats().active_cells(), 0); assert_eq!(receiver.stats().resident_bytes(), 0); assert_eq!(receiver.stats().file_descriptors(), 0); diff --git a/docs/bundle-coverage-implementation.md b/docs/bundle-coverage-implementation.md index 2fbf1be1..295b748c 100644 --- a/docs/bundle-coverage-implementation.md +++ b/docs/bundle-coverage-implementation.md @@ -1,6 +1,11 @@ # Bundle coverage implementation -The current paired 2,000-Cell measurements +Fresh Node Bundle v1 measurements pass a five-minute 500-offered-write/s +milestone; the separate 2K-write/20K-read target remains unqualified. See +[publication-progress measurements](publication-progress-measurements.md) for +the current source, resources, errors, drops and complete ACK audits. + +The historical paired 2,000-Cell measurements complete 502.28 Fleet writes/s versus 455.73 before and 1,999.83 for fresh celld. Metadata windows reduce observed GET/range work, but successful scheduled write p99 worsens 827.14→1,454.42 ms and returned errors increase. Read-only @@ -236,7 +241,8 @@ reverified. The 215-command checkpoint density is now represented and measured for a small-image fixture; total lifecycle cost is not qualified. Host memory, native-job admission and materializer progress still require qualification. The actor now owns bounded admitted scheduling and joins accepted jobs; its -latest application measurement fails availability. See the +fresh mixed application measurement fails delivery at the full offered targets, +while the smaller 500-write/s milestone passes. See the [node performance design](../crates/cellule-runtime/docs/write-performance-design.md). ## Verification and remaining delivery diff --git a/docs/fleet-operations-plan.md b/docs/fleet-operations-plan.md index 9eb52b0a..629d3936 100644 --- a/docs/fleet-operations-plan.md +++ b/docs/fleet-operations-plan.md @@ -1853,7 +1853,10 @@ combined reader/follower `maintenance-roles` with the evidence limits recorded in execution evidence. Reader-only, live-follower-only and combined reader/follower maintenance each have focused end-to-end minion scenarios, as does failed-owner follower -maintenance. A role-complete maintenance CLI and the full fault/provider +maintenance. Live-follower errors identify the failing reconciliation, rotation +or readback phase and preserve the original source error. This context supports +diagnosing intermittent failures; it does not establish their earlier cause. +A role-complete maintenance CLI and the full fault/provider qualification remain required: ```sh diff --git a/docs/managed-bundle-pipeline.md b/docs/managed-bundle-pipeline.md index 1f0b4f3f..b00448e9 100644 --- a/docs/managed-bundle-pipeline.md +++ b/docs/managed-bundle-pipeline.md @@ -1,16 +1,17 @@ # Managed bundle publication pipeline -October 10, 2026. The runtime-owned producer can stage and upload one successor +October 11, 2026. The runtime-owned producer can stage and upload one successor while its predecessor is publishing. Selection and receipt confirmation remain -ordered. The latest completed production diagnostic reaches 1,249.90 writes/s -and 12,580.70 reads/s, with 240.5-ms write request p99, dropped offers and growing -root-materialization debt. Removing the checkpoint serial guard did not improve -completed throughput in its complete pair and remains an unadopted experiment. -**The 2,000-Cell mixed-load goal remains unmet; these short runs do not qualify -sustainable capacity.** +ordered. Stage accounting and phase telemetry now identify retained-byte pressure +and root work separately. Optional overlap and parallel materialization preserve +foreground headroom; historical frame reads overlap within the original admitted +suffix bound. -These measurements predate the [Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds). -They are historical evidence, not measurements of the new format. +Fresh 2,000-Cell runs with the current format are recorded in +[publication-progress measurements](publication-progress-measurements.md). +The historical application table below predates the +[Node Bundle v1 cutover](bundle-coverage-implementation.md#format-and-bounds); +it does not qualify the current implementation or the mixed-load goal. ## Publication and admission @@ -24,7 +25,7 @@ The manual `select` method uses these same three phases. | Event | Managed behavior | | --- | --- | | First cohort starts | Keep the existing 20-MiB working reservation and 2-MiB shard cache | -| Another capture arrives during publication | Try to admit another 20 MiB plus conservative receipt bounds for both cohorts on the original ledger | +| Another capture arrives during publication | Admit the first proposal's encoded body and retained native metadata plus conservative receipt bounds for both cohorts; share the original preparation/verification scratch while preserving foreground burst headroom | | Extra credit unavailable | Keep that capture for the next serial round; do not wait while holding catalog ordering | | Successor PUT finishes first | Retain uploaded bytes; grant no coverage or response credit | | First selection succeeds | Confirm its original receipts and release its native capture credit independently of the second PUT | @@ -36,6 +37,12 @@ bounds. Receipt admission covers the maximum native history, live assignments, bounded control allocation and origin path. Confirmation still checks actual costs before transferring reservations or waking any capture. +Retained receipts charge locator and assignment capacities, the origin path, +the control's endpoint capacity, and the inline proof and wrapper storage. +The decoded control holds no serialized copies; those transient buffers belong +to the unchanged publication working reservation. An exhaustive control-field +match keeps future heap-owning fields part of this accounting decision. + The two-cohort window is lazy: a capture can arrive after the first PUT has started. There is no requirement to collect both cohorts before dispatching the first. Selection advances only the original complete assignments and @@ -43,11 +50,31 @@ preserves heartbeat and coverage changes observed after staging. ## Checkpoints and joined lifetime -A first cohort containing root checkpoint notifications uses serial publication. -The lookahead cohort contains native captures only. Ready root notifications -join the next first cohort or the existing idle checkpoint path. This preserves -the original root-credit release path and bounds callback retention during a -held upload. +Materialization overlaps at most eight original authenticated frame-range reads +within its existing `6 × suffix bytes + 4 MiB` working reservation. Verification +consumes frame facts in locator order. A failure stops new dispatch, joins the +accepted window, preserves the first source error and grants no root credit. +Base-origin verification remains fresh and precedes those reads. + +When root I/O is already alive and even the minimum working reservation +cannot fit, dispatch skips the repeated dirty-fleet walk and sort. This is only +a scheduling hint; the original atomic admission still decides each job. + +Both cohorts may combine ready root checkpoints and native captures within the +original 64-row bound. A successor inherits its predecessor's encoded checkpoint +bases. Each callback receives credit only after its own fresh verification and +ordered authority CAS; an uploaded successor cannot credit either cohort early. +Standalone checkpoints never run while that round holds catalog ordering. + +After encoding and self-verification, a staged proposal retains its immutable +body and only the native bindings needed for selection. Loaded shard/history +snapshots and unrelated bindings are released; the complete wire bytes remain +unchanged. Staged metadata is bounded to 4 MiB and its capacities are charged +with the body under the optional overlap reservation. Heavy successor preparation +and predecessor verification share one scoped scratch lane. Uploads may overlap, +while the existing five-buffer working reservation, 64-MiB budget and foreground +headroom threshold remain unchanged. Retained receipts keep their separate +conservative admission. When extra receipt credit is unavailable, the round returns catalog ordering before the existing receipt-admission wait. That wait continues to service @@ -58,6 +85,17 @@ Managed failure, fencing and shutdown do not cancel already-dispatched uploads or checkpoint I/O. Cancelling a shutdown caller leaves the original worker alive; a later shutdown joins that worker and observes its terminal result. Manual callers continue to own their admission and accepted I/O lifetimes. +An accepted checkpoint waiter also joins its original callback. Fencing cannot +replace that callback's source error while a sibling upload delays the worker's +terminal status; a successful callback still checks the original lease. + +The first follower proof also owns a node authority activation obligation. +One epoch-owned driver coalesces activation waiters; cancellation or an Object +proof winning cannot abandon an accepted CAS before the adapter joins its reply. +Shutdown freezes further activation starts and joins that driver before coverage +or closure, including earlier shipping/publication failures. Lease fencing remains +terminal. A failed callback grants no Fleet proof; complete Object coverage can +still drain the epoch. ## Regression evidence @@ -75,6 +113,14 @@ maximum-history and full-cohort fixtures check the conservative receipt bound. The real example authority also exercises two staged cohorts, heartbeat renewal, blocked binding and rejection of out-of-order selection. +Checkpoint regressions hold a checkpoint-bearing first upload and observe the +successor upload under the unchanged 64-MiB budget. A second case composes +checkpoints in both cohorts, releases the first callback while the second upload +is held, and cold-restores both exact histories. Upload failure, fencing, +cancelled shutdown and a missing checkpoint base preserve source errors, refuse +early credit and join every accepted upload. A selected-debt fixture verifies +overlap without consuming foreground burst headroom. + A longer-stall regression holds the first PUT across 31 seconds and renews the writer during that wait. Its initial implementation rejects successor staging because the advertisement saved at round entry has expired. The round now pins @@ -88,7 +134,7 @@ compatibility, dependency verification and two-preparation model checks. Those model checks do not establish host memory accounting or adapter liveness; the managed tests cover those lifetimes separately. -## Current application measurement +## Historical application measurement The completed production baseline uses `4712c4c`, whose Rust/Cargo sources match `ea4903d` on main. One complete pair and an unpaired candidate repeat use an @@ -122,15 +168,82 @@ Builds, failed/passed logs and snapshots are retained outside Git under `3e62e448476193424d8c555ca9b021e130819f01f576ff180ff089bff2869b85`. These diagnostics establish neither sustainable capacity nor celld parity. -CNB3 stores a 32-KiB authenticated header, changed catalog shards, new native +The historical format stored a 32-KiB authenticated header, changed catalog shards, new native frames and detached Cell histories in one immutable upload. Unchanged extents -remain exact references to earlier objects. The -[format and recovery contract](bundle-coverage-implementation.md) describe its -digest chain and CNB1/CNB2 reader compatibility. The experiment changes no bytes. +remained exact references to earlier objects. The experiment changed no bytes. +The current [format and recovery contract](bundle-coverage-implementation.md) +uses Node Bundle v1 exclusively; legacy bundle readers were removed at cutover. + +## Publication diagnostics + +The runtime's `CellTelemetry` sink observes managed bundle preparation, upload, +selection, receipt admission, materialization, checkpoint callbacks and worker +binding. Install the same `CellTelemetryHandle` on `NodeDirectory` with +`with_telemetry` to separate fresh dependency verification and the authority +CAS inside selection. Phase callbacks include success, source failure and +cancellation; the sink captured at phase entry receives both entry and exit. +Callbacks must remain bounded and nonblocking. + +The SQL example exports these observations under `node_publication`, with raw +cumulative histograms under `histograms.node_publication_*`. Selection includes +verification and CAS; materialization includes root reconstruction and its Cell +CAS. These nested durations and overlapping pipeline work cannot be summed as +request latency. `checkpoint_wait` includes the original queued notification; +`checkpoint` measures a standalone authority checkpoint. Native cohort counters +report captures, frames, bytes and included checkpoint notifications. Serial +fallback counters distinguish checkpoints, capacity, foreground headroom and an +absent successor. Optional overlap admission atomically stays below the same +75% retained-byte threshold used by ordinary write admission, reserving another +eighth of the budget below that threshold for foreground bursts. Parallel root +materializers use the same working threshold. When no root I/O is alive, one +required cleanup may use the original full budget so existing debt cannot starve +its own reduction. The working reservation releases this opportunity when root +I/O and its Cell CAS join. Prefix metadata remains admitted through checkpoint +and worker binding, without serializing another Cell's root I/O with that wait. +The original eight-task limit includes both working and checkpoint phases. +These rules do not increase the total budget or reduce receipt or materialization +bounds. + +`CellRuntime::retained_work` returns one locked ledger snapshot whose +`retained_by_stage` counts sum to its retained total. Ownership classes follow +the original reservation through split, reclassification, shrink and drop; +they report admitted reservations rather than sampled heap use. Unclassified work +remains explicit as `other`. The separate native-log byte window still has its +own accounting. The example exports this snapshot under `retained_work`. +Its `working_materializations` count reports original root I/O admissions in the +same locked snapshot; retained checkpoint metadata does not hold that count. + +Actor admission refusals are counted separately from application HTTP failures: +a backlog-refused mutation may still resolve an existing durable outcome. +The example's finite HTTP failure classes preserve the source chain and public +error envelope, including post-commit capacity failures. Neither counter grants +an outcome or recovery proof. Use raw request journals to reconcile offered, +successful, failed, dropped and late requests. + +The 45-second root age trigger is a materialization policy, not a guarantee +that bucket selection is delayed by 45 seconds. Compare issued and selected +node-log frontiers, retained ownership, root debt age and phase service rates +before attributing an old root to a slow bundle upload. ## Verification -All 13 contributor routes pass on an isolated snapshot, including 2,095 +The publication-progress changes pass 2,112 workspace tests and 15 SQL example +tests on an isolated snapshot, plus strict all-target/all-feature Clippy, +formatting, boundary/layout checks and document/SQL validators. Focused tests +reproduce the original pipeline and materializer headroom failures, then verify +foreground admission and continued cleanup against the same ledger. A held-first +frame fixture reproduces serial historical reads; its bounded-overlap replacement +cold-restores the exact outcomes. The failure-window fixture verifies joined +accepted reads, preserved source errors and unchanged root authority. +Additional regressions distinguish joined root working admission from retained +checkpoint metadata, charge spare endpoint allocation without retaining codec +scratch, join accepted activation after proof-waiter cancellation, and bound the telemetry wrapper's async frame. The latter +reproduces a large inline child frame; boxing work at construction preserves phase +outcomes and avoids the webhook scenario's default-worker stack overflow. Its +unoptimized demo and complete process journey pass without enlarging stacks or +changing the original fault assertions. + +At the earlier pipeline milestone, all 13 contributor routes passed on an isolated snapshot, including 2,095 workspace tests with 43 environment-dependent cases ignored, strict Clippy, all targets/features, API docs, local LTX, boundary/layout checks and document validators. The focused bundle suite passes 125 cases. The seven TLS-dependent diff --git a/docs/publication-progress-measurements.md b/docs/publication-progress-measurements.md new file mode 100644 index 00000000..dbb8bdc0 --- /dev/null +++ b/docs/publication-progress-measurements.md @@ -0,0 +1,980 @@ +# Publication-progress measurements + +These public-application diagnostics use Node Bundle v1, two required followers, +and the existing 64-MiB retained-work budget. The earlier 500-write/s milestone +uses an 8-vCPU owner. Fresh verification on three separate 1-vCPU/2-GiB VMs +has not passed latency and backlog qualification. The separate +2K-write/20K-read qualification remains open. + +## Fresh receiver-clock and peer-partition verification + +October 12, 2026 UTC. A three-VM partition repeat failed before the partition +was installed: sender verification rejected a signed append grant and disabled +fleet proofs. An isolated diagnostic recorded both receiver issue times ahead +of the sender's clock by 31 and 52 ms. Signature, boot, certificate, key, +ensemble and sequence-window checks passed; only the issue-time check failed. +The same grant implementation exists byte-for-byte on main `c79834d`. + +Sender verification now applies the existing node-advertisement clock-skew +bound. The signed lifetime, sender's original monotonic cache horizon, +receiver's strict local validity, native fsync gate and terminal fencing remain +unchanged. A signed-grant/native-append regression test fails before the change +with just 1 ms of clock difference. All nine grant tests pass afterward, and +all 903 runtime library tests pass (four environment-dependent tests ignored). +Strict runtime all-target/all-feature Clippy and SQL-example test Clippy pass. +The SQL example's 16 ordinary tests pass (eight environment-dependent tests +ignored). The remote fault child runs explicitly through the external driver. + +Two final partition repeats pass using the original mTLS and signed-grant +transport on three separate 1-vCPU/2-GiB VMs with native ext4. Each obtains one +unpublished ACK, blocks both actual follower TCP paths, verifies that a captured +second write and its observed read cannot return success, and kills the owner. +Recovery refuses both unreachable followers without taking over the Cell or +attaching an overlay. Reaching one complete original follower then recovers the +ACK and exact retry while the other path remains blocked. Both paths are healed +before joined shutdown; all owned roles become terminal without OOM. Actual +iptables packet-drop counters are retained at each partition stage. + +The original failures, diagnostic source/binaries and two final cases +(`peer-partition-v35-r9` and `r10`) remain under +`cellule-peer-partition-20261012-v35`. The before/after regression and complete +runtime library logs remain under `cellule-recovery-retry-20261012-v36`. +Temporary debug instrumentation is absent from the corrected source. This is +process-loss and follower-path verification, not physical power-loss or capacity +qualification. It does not identify the unlogged append failure in the earlier +750-write/s run or the original intermittent maintenance `Fenced` cause. + +The next matched capacity comparison applies the common transport correction +to both main and the PR, preserving the original workload, budgets, guards and +every-ACK audits. The workstation remains busy; no corrected capacity result or +performance improvement is claimed. + +Rust CI run `38286916209` at `48ff24b` fails the host lifecycle test's startup +snapshot equality, before shutdown. The default Host disk budget is +process-shared, and another parallel test changes its usage between the two +samples. An unrelated 4,096-byte reservation reproduces the same assertion +deterministically. The fixture now owns a private budget with the same capacity +and retains both the forced interleaving and the complete stats assertion. +The focused test, all 131 host node tests with default parallel execution, and +strict all-target/all-feature host Clippy pass. Original CI and local +before/after evidence remain retained under +`cellule-host-starting-stats-20261012-v42`. This fixture change leaves the +prepared main/PR benchmark binaries and all performance gates unchanged. + +Rust CI run `38289876672` at `14dcb55` fails the LTX concurrent-open test at +600 ms against its 500-ms assertion. The unchanged test reproduces 20/20 times +with two available job slots: three opens require two waves. Its fixture now +owns three job slots, so the assertion measures independent opens without +competing for the default process-wide pool. The same 500-ms assertion passes +100/100 times, and all 76 LTX host tests pass with default parallel execution +(three environment-dependent tests ignored). CI logs and the isolated +before/after evidence remain under `cellule-ltx-open-wave-20261012-v43`. +This test fixture change leaves production admission limits and the prepared +performance binaries unchanged. + +Rust CI run `38303478642` at `820b9e5` fails the receiver-credit test's +whole-runtime zero-byte assertion while its activated Cell is still serving: +112 bytes remain. The unchanged test reproduces that exact failure once in +188 attempts. A resolve completion owns 48 input bytes and 64 result bytes; +its reply releases the request slot before those reservations drop. Holding +that actual completion boundary briefly in an isolated diagnostic reproduces +the exact failure 5/5 times. A subsequent FIFO-only probe still fails 3/5 times +with other live native reservations, and that failed approach remains retained. + +The fixture now checks that the prepared receipt is closed and the activated +Cell remains serving, then asserts exact zero retained bytes after its existing +joined shutdown. Every original zero-resource, exact-root, SQL and fenced-writer +assertion remains. The corrected test passes 5/5 diagnostic runs and 1,000/1,000 +uninstrumented repeats with four parallel processes. All nine receiver tests, +all 246 runtime integration tests and strict integration-target Clippy pass; +six tests require their documented external environments and remain ignored. +An initial direct-executable suite omitted Cargo's movement-probe environment +and failed four process tests; its log is retained, and the complete suite +passes when launched through Cargo. Temporary instrumentation is absent from +the workspace and clean verification snapshot. Evidence remains under +`cellule-ci-failure-820-20261012-v51`. This fixture change does not alter +production accounting or the prepared performance binaries, and does not +diagnose the older unlogged maintenance `Fenced` failure. + +Rust CI run `38292326937` at `df37b68` passes the workspace tests, then fails +the original combined maintenance scenario with +`Node("native inventory category changed")` during observer confirmation. +A held native recheck and real second-Cell activation reproduce that error +5/5 times. The reference observer now discards that interval's role and policy +proofs, leaves maintenance blocked, and requires a fresh complete collection. +The seven-category verifier and its fencing/authority checks remain unchanged. + +Both the invalidated-inventory and fenced-node source-error tests pass 20 +repeats, and both original maintenance scenarios pass 10 repeats. Strict +example/test Clippy passes. The broader persisted-observer suite records +27 passes and two failures: a snapshot interval refusal in the new test's +fresh observation, and an operational sample timestamp ahead of its +advertisement in the real-expiry restart test. That failed attempt remains +retained; the unchanged clock-discipline repeat is recorded below. Evidence +remains under +`cellule-maintenance-inventory-20261012-v46`. This diagnoses the fresh inventory +race, not the original unlogged intermittent maintenance `Fenced` cause. +Only the reference example/tests/docs change; prepared SQL benchmark code and +all performance gates remain unchanged. + +## Fresh VM clock verification + +October 12, 2026 UTC. A 30-second read-only support-VM probe captured a +100.1-ms backward wall-clock step while monotonic time advanced normally. Its +sample timestamp matches the Lima guest-agent `SyncTime` adjustment in the +guest journal. All three target VMs also logged approximately 100-ms clock +adjustments. Installed Lima 2.1.4 has a matching +[upstream issue](https://github.com/lima-vm/lima/issues/5543). + +Only the four owned test VMs receive a reversible guest-agent systemd drop-in +removing `CAP_SYS_TIME`. Existing NTP services continue running. Five-minute +wall/monotonic probes then observe no backward steps in 53,610 total samples. +Original boot IDs, active guest agents and synchronized NTP are verified; +VM resources, Cellule code, original timestamps, deadlines and guards remain +unchanged. The performance controller resumes its original quiet-start policy. + +The exact frozen binary from the earlier failed persisted-observer run passes +all 29 tests in each of three repeats, including both timestamp-refusal cases +and the invalid-clock rejection test. The earlier 27-pass/two-failure attempt +remains retained. Clock steps are confirmed; their occurrence at those two +original test failures was not captured. This does not identify the original +unlogged maintenance `Fenced` cause or establish a performance improvement. +Before/after journals, every clock sample, capability/boot/NTP checks, rollback +procedure, unchanged binary hash and test logs remain under +`cellule-clock-discipline-20261012-v49`. No corrected idle-host capacity case +has yet been admitted. + +## Fresh hosted routing comparison + +October 12, 2026 UTC. Compose run `38274456201`, attempt 1, compares main +`c79834d` with merge revision `0816881` (parents main and PR head `9e68a3e`). +It retains four alternating pairs with the same harness on one hosted Linux +runner. Each command lane executes 1,024 commands and each warm query lane +executes 4,096 queries per run. The table reports medians of all four runs at +concurrency 16; latency is measured from invocation, without paced offers. + +| Lane | Main / PR operations per second | Main p50 / p95 / p99 | PR p50 / p95 / p99 | +| --- | ---: | --- | --- | +| Local writes | 619.80 / 616.28 | 24.54 / 36.15 / 41.99 ms | 24.63 / 35.38 / 43.35 ms | +| Forwarded writes | 497.19 / 504.74 | 29.45 / 40.15 / 45.69 ms | 29.15 / 39.68 / 44.48 ms | +| Local warm reads | 16,297.29 / 16,524.87 | 0.501 / 0.897 / 1.039 ms | 0.491 / 0.883 / 0.959 ms | +| Forwarded warm reads | 4,261.77 / 4,300.75 | 2.030 / 3.480 / 4.283 ms | 2.051 / 3.471 / 4.205 ms | + +The leased routing gate passes. Local writes deliver 0.57% less throughput and +3.24% higher p99; forwarded writes deliver 1.52% more throughput and 2.65% lower +p99. These small differences do not establish a general improvement. The +expired-burst local-query lane also retains a p99 alert (1.574 times main), +within the unchanged 2.0 gate. All lanes and the earlier failed comparison +remain included. + +The same run's object-only gate also passes. At concurrency 16, local writes +measure 424.25/s on main and 426.34/s on the PR, with p99 56.76 and 53.34 ms. +Forwarded writes measure 371.73/s and 367.92/s, with p99 58.54 and 59.50 ms. +Forwarded reads retain a p99 alert: 9.55 ms on main and 10.70 ms on the PR +(1.120 times main). Their throughput is 1,930.15/s and 1,870.17/s. These +results also show mixed small changes rather than a consistent gain. + +This fixture publishes per-cell roots. It does not run the three-VM shared-log +workload, enforce its 64-MiB profile, or qualify its sustainable capacity. +The raw `manifest.json`, `comparison.json`, binaries and all request logs are +retained under `cellule-ci-routing-9e-20261012-v37/{leased,object-only}`, from +[the leased CI job](https://github.com/crabbuild/cellule/actions/runs/38274456201). +The matched three-VM 750-write/s comparison is still waiting for five +continuous idle minutes on the workstation. Higher-rate, read and mixed-load +qualification remain open. + +Rust CI run `38274456048` at `9e68a3e` fails four idempotent recovery-retry +tests: the new renewal guard tries to validate a claim that the committed seal +has already cleared. An isolated Linux run reproduces all four failures. The +correction adopts only the canonical terminal result, checks exact controls, +manifest and referenced overlays, and performs no recovery authority writes. +All eight focused recovery tests pass, including rejection of stale controls; +all 1,317 runtime tests and doctests pass with all features. Strict all-target +runtime Clippy passes. Original CI logs and both local runs remain under +`cellule-recovery-retry-20261012-v36`. This is a recovery correctness fix; +the original intermittent maintenance `Fenced` cause remains unproved. + +## Fresh hosted routing repeats + +October 12, 2026 UTC. The newest Compose run `38303478631` completes four +alternating pairs with main `c79834d` and merge `ab5fcda` (PR head `820b9e5`). +Both measured binaries are byte-identical to runs `38286916365`, `38289876505` +and `38292326894`, and to `38299468996`; test fixtures, the reference observer and docs changed +between these revisions. All 242,192 raw latency samples per mode, publication +coverage and every four-run median reconcile through the canonical verifier. +The table reports the newest run, with invocation latency. + +| Leased lane, concurrency 16 | Main / PR operations per second | Main p50 / p95 / p99 | PR p50 / p95 / p99 | +| --- | ---: | --- | --- | +| Local writes | 950.69 / 873.67 | 15.710 / 24.102 / 31.714 ms | 15.652 / 26.950 / 34.894 ms | +| Forwarded writes | 714.00 / 740.91 | 19.450 / 30.864 / 43.050 ms | 19.609 / 28.342 / 36.139 ms | +| Local warm reads | 28,048.23 / 28,349.27 | 0.282 / 0.510 / 0.540 ms | 0.284 / 0.503 / 0.540 ms | +| Forwarded warm reads | 7,183.39 / 7,161.29 | 1.185 / 2.258 / 2.760 ms | 1.183 / 2.334 / 2.865 ms | + +The paired signs vary across hosted runs on identical binaries: + +| Run, four alternating pairs | Local write throughput / p99 change | Forwarded write throughput / p99 change | +| --- | ---: | ---: | +| `38286916365` | -1.65% / +7.24% | +6.59% / -21.68% | +| `38289876505` | -0.79% / +7.12% | -3.65% / +1.00% | +| `38292326894` | +0.33% / -3.62% | -0.43% / +0.46% | +| `38299468996` | -1.52% / +3.69% | -2.88% / +5.17% | +| `38303478631` | -8.10% / +10.03% | +3.77% / -16.05% | + +Both original coarse gates pass in the newest run. Leased local writes retain +p95/p99 alerts of 1.1181/1.1003 times main; leased expired-burst local reads +retain a p99 alert of 1.1752 times main. Object-only local reads at concurrency +one and fresh-client local reads at concurrency 16 retain p99 alerts of +1.6335/1.1014 times main. Object-only concurrency-16 local/forwarded writes +achieve 821.90/694.18 per second, against main's 828.60/690.97; their p99s are +32.361/33.037 ms, against 36.882/33.999 ms. Passing the original coarse gates +does not establish an improvement. All earlier alerts, including the previous +run's object-only forwarded-write p99 regression, remain retained. + +All lanes, object-only results, samples, binaries and manifests remain under +`cellule-ci-routing-820-20261012-v52`; previous runs remain under +`cellule-ci-routing-90a-20261012-v50`, +`cellule-ci-routing-df-20261012-v48`, `cellule-ci-routing-48-20261012-v44` and +`cellule-ci-routing-14-20261012-v47`. +[The newest CI run](https://github.com/crabbuild/cellule/actions/runs/38303478631) +uses per-cell roots and does not qualify the three-VM shared-log workload. +The matched idle-host capacity comparison and conditional rollback decision +remain open. Rust CI run +[`38299468998`](https://github.com/crabbuild/cellule/actions/runs/38299468998) +at PR head `90a2fab` passes both workspace and MSRV jobs. +The follow-up `820b9e5` workspace run fails the receiver fixture described above; +its passing performance jobs do not override that CI failure. + +## Latest 750-write/s PR #76 verification + +October 12, 2026 UTC. The fresh matched cases use 2,000 Cells and three +separate 1-vCPU/2-GiB Linux VMs, with native disk-backed node volumes. Both +use 96-byte SQL ledger rows, a 30-second warmup, a 300-second write window, +the unchanged 64-MiB retained-work budget and 128/128 client limits. Workload, +auditor and runner bytes match. Main `c79834d` includes only the common +request-deadline correction; the PR binary is built from `a0860db`, whose +production code matches PR head `89d4599`. + +| Fresh run | Completed inside window / offered | Errors / dropped offers | Scheduled p50 / p95 / p99, all attempts | Warm / cold ACK audits | +| --- | ---: | ---: | --- | --- | +| Main + common deadline correction | 85,803 / 225,000 (286.01/s) | 61,824 / 77,258 | 89.2 / 1,280.4 / 1,966.6 ms | All 99,711 pass / all 99,711 pass | +| PR #76 | 98,015 / 225,000 (326.72/s) | 405 / 126,580 | 279.3 / 2,463.0 / 4,110.5 ms | 110,861 checked, 73,262 errors / not reached | + +Successful-write scheduled p50/p95/p99, calculated directly from the request +journals using nearest rank, are 106.488/1,269.665/1,868.134 ms on main and +277.601/2,434.809/3,933.745 ms on the PR. All failed attempts and drops remain +qualification failures. These rates are delivered throughput at 750 offered +writes/s; they do not establish sustainable capacity. + +Both cases fail the existing delivery, 50-ms p99 and publication-backlog gates. +The last three one-minute segments have positive unpublished-byte slopes of +20,225 B/s on main and 205,167 B/s on the PR, and positive oldest-debt-age +slopes. Main's joined drain takes 90.62 seconds. The PR's warm audit reports +HTTP 503 fencing refusals; its owner then exceeds the unchanged 120-second +drain deadline and is stopped with exit 137. Both followers exit 0. No node +reports an OOM or swap activity. A refused audit response does not establish +that the acknowledged data was lost. + +The PR records one follower-append failure near the end of the timed window, +disables fleet proofs and enters rotation. The original failure cause was not +logged at that boundary. A failure-only warning now preserves the member, +sequence range and original error for the next reproduction; successful appends +do not emit it. The original case, all 110,861 ACKs, provider objects and native +volumes remain retained for canonical recovery investigation. + +Each window has 17 usable host/guest samples with zero collection errors. +Host CPU idle ranges are 23.13–62.60% on main and 8.39–70.71% on the PR. +Compilers appear in both traces; another native benchmark also appears during +the PR window. Neither window qualifies as an idle-host comparison. The +controller stops after the failed PR audit; it does not admit the second pair, +higher write rates or read/mixed cases. No performance gain or causal rollback +verdict is established. The original 2,000-Cell 750→1,000→1,500→2,000 ramp, +read/mixed qualification and original maintenance-fence diagnosis remain open. + +The retained `fresh-write-comparison-v30.json` and `.md` reports under +`cellule-origin-comparison-20261011-v19` reconcile every request journal, +original histogram, ACK manifest, publication frontier and resource window. +The PR case is under `cellule-grant-current-20261011-v18`. + +The earlier Compose CI run `38217748520` also retains a failed leased +local-command/c16 gate: main 553.509/s, PR 498.1505/s, ratio 0.899986 against +the unchanged 0.90 minimum. Its p95 ratio is 1.491617 and p99 ratio 0.854865. +All four pairs remain included. This hosted routing fixture publishes per-cell +roots and does not test the three-VM follower-log capacity path. Earlier Rust +CI run `38217748541` passes; that does not override the performance gate failure. + +## Earlier low-rate PR #76 write diagnostics + +October 12, 2026 UTC. These new runs use three separate Linux VMs, each with +1 vCPU and 2 GiB, on the shared workstation. Each has fresh native disk-backed +node storage. The provider and client run separately. Each workload has 16 Cells, +96-byte values, a 30-second warmup, a 300-second write-only window at 10 offered +writes/s, and the unchanged 64-MiB budget and 128/128 client limits. + +| Fresh run | Writes completed inside window / offered | Errors / drops | Scheduled p50 / p95 / p99 | Warm / cold ACK checks | +| --- | ---: | ---: | --- | --- | +| Main `c79834d`, common deadline correction only | 3,000 / 3,000 (10.000/s) | 0 / 0 | 25.6 / 476.0 / 1,341.3 ms | All 3,317 pass / all 3,317 pass | +| PR #76 with two failure-only tracing overlays | 2,705 / 3,000 (9.017/s) | 295 / 0 | 42.9 / 1,360.3 / 5,007.9 ms | 3,022 checked, 126 errors / not reached | +| PR #76, uninstrumented repeat | 2,998 / 3,000 (9.993/s) | 0 / 0 | 33.1 / 691.3 / 2,314.5 ms | All 3,317 pass / all 3,317 pass | + +The table retains canonical all-attempt percentiles; the tracing row includes +failed attempts. The uninstrumented repeat completes its remaining two writes +during in-flight drain. Its production code is identical to PR head `08c43e5`; +the binary was built from `a0860db`, with no measurement overlay. Main retains +its original append-grant reuse and includes only the request-deadline correction +common to both sides. Workload and auditor binaries are byte-identical. + +Both completed cases pass every ACK's warm/cold readback and original-request +retry. Joined drain takes 11.50 seconds on main and 11.59 seconds on the PR; +cold startup takes 1.16 and 2.51 seconds. Neither passes the existing 50-ms +scheduled write-p99 gate. The PR repeat also fails the unchanged backlog gate: +unpublished bytes have a positive 612.63 B/s least-squares slope over the last +three one-minute segments. Main's corresponding byte and age slopes are +non-positive. These are individual diagnostics, not maximum throughput results. + +All three windows have 17 usable host/guest samples and zero collection failures. +Host CPU idle ranges are 6.7–66.78% on main, 0–28.57% during tracing and +0–64.88% on the PR repeat. Unequal competing load prevents a causal performance +comparison. No gain over main has been established; the alternating idle-host +comparison and conditional rollback remain pending. The original 2,000-Cell +750→1,000→1,500→2,000 write/s ramp and its evidence requirements remain intact. + +The tracing run reproduces a distinct five-second SQL timeout, fencing two +Cells while the fleet remains active. It records zero failed log-append rounds +or follower append failures; the earlier pre-load append failure does not recur. +The warm audit fails, cold restore is not reached and owner shutdown exceeds its +unchanged deadline. A passing uninstrumented repeat does not erase that failure. +The earlier 2,000-Cell readiness timeout and all-failed 16-Cell sweep remain +retained, including the latter's missing in-window host/guest evidence. + +Raw artifacts are under `cellule-origin-comparison-20261011-v19`, +`cellule-append-trace-20261012-v23` and `cellule-grant-current-20261011-v18`. +The compact `fresh-write-comparison-v25.json` and `.md` reports reconcile request +journals, original histograms, source/binary manifests, ACK audits, publication +frontiers, provider lifecycle and resource traces. Earlier measurements follow. + +## Workload and resources + +| Setting | Actual configuration | +| --- | --- | +| Workload | 2,000 uniformly active Cells; distinct 96-byte SQL ledger INSERT plus SELECT; durable outcome retained for two hours | +| Requests | 30-second warmup; 128 in-flight requests and 128 queued offers; no timed-request retry | +| Serving owner | 8 vCPUs, 16 GiB; pinned CPUs 0–7 | +| Each follower | 1 vCPU, 2 GiB | +| Object provider | RustFS, 2 vCPUs, 2 GiB; fresh provider volume | +| Client/auditor | 4 vCPUs, 2 GiB | +| Linux VM | 12 vCPUs, 24 GiB; support roles share CPUs 8–11 | +| Host | Mac14,13; 12 physical/logical CPUs; 32 GiB RAM | +| Local database/follower storage | tmpfs; provider data on the VM disk | +| Evidence | Every ACK is read back and retried with the original request before and after joined drain/cold restore | + +The v6 and v7 write-only VM disks were under `/Users/haipingfu/.colima`, on the internal host volume. +The installed Colima CLI ignored the attempted `COLIMA_HOME` override; no +external-to-internal disk relocation occurred. The earlier failing profile and +its disk images remain preserved. Host process samples retain competing compiler +load. This shared development host is not a dedicated capacity-qualification +machine; these runs do not establish causal speedup percentages, physical disk +survival, maximum TPS, or celld parity. The runtime reads in these diagnostics +use the current live-owner lease mode; they do not implement a new per-read +follower barrier protocol. + +## Five-minute 500-write/s milestone + +Source `4ee7cf523e8b28f62c97601074db2752e42cf763`; all eight framework packages +were cleaned before building with pinned Rust 1.98.1. The SQL binary SHA-256 is +`5299a7dc2ce33b1cca3992f7f2e0001e5f2dd5ab14dec6f9f3617b4c88460dee`. + +| Measurement | Result | +| --- | ---: | +| Planned/generated writes | 150,000 / 150,000 | +| Completed within 300 seconds | 149,997 (499.99/s) | +| Completed including final in-flight requests | 150,000 | +| Errors / queue drops / unissued offers | 0 / 0 / 0 | +| Warmup errors / drops | 0 / 0 | +| Scheduled latency p50 / p95 / p99 | 6.6 / 10.3 / 16.1 ms | +| Request latency p50 / p95 / p99 | 5.8 / 9.1 / 13.9 ms | +| ACKs checked warm and cold, including setup/warmup/probe | 167,001 | +| Warm/cold readback or exact-retry failures | 0 / 0 | +| Joined owner/follower drain | 47.28 s | +| Cold startup | 11.84 s | + +Three offers complete just after the measurement boundary; the delivery test +counts them separately. Minute snapshots report zero admission refusals. + +| Window sample | Retained reservation bytes | Oldest root age | Issued minus bucket-covered frames | Pending root publications | +| --- | ---: | ---: | ---: | ---: | +| Start | 32,664,702 | 45.019 s | 8 | 1,902 | +| Minute 1 | 28,860,412 | 41.999 s | 6 | 1,905 | +| Minute 2 | 29,162,452 | 44.588 s | 8 | 1,980 | +| Minute 3 | 29,281,360 | 43.721 s | 25 | 1,860 | +| Minute 4 | 29,607,202 | 42.818 s | 9 | 1,851 | +| End | 39,664,838 | 46.113 s | 14 | 1,969 | + +Root age and population remain bounded around the existing 45-second policy +trigger. The end reservation includes 11,305,976 bytes of active materialization; +selection retains 5,242,400 bytes, and the publisher reserves 23,068,672 bytes. +These are ledger reservations, not a measurement of total process heap. + +| Completed phase | Count | Mean duration | +| --- | ---: | ---: | +| Preparation | 18,683 | 5.04 ms | +| Upload | 18,683 | 3.97 ms | +| Selection | 18,682 | 4.42 ms | +| Fresh verification inside selection | 18,682 | 1.51 ms | +| Authority CAS inside selection | 18,682 | 2.55 ms | +| Root materialization | 12,397 | 17.70 ms | +| Checkpoint notification wait | 12,395 | 32.64 ms | + +Every completed phase in this window reports zero failure/cancellation. +Nested and overlapping durations cannot be summed into request latency. Serial +fallback counts are 12,525 for foreground headroom, 6,127 for checkpoints and +31 for capacity; fallback preserves progress rather than rejecting the request. + +Raw artifacts are outside Git under +`cellule-publication-progress-20261011-memory-v6`, case +`cellule-fleet-parity-goal-memory-v6-internal-r1`. Build/source manifests, +request journals, minute snapshots, independent audits, process exits and +`internal-publication-analysis.json` are retained together. + +## Repeat after joining activation lifetimes + +Source `03d2eba21c64105841c5ff0d2cb671c4148edf53`, with SQL binary SHA-256 +`902ed4cd327ff514f79f5e6a47c2798a75a6894a5245015625823f79fb2fac70`. +The same five-minute workload and limits pass again after the authority activation +callback moves outside proof-waiter lifetimes. All eight framework packages were +cleaned for this fresh binary; its source manifest has no Rust/Cargo differences +from the committed implementation. + +| Measurement | Result | +| --- | ---: | +| Planned/generated/completed writes | 150,000 / 150,000 / 150,000 | +| Completed within the window | 149,990 (499.97/s) | +| Errors / queue drops / unissued offers | 0 / 0 / 0 | +| Warmup errors / drops | 0 / 0 | +| Scheduled latency p50 / p95 / p99 | 6.3 / 9.7 / 16.0 ms | +| Request latency p50 / p95 / p99 | 5.5 / 8.6 / 13.5 ms | +| All-ACK warm/cold readback and original retry checks | 167,001 each; zero failures | +| Joined fleet drain | 39.25 s | +| Cold startup | 9.73 s | + +Ten writes finish just after the window. Minute snapshots show zero admission +refusals, root age 42.0–48.9 seconds and 1,858–1,954 pending roots; the end age is +46.015 seconds. Retained reservations start at 28,399,209 bytes and end at +39,737,580 bytes, including root materialization in progress. Bucket frame lag +ranges from 6 to 54 in these samples, independently of root age. + +Completed selection averages 3.94 ms across 21,272 rounds; preparation 4.20 ms, +upload 3.44 ms, materialization 15.52 ms and checkpoint wait 29.16 ms. These are +nested/overlapping phase observations, not an additive response-time model or +an attributable speedup comparison. Host disk headroom decreased under swap +during the run. The next workload was held until the completed task-owned VM +could be stopped and its disks copied and verified on Workspace. + +Artifacts are under `cellule-publication-progress-20261011-activation-v7`, case +`cellule-fleet-parity-goal-activation-v7-corrected-r1`; +`acceptance-publication-analysis.json` pins the raw summary and snapshot hashes. +The first v7 attempt stopped before timed work because the copied harness +omitted its unchanged seed helper. Its logs and provider volume remain retained; +the corrected run uses the v6 helper byte-for-byte. + +## Fresh mixed and read-only windows + +The same v7 binary runs after the completed task-owned VM is stopped and both +images are copied byte-for-byte onto Workspace. `vm-relocation.json` records +matching pre-restart source/destination SHA-256 hashes, actual paths and the +unchanged 12-vCPU/24-GiB VM. Original provider volumes remain retained in that +image; the new case uses a fresh provider volume. The primary Colima profile is +untouched. This physical move occurred after the write-only repeat; it must not +be confused with the earlier ignored `COLIMA_HOME` override. + +Each window has a 30-second warmup and 60-second measurement with the same +2,000 Cells, SQL ledger, 64-MiB budget and 128/128 client limits. + +| Window | Offered/s | Successful/s in window | Returned errors | Dropped offers | Request p50 / p95 / p99 | Scheduled p50 / p95 / p99 | +| --- | ---: | ---: | ---: | ---: | --- | --- | +| Mixed writes | 2,000 | 971.25 | 53,037 | 8,662 | 6.2 / 18.5 / 36.6 ms | 7.7 / 20.9 / 39.3 ms | +| Mixed reads | 20,000 | 18,529.95 | 0 | 88,152 | 0.6 / 3.6 / 9.5 ms | 2.0 / 6.5 / 12.7 ms | +| Read-only reads | 20,000 | 15,806.40 | 0 | 251,484 | 0.5 / 4.2 / 17.0 ms | 1.9 / 10.8 / 38.5 ms | + +Every planned offer is generated: 120,000 mixed writes and 1,200,000 reads per +window. Mixed completions including in-flight drain are 58,301 writes and +1,111,848 reads; read-only completions are 948,516. Warmup mixed writes have +23,837 errors and 5,903 drops, mixed reads 61,229 drops, and read-only reads +54,460 drops. Warmup read errors are zero. These windows fail delivery at the +offered targets; successful-request latency alone does not establish capacity. + +All 90,562 ACKs, including setup/warmup/probes, pass both warm and cold readback +and original-request retry checks with zero errors. Joined fleet drain takes +38.77 seconds and cold startup 7.48 seconds. Correct ACK recovery and sustained +offered throughput remain separate observations. + +### Where the mixed window stalls + +Window telemetry classifies 53,035 new write failures as publication-backlog +refusals, with no new failures in the other finite classes. Client counts include +the final in-flight completions beyond the snapshot boundary. Retained +reservations stay near the 48-MiB foreground threshold: + +| Stage | Start | End | +| --- | ---: | ---: | +| Capture | 20,676,012 B | 13,796,358 B | +| Selection | 6,131,764 B | 8,023,906 B | +| Materialization | 455,328 B | 5,319,898 B | +| Publisher | 23,068,672 B | 23,068,672 B | +| Total, including small request/other reservations | 50,331,776 B | 50,244,114 B | + +Oldest root age grows 57.700→88.718 seconds. Issued-minus-bucket-covered frame +lag is 6,154→4,046, while follower proof ends only 21 frames behind issuance at +the end. The publisher completes 1,022 cohorts, including 60,234 captures and +63,892 frames: about 58.9 captures and 62.5 frames per cohort, near the existing +64-frame bound. Of these rounds, 1,019 use serial fallback for checkpoints. +There are two capacity fallbacks and no headroom fallbacks in the measured delta. + +| Completed phase | Mean duration | +| --- | ---: | +| Preparation | 24.52 ms | +| Upload | 22.06 ms | +| Selection, including fresh verification and CAS | 11.13 ms | +| Fresh verification | 4.91 ms | +| Authority CAS | 5.56 ms | +| Root materialization | 31.99 ms | +| Checkpoint notification wait | 100.02 ms | + +All completed phases have zero failure/cancellation. This points to shared +publication and checkpoint service under retained-byte pressure rather than +follower lag. It does not justify raising the budget or removing the checkpoint +ordering guard. The following read-only start/end snapshots show no pending +publication, no capture credit and zero bucket lag. + +The read-only window contains a sampled competing compiler load of 335.1% CPU; +the two mixed-window samples report zero. Read-only request maximum is 2.52 +seconds. The slower read-only result cannot establish a repeatable regression, +maximum throughput or a causal explanation; isolate generator/host stalls from +server execution in a controlled repeat. + +Artifacts: case `cellule-fleet-parity-read-mixed-activation-v7-workspace-r1` +under the v7 directory, with `mixed-publication-analysis.json`, +`read-only-publication-analysis.json` and `host-load-window-analysis.json`. + +## Next capacity work + +1. Qualify the checkpoint-overlap candidate below on an idle host. The earlier + mixed path fell back to serial publication in 1,019 of 1,022 rounds; the + candidate removes that fallback while preserving exact dependency verification, + ordered root credit and the authority CAS. Its first fresh 750-write/s run + failed delivery and debt gates under competing compiler load. +2. Reduce preparation/verification work and bytes per cohort. Use the new phase + and stage measurements to check actual allocation bounds and repeated catalog + work. Ramp fresh five-minute offers through 750, 1,000, 1,500 and 2,000 writes/s, + requiring zero refusals/drops, stable debt and every-ACK warm/cold checks under + the same 64-MiB budget. Smaller component timings are insufficient. +3. Repeat read and mixed windows on a controlled host, separating client schedule + stalls, synchronous SQL execution and support-role contention. Qualify hot-Cell + fairness and long-query isolation rather than extrapolating uniform warm reads. +4. Extend physical-media, crash/partition and maintenance qualification. The + activation-cancellation regression and process audits cover concrete runtime + lifetimes; tmpfs tests and joined cold restore cannot certify device loss. + +The hosted workspace run also retained two `Fenced` follower-maintenance example +failures. Both exact cases pass all-feature local execution and five repeated +runs. The full isolated public controller suite passes all 382 cases, including +both original failures. The original CI logs remain and the job is retried +without weaker assertions or longer deadlines; a passing repeat does not identify +or fix the intermittent failure's cause. + +### Checkpoint-overlap candidate + +The next candidate composes checkpoints in either of its two ordered cohorts, +releases duplicate preparation snapshots after encoding, and shares heavy +preparation/verification scratch. It preserves the existing 64-MiB budget, +foreground headroom, fresh dependency checks and authority CAS. An accepted +checkpoint joins its original callback to preserve upload source errors even +when a sibling upload delays worker termination. + +The focused bundle suite passes 150 cases; the subsequent six-case checkpoint +suite also covers a checkpoint in each cohort and independently ordered credit. +Exact encoding comparisons retain complete sibling catalogs and require +byte/digest equality. These are correctness and bounded-lifetime regressions; +the candidate's fresh application capacity ramp has not yet completed. The +500-write/s and mixed/read-only tables above still describe source `03d2eba`, +not this candidate. See the [managed protocol](managed-bundle-pipeline.md). + +Hosted workspace run [38206015346](https://github.com/crabbuild/cellule/actions/runs/38206015346) +at `9f22376` failed the checkpoint shutdown-cancellation test. An assertion-only +isolated reproduction showed its successor-entry notification returning with +only one current PUT entered: the notification belonged to the preceding +cohort. The test now consumes that prior notification before resetting its +counter and requires two current PUTs before injecting fencing. All six +checkpoint cases pass with the original deadlines, shutdown assertions, +resource-release checks and 64-MiB budget. Production code is unchanged; +this test-barrier correction does not establish a performance gain or identify +the original maintenance failure's cause. Artifacts are retained under +`cellule-checkpoint-barrier-20261012-v27`. + +Fresh hosted Rust run [38209589260](https://github.com/crabbuild/cellule/actions/runs/38209589260) +passed the full workspace, both original maintenance cases, MSRV and strict +lint at `23f035d`. The separate routing performance run retained a failure and +a source-identical repeat: + +| Forwarded writes, concurrency 1 | Main writes/s | PR writes/s | Main p95 / p99 | PR p95 / p99 | +| --- | ---: | ---: | ---: | ---: | +| [Attempt 1](https://github.com/crabbuild/cellule/actions/runs/38209589255/attempts/1), failed | 80.33 | 69.60 | 22.37 / 92.93 ms | 37.53 / 140.66 ms | +| [Attempt 2](https://github.com/crabbuild/cellule/actions/runs/38209589255/attempts/2), passed | 73.38 | 73.25 | 21.04 / 26.66 ms | 20.84 / 26.67 ms | + +Each attempt retains all four paired repeats against main `c79834d`, with the +same measurement harness and identical baseline/candidate binary hashes across +attempts. Comparisons were independently recomputed. No slow pair, threshold or +deadline was removed. Attempt 1 remains a performance failure; its regression +did not reproduce, and attempt 2 does not prove a fleet improvement. These are +single-Cell root-publication routing tests on hosted x86_64 Linux/RustFS, +without shared follower-log durability. They do not replace the three-VM +capacity ramp or its every-ACK audit. + +Two fresh [remote unpublished-tail cases](remote-follower-tail-verification.md) +also passed on three separate 1-vCPU/2-GiB VMs with native ext4. Each acknowledged +one write on both remote followers while bucket coverage remained zero, killed +the owner, waited for actual lease expiry, and recovered through the canonical +remote sealing/tail protocol into a fresh successor volume. Exact retry returned +the original result without executing its handler. Successor and followers +joined; all owned containers were terminal without OOM, and follower VM boots +and OOM counters were unchanged. Strict SQL-example test lint passed. The first +failed fixture invocation and all source/binary/resource evidence remain in +`cellule-remote-tail-20261012-v28`; successor enrollment then preceded the old +lease's expiry. These cases qualify one unpublished ACK each after process loss, +not physical power loss, peer partition, or throughput. + +The missing support VM and four raw TCP tunnels were restored. The first matched +750-write/s cases subsequently ran and failed qualification as recorded above; +the controller stopped after the PR audit failed. The original full ramp, +read/mixed tests and conditional rollback remain outstanding. + +A subsequent diagnostic attempts canonical recovery of that retained PR case, +using its original provider and follower disks. All native state was archived +before the attempt. Recovery discovers 1,968 still-owned Cells, then returns +`Fenced` after 42.40 seconds; no ACK recovery qualification is claimed. The +diagnostic driver's initial cleanup stopped the provider before joining the +followers, causing their later storage errors. The corrected dependency order +joins followers first. Both the recovery failure and failed cleanup remain +retained under `cellule-retained-audit-20261012-v31`. The original two-hour +request identities have expired; the diagnostic does not change clocks or +identities to manufacture an exact-retry pass. + +A second retained-case attempt reproduces the recovery failure. Its targeted +probe shows the same claimant, claim generation, recovering log and bundle head; +the current time is 11.180 seconds beyond the 30-second claim expiry. This +identifies a recovery-claim lifecycle failure, independently of the original +append failure and intermittent maintenance CI failure. + +The managed recovery path now renews that exact claim during verification and +materialization, without changing its lifetime or permitting expired claims. +Renewal and terminal authority CAS are serialized; immutable terminal inventory +verification runs outside that lock. A real 33-second root-read regression test +fails before the fix and passes afterward. All 18 focused recovery tests, +80 node tests and strict runtime test lint pass in an isolated Linux snapshot. +The fixed native-VM diagnostic completes in 158.45 seconds, including restoring +all 2,000 exact roots and joining the runtime and owner. Both followers and the +provider also join with exit 0, with no OOM. The offline audit checks every one +of the 110,861 original ACKs against the restored row, exact receipt position, +runtime metadata and canonical stored outcome, with zero violations. It takes +207.87 seconds. The original capacity and HTTP-retry failures remain failures; +this diagnostic does not establish a performance improvement or a fresh HTTP +exact-retry pass. All original journals, hashes and failed attempts remain. + +The maintenance investigation also reproduces a distinct capture race: a real +same-boot heartbeat between two directory observations caused `Fenced` through +full-advertisement equality. Capture now retains the exact Open epoch, membership +and signed boot identity while reobserving monotone generation, activation, +coverage and lease progress. Its original heartbeat regression fails before the +change and passes after it; a held-read closure case still rejects. The original +CI failures and 50 passing local repeats remain retained. Hosted workspace run +38181130233 subsequently passed, including the public controller models. This +concrete fix and passing run do not establish that the heartbeat race caused +the original hosted failures. + +An isolated controller repeat passed 383 of 384 cases. The remaining failure +was fixture construction: separate clock reads could sign a 30,001-ms +advertisement against the 30,000-ms maximum. Deriving both timestamps from one +reading preserves the lifetime contract; all three inherited-recovery cases +then pass with their original assertions. + +The device-backed sparse-owner process test also passes against fresh ext4 +SQLite and RustFS volumes. After the child acknowledges one mutation, the test +kills its process, deletes its local writer files, restores the authoritative +root, and retries the original request with a handler that fails if invoked. +The retry returns the original result and sequence; the next mutation verifies +the recovered state. This qualifies that published-root process-loss path, +not unpublished follower tails, network partitions or physical power loss. +The initial missing-import build failure and corrected passing run are both +retained under the v9 artifact directory. + +Two further ignored integration cases passed against fresh ext4 follower, +SQLite and RustFS volumes. The exact two-follower ensemble is authoritative +before the child can acknowledge. The first case holds the original data PUT, +verifies a follower-based success while the authoritative root remains at +sequence zero, kills the child and removes its local writer files. Recovery +reopens both native follower logs, verifies their exact retained frame, restores +the acknowledged state and returns the original retry without running its +mutation again. + +The second case forwards the child's real RustFS traffic through a test-owned +TCP proxy, publishes one mutation, then closes existing connections and rejects +new connections. The next mutation receives no success before its client +deadline. Its captured state is nevertheless present on both follower disks; +recovery retains that uncertain mutation as well as the earlier ACK. Both +original request IDs return their stored outcomes without reexecution. The +proxy recorded 49 rejected or closed connections in the passing run. + +The native followers share the killed process and use separate device-backed +directories. These cases qualify process loss, native follower persistence and +an owner-to-object-store TCP partition, not remote peer transport, follower +network partitions or power loss. Initial setup and incorrect uncertain-tail +expectation failures remain retained alongside the passing `device-crash-r5` +logs. Strict runtime test Clippy and the isolated shared-store integration suite +pass: 246 cases, with six environment-dependent cases ignored. + +### Fresh 750-write/s checkpoint-overlap run + +Source `da8887d`, five-minute offered window, 2,000 Cells, unchanged 64-MiB +budget and the same client/auditor binaries: + +| Measurement | Result | +| --- | --- | +| Offered writes | 225,000 | +| Successful writes within the window | 210,347; 701.16/s | +| Returned errors / dropped offers | 11,915 / 2,731 | +| Scheduled p50 / p95 / p99 | 9.7 / 50.1 / 318.4 ms | +| Warmup errors / drops | 0 / 42 | +| Every-ACK warm and cold readback and original retries | All 234,813; zero errors | +| Joined fleet drain / cold startup | 108.56 / 29.75 seconds | +| Final three-minute debt / age slopes | +866,204 bytes/s / +393 ms/s; failed | +| Pipeline checkpoint / headroom fallbacks | 0 / 3,579 | + +The case completed its audits but failed the capacity gate. Host samples inside +the measured window show competing compiler activity in 19 of 20 observations, +averaging 349% sampled CPU and peaking at 696%. It does not establish idle-host +capacity or isolate a causal throughput effect of the candidate. Retain this +failed attempt and repeat on an idle host before advancing the ramp. + +Within this window, materialization averaged 44.17 ms and checkpoint waiting +93.83 ms. Preparation, upload and selection averaged 20.62, 20.31 and 12.34 ms. +The publisher retains its original 22-MiB scratch/cache reservation; sampled +root I/O concurrency falls from two to one as retained selection metadata and +captures grow. These observations identify admission and root work to investigate; +they do not justify changing the budget or weakening the debt gate. + +Artifacts: `cellule-checkpoint-overlap-20261011-v9`, case +`cellule-fleet-parity-checkpoint-overlap-v9-750-r1`, with +`capacity-750-derived.json`, `capacity-750-host-window.json` and the case's +`publication-analysis.json`. The 1,000/1,500/2,000 ramp and fresh read/mixed +qualification remain outstanding. + +### Removing a duplicate LTX scan + +A diagnostic captured an actual SQLite mutation and encoded 64 distinct Cell +rows from the same pinned 203-byte LTX segment. The in-memory CRB1 constructor +verified each segment before copying it, then verified it again through the +external decoder. Temporary counters in an isolated snapshot measured 128 full +segment validations per encoding. The candidate performs 64, retaining complete +footer/layout checks and exact comparisons with the original verified rows. +Its independent decoder still performs all 64 validations. The complete +40,144-byte envelope and digest match exactly before and after. + +Eight debug-build samples of 100 encodings had median durations of 1,041.49 ms +before and 636.36 ms after. These shared-host component timings do not establish +application throughput or a causal fleet speedup. Instrumentation was removed +before verification: the full LTX suite passed 323 tests/doctests with seven +ignored, the runtime suite passed 1,316 with thirteen ignored, local-only LTX +passed 66, and strict workspace Clippy passed. Existing origin verification, +formats and resource bounds remain intact. Artifacts are in the v9 +`bundle-encode-diagnostic` directory, including the pinned input, raw samples, +counter comparison and contract results. + +### Requested three-node resource profile + +The `three-node-1cpu` profile uses one owner and two required followers, each +limited to 1 vCPU and 2 GiB memory with swap disabled. Their guest CPU sets are +0, 1 and 2; RustFS occupies 3–4 and the client 5–8. A live Docker probe confirmed +the configured limits and effective cgroup quota, memory and zero-swap limits +for all five containers. This establishes resource enforcement, not application +capacity. The original queued 8-CPU-owner repeat was stopped before startup in +response to this resource request; its deferral and observations remain retained. + +The preliminary 2,000-Cell diagnostic used 30 seconds warmup and 60-second +measurement windows. At 100 offered writes/s plus 1,000 reads/s, it delivered +99.20 writes/s and 992.18 reads/s, with zero returned errors but 47 write drops +and 468 read drops; one read also dropped during warmup. Read-only offered +1,000/s delivered 999.98/s with zero errors or drops. All 10,954 acknowledged +mutations passed warm readback and exact retry, but the owner exceeded the +original 120-second drain deadline and the cold audit did not run. The case +failed; these short, busy-host measurements do not qualify capacity. +Artifacts are in `cellule-three-small-nodes-20261011-v10`, including +`setup-verification-result.json`, raw windows and failed lifecycle evidence. + +The user clarified that the required topology is **three separate VMs**. The +`three-vm-1cpu` profile routes each node to its own Docker host and retains +fresh node volumes on those VMs' disks. Hypervisor configurations allocate +one vCPU and 2 GiB each; distinct guest boot IDs and Docker host identities, +usable memory, disabled guest/container swap and OOM counters are recorded. +RustFS and the client run in a separate 6-vCPU, 8-GiB support VM. Loopback TCP +routes use SSH tunnels while retaining end-to-end pinned peer mTLS; their +transport overhead and shared Mac host belong to the measurement scope. +The 2,000-Cell, 64-MiB, five-minute capacity run still requires an idle host, +every-ACK warm/cold audits, stable debt and the original drain deadline. +New topology artifacts are in `cellule-three-vms-20261011-v11`. + +A fresh 16-Cell smoke run completed on the separate VMs. Each window used +30 seconds warmup and 60 seconds measurement, with the unchanged 96-byte value +and 64-MiB budget: + +| Window | Delivered writes/s | Delivered reads/s | Returned errors / drops | Scheduled p99, write / read | +| --- | ---: | ---: | --- | --- | +| 10 offered writes/s + 100 reads/s | 10.00 | 100.00 | 0 / 0 | 2,058.6 / 1,885.5 ms | +| Read-only, 100 offered reads/s | 0 | 100.00 | 0 / 0 | — / 44.7 ms | + +All 917 ACKs passed warm and cold readback and exact retry. Fleet drain took +8.28 seconds; fresh bucket-only startup took 1.10 seconds. No guest or container +OOM occurred. Mixed response-source deltas were 134 follower proofs and 466 +bundle proofs; each follower recorded four failed append attempts and its log +reported `Fenced`. These observations do not identify the earlier CI failure. +Host samples showed competing work, with mixed-window CPU idle values of +38.81%, 38.34%, 0% and 61.90%. The latency target failed, and a 16-Cell smoke +cannot qualify 2,000-Cell capacity. Clean shutdown and cold bucket restore also +do not substitute for remote-VM crash recovery of an unpublished follower tail. +Exact resource identities, metrics, hashes and raw evidence are retained in +`three-vm-smoke-results.json` and `cellule-fleet-parity-three-vms-smoke-r1`. +The fresh 2,000-Cell, five-minute 750-write/s case waits for the original idle-host +start check; higher ramp points remain pending. + +A separate 120-second diagnostic on the same three VMs traced all 41 rejected +append attempts to grant expiry **after native fsync**. The lane remained open +and the token digest matched; the original post-write check refused the proof. +Failed attempts spent 22.2–872.0 ms in fsync. Error-only instrumentation retained +the original errors and authority checks. The window delivered 9.88 writes/s +and 98.81 reads/s, with zero returned errors but 14 write and 143 read drops; +scheduled write/read p99 was 3,363.3/3,011.9 ms. All 1,503 ACKs passed warm/cold +readback and exact retry; drain took 18.10 seconds. This busy-host diagnostic +does not qualify capacity. Evidence is in `cellule-grant-fenced-20261011-v12`. + +A one-second early-renewal prototype then failed for a separate reason: a signed +grant request exceeded the follower's unchanged ten-second horizon by 1 ms. +The owner entered rotation and its Cells stopped serving. All 1,200 measured +write and 12,000 read attempts failed; the warm audit could not read the 140 +earlier ACKs, and the cold audit did not run. Owner drain exceeded 120 seconds +and cleanup stopped it; followers stopped normally, with no container OOM. +That failed early-renewal prototype was not promoted. +`cellule-grant-renewal-20261011-v13` retains the failed attempt and exact deadline +samples. These append/grant observations do not establish the cause of the +earlier maintenance CI failures, whose fixture uses local follower transport. + +The client now leaves one second below the receiver's existing ten-second +horizon; earlier grant deadlines still take precedence. Five signed-transport +tests passed, including the reproduced 1-ms lag, the headroom boundary, expired +grant deadlines and overflow. A fresh 120-second run with this deadline change +and the original grant reuse completed: 9.46 writes/s and 94.58 reads/s, zero +returned errors, 65 write drops and 650 read drops. Scheduled write/read p99 was +6,866.7/6,842.5 ms. All 1,452 ACKs passed warm/cold readback and exact retry; +drain took 10.27 seconds. There were 39 post-fsync grant-expiry rejections, with +failed fsyncs spanning 15.3–1,469.7 ms. The deadline fix preserves serving across +the observed clock difference; this busy-host diagnostic still fails delivery +and latency qualification. Evidence is in `cellule-request-deadline-20261011-v14`. + +The committed revision `d821b6e` was then rebuilt without diagnostic patches and +verified on the three separate VMs. Both windows used 16 Cells, 96-byte values, +the unchanged 64-MiB budget, 30 seconds warmup and 120 seconds measurement: + +| Window / request | Offered/s | Delivered/s | Scheduled p50 / p95 / p99 | Errors / drops | +| --- | ---: | ---: | --- | --- | +| Mixed writes | 10 | 9.04 | 34.3 / 2,650.2 / 10,284.6 ms | 0 / 115 | +| Mixed reads | 100 | 90.51 | 4.9 / 2,313.0 / 10,217.1 ms | 0 / 1,139 | +| Read-only | 100 | 100.00 | 4.7 / 13.1 / 53.7 ms | 0 / 0 | + +The mixed scheduled histogram overflowed for 13 write and 127 read samples, +so its table percentiles use nearest-rank quantiles from all attempted measured +request journals. Dropped offers have no latency sample; the original driver's +overflow and failed qualification are retained. All 1,402 ACKs passed warm/cold +readback and exact retry. Drain took 15.78 seconds; cold startup took 2.18 seconds. +All node processes stopped normally; guest swap and OOM counters stayed zero. +Mixed response-source deltas were 86 follower proofs and 999 bundle proofs; +the followers recorded 13 and 12 failed appends. Their failures grant no Fleet +credit. Start, minute and end samples in both windows showed active Fleet in +epoch 1 without fencing or rotation. + +Host CPU idle ranged from 3.16–52.17% in mixed samples and 0.17–66.64% in +read-only samples. This verifies functional behavior at the stated workload; +it does not qualify capacity. `cellule-three-vms-current-20261011-v15` retains +the source/binary identity, raw windows, `three-vm-current-results.json` and +`three-vm-current-results.md`. The waiting high-rate run now uses this committed +build, the original 2,000-Cell/five-minute/64-MiB profile, unchanged timed driver +and previously verified setup-only pacing. Earlier waiting queues were joined +before any high-rate case was admitted. + +After the deadline fix, a fresh uninstrumented early-renewal probe used the +same three VMs, 16 Cells and 120-second mixed window. The client renews cached +grants when one second or less remains. Signed five-second expiry, native +pre/post-fsync authority checks and the bounded exact-frame retry stay unchanged. +Both followers recorded zero failed appends and neither peer log contained +`Fenced`, compared with 13 and 12 measured failures in the preceding separate +run. The renewal change is now applied to the SQL example transport; it leaves +time for transport and fsync but cannot guarantee an append finishes before +expiry. + +The probe delivered 9.51 writes/s and 95.18 reads/s, with zero returned errors +and 58 write / 578 read drops. Scheduled write/read p99 was 4,897.4/4,747.7 ms. +One write completed after the window and is excluded from delivered throughput. +All 1,459 ACKs passed warm/cold readback and exact retry; joined drain took +11.01 seconds and cold startup took 2.24 seconds. Node processes stopped +normally, guest OOM/swap counters remained zero, and boot IDs stayed stable. +Every mixed-window host sample reported 0% CPU idle. These separate busy-host +runs do not establish a throughput gain; dropped offers and seconds-long tails +still fail capacity and latency qualification. The explicit patch, source and +binary hashes, raw windows and `three-vm-headroom-results.json` are retained in +`cellule-grant-headroom-20261011-v16`. A fresh committed build and the unchanged +idle-host ramp remain required. + +The committed early-renewal revision `a0860db` was rebuilt without overlays and +completed both fresh 120-second windows on the same three VMs: + +| Window / request | Offered/s | Delivered/s | Scheduled p50 / p95 / p99 | Errors / drops | +| --- | ---: | ---: | --- | --- | +| Mixed writes | 10 | 9.98 | 47.8 / 1,070.8 / 1,870.8 ms | 0 / 2 | +| Mixed reads | 100 | 99.77 | 6.2 / 796.4 / 1,666.0 ms | 0 / 28 | +| Read-only | 100 | 100.00 | 5.1 / 12.5 / 34.6 ms | 0 / 0 | + +All 1,515 ACKs passed warm/cold readback and exact retries. Joined fleet drain +took 10.81 seconds; cold startup took 3.28 seconds. Both followers recorded +zero measured append failures. The host remained busy: mixed CPU idle samples +ranged from 0–60.23%, and read-only samples from 0–50.30%. Dropped offers still +fail capacity qualification. These rates describe the stated offered workloads; +they are not maximum write/read capacity. Source/binary identity, guest resource +observations and raw evidence are in `cellule-grant-current-20261011-v18`. + +Following the user's regression concern, the high-rate candidate queue is held +pending a baseline comparison. `origin/main` at `c79834d` is being rebuilt with +only the proven request-deadline correction common to both builds. Baseline +grant reuse remains unchanged. The comparison alternates baseline/candidate +order on identical VM limits, workloads and budgets, retaining every ACK audit +and host resource sample. It includes the original 2,000-Cell, five-minute, +750-write/s profile. Performance changes that fail to improve the controlled +throughput/tail-latency comparison must be reverted; correctness fixes and +recovery evidence remain required. No paired result or rollback decision exists +yet. The fresh idle-host comparison is retained in +`cellule-origin-comparison-20261011-v19`. + +The four ext4-backed process/partition cases were also repeated with fresh +volumes after a prior cleanup observation was interrupted. All four passed in +51.36 seconds; the repeat recorded both provider and worker stopped without +OOM. This is device-backed process evidence, separate from remote-node VM +failure or power-loss qualification. Both attempts remain retained. + +## Retained failed attempts + +| Attempt | Outcome | +| --- | --- | +| Working admission v2, 500/s for five minutes | Zero returned errors but 144 drops; root age grew; not accepted | +| Ordered frame reads v3, two five-minute repeats | Errors/drops remained under competing compiler load; both completed all-ACK warm/cold audits and drained within 120 seconds | +| Working admission v5, 500/s for five minutes | 482.84/s; 3,746 errors and 1,398 drops; growing root age; all 161,857 ACKs passed warm/cold checks and drain took 51.92 seconds | +| Earlier profile/provider filesystem failure | Guest-wide I/O errors before timed work; no throughput result; stopped disks and failure artifacts preserved | + +No budget, qualification profile, evidence requirement, fault assertion or +deadline was increased to make these failures pass. Correct readback of ACKs +and sustainable offered throughput are separate requirements. diff --git a/docs/remote-follower-tail-verification.md b/docs/remote-follower-tail-verification.md new file mode 100644 index 00000000..c2809855 --- /dev/null +++ b/docs/remote-follower-tail-verification.md @@ -0,0 +1,100 @@ +# Remote unpublished-tail verification + +This fixture checks a follower-acknowledged write after owner process loss or +a follower TCP partition, using the SQL example's original signed mTLS transport +and the public recovery +coordinator. It requires three separate Linux VMs and a support Docker host. +It is an environment-dependent fault test, not a throughput benchmark. + +| Requirement | Verification | +| --- | --- | +| Three separate node VMs | Distinct Docker host IDs and kernel boot IDs | +| Each node has 1 vCPU and 2 GiB | Host inventory and container cgroup limits | +| Native durable storage | Fresh ext4 Docker volumes for SQLite and follower logs | +| Remote durability before publication | Both followers acknowledge; Cell root and selected bundle coverage remain zero | +| Owner process loss | SIGKILL after the ACK; successor uses a different fresh volume | +| Recovery authority | Actual predecessor lease expiry, fencing CAS, remote sealing and complete tail recovery | +| Exact retry | Original result and commit sequence; retry handler fails if executed | +| Partitioned output | After blocking both follower paths, a captured second mutation and its observed read cannot return success | +| Missing recovery witness | Recovery refuses both unreachable followers before installing an overlay or taking over the Cell | +| One complete witness | Reaching one original sealed follower recovers the first ACK while the second path remains blocked | +| Joined resources | Successor and followers exit successfully; all owned containers become terminal without OOM | + +The support host runs the pinned RustFS provider. Configure loopback routes +between these hosts before running: every node must reach the provider at +`127.0.0.1:9000`, and the owner must reach the two followers at +`127.0.0.1:8090` and `127.0.0.1:8100`. Forward raw TCP; the application keeps +mTLS and signed peer authorization end to end. The support host needs two +available CPUs for its provider, and node VMs must have swap disabled. + +Build the SQL example's **test executable** for the node VMs' Linux architecture +in an isolated source snapshot. Use the pinned Rust image from +[the capacity driver](../scripts/perf/run.py), with `CARGO_TARGET_DIR` under the +mounted Workspace target directory. The build command is: + +```sh +cargo test --release --locked -p cellule-axum --example sql --no-run --message-format=json +``` + +Use the `compiler-artifact` executable for target `sql` with `profile.test=true`, +rather than the ordinary SQL server executable. The driver checks that it +contains the exact ignored `fleet::remote_fault_tests::remote_crash_fixture_child` +test. Supply that executable and the Docker contexts: + +```sh +python3 scripts/verify-remote-follower-tail.py \ + --artifacts "$HOME/Workspace/crabbuild-target/cellule-remote-tail" \ + --binary /absolute/path/to/linux-sql-test-executable \ + --context support-context \ + --node-contexts owner-context follower-1-context follower-2-context \ + --tag remote-tail-r1 +``` + +Use a new tag for each case. The driver creates private fixture certificates, +an isolated object prefix, and fresh provider/node volumes. It retains lifecycle +records, logs, native-storage/resource observations, and the recovered result +outside the source checkout. Completed containers and volumes remain available +for inspection. Run it separately from performance measurements on these hosts. + +For the partition case: + +```sh +python3 scripts/verify-remote-follower-tail.py \ + --artifacts "$HOME/Workspace/crabbuild-target/cellule-remote-tail" \ + --binary /absolute/path/to/linux-sql-test-executable \ + --context support-context \ + --node-contexts owner-context follower-1-context follower-2-context \ + --tag peer-partition-r1 \ + --fault peer-partition \ + --owner-ssh-config /absolute/path/to/owner-vm/ssh.config \ + --owner-ssh-host owner-vm-alias +``` + +The SSH account needs passwordless access to `sudo iptables`. The driver checks +that SSH and Docker reach the same owner kernel, then installs two uniquely +tagged OUTPUT rules for the follower TCP ports. Packet counters must show actual +drops on both paths. It restores one path for recovery, verifies the original +ACK and exact retry while the other remains blocked, then heals that path before +joining the successor and followers. Cleanup removes only its own rules, and +stops the provider after all node roles become terminal. All markers are fsynced +on the roles' native volumes. + +Two fresh cases passed with one unpublished ACK per case on the three-VM +profile. The original failed fixture invocation is retained: successor +enrollment occurred before predecessor lease expiry, exceeding the example's +three-live-session bound. The corrected driver waits for actual expiry without +changing that bound or spoofing time. See +[the measurement record](publication-progress-measurements.md). + +Two corrected peer-partition cases also pass with one unpublished ACK each. +Earlier failed cases remain retained, including a pre-partition grant rejection +caused by receiver issue times ahead of the sender's clock. See the same +measurement record for the diagnosis, correction, native packet counters and +per-case limits. + +This checks process loss and, when selected, the two follower TCP paths. It does +not simulate physical power loss or partition the object-store path. +The existing [device-backed fault tests](../crates/cellule-runtime/tests/runtime/lifecycle/ownership/follower_process.rs) +cover separate object-store TCP-partition cases. Neither fixture establishes +capacity, latency improvement, or the cause of the original intermittent +maintenance `Fenced` failures. diff --git a/scripts/bench-axum-rustfs.py b/scripts/bench-axum-rustfs.py index a76563d0..8d634218 100644 --- a/scripts/bench-axum-rustfs.py +++ b/scripts/bench-axum-rustfs.py @@ -294,8 +294,8 @@ def steady_reads(args, directory, service, concurrency, acknowledged): def verify_active_owner_refused(binary, directory, prefix, cells, workers): env = os.environ.copy() - # Exercise the same Cell authority refusal without trying to enroll another - # boot for the fixture's already-live physical leader identity. + # Without directory enrollment this boot cannot prove takeover of the + # fixture's active predecessor. Check that precise startup refusal. env.pop("CELLULE_AXUM_FLEET_DIR", None) env.pop("CELLULE_AXUM_FOLLOWER", None) env.update(CELLULE_TEST_PREFIX=prefix, CELLULE_AXUM_BIND="127.0.0.1:0", @@ -304,7 +304,7 @@ def verify_active_owner_refused(binary, directory, prefix, cells, workers): with log.open("w") as output: candidate = subprocess.run([str(binary)], env=env, stdout=output, stderr=subprocess.STDOUT, timeout=120) require(candidate.returncode != 0, "second service took an active Cell") - require("idle acquisition requires a published idle control" in log.read_text(), "second service failed for an unexpected reason") + require('Node("crash takeover requires real directory enrollment")' in log.read_text(), "second service failed for an unexpected reason") def point(args, directory, repeat, concurrency, cells): diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 4d4158fa..b8666121 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -61,6 +61,51 @@ separate profile: do not compare it as an algorithm change against historical shared-VM runs. Guest CPU placement does not isolate host services or qualify physical-device durability; report those limits separately. +`--resource-profile three-node-1cpu` runs one owner and two required followers, +each with 1 CPU, 2 GiB memory and no swap. They occupy guest CPUs 0, 1 and 2. +RustFS uses CPUs 3–4 with a 2-CPU/2-GiB limit; the client uses CPUs 5–8 with +a 4-CPU/2-GiB limit. Each application node uses one SQLite worker and one +Tokio worker. This profile requires at least nine guest CPUs and 12 GiB usable +memory and saves the same Docker and effective cgroup checks. It keeps the +64-MiB retained-work budget and all delivery, debt and ACK audit requirements. +These are three capped containers in one Linux VM; host failures remain shared. +Record this profile separately from either 8-CPU owner profile. Node state is +still tmpfs, so device-backed crash tests remain a separate verification route. + +```sh +python3 scripts/perf/run.py cellule-fleet --context dedicated-linux \ + --artifacts /absolute/external/candidate --resource-profile three-node-1cpu \ + --tag three-small-nodes --cells 2000 --seconds 300 --warmup 30 \ + --repetitions 1 --write-rates 750 1000 1500 2000 +``` + +`--resource-profile three-vm-1cpu --node-contexts OWNER FOLLOWER1 FOLLOWER2` +instead places each node on a separate Docker host. Each node VM must have +one guest CPU and a 2-GiB configured memory limit; the runner checks one CPU, +usable RAM between 1.75 and 2 GiB, and three distinct Docker host identities. +Record hypervisor configuration separately to verify the total VM allocation. +Node containers use CPU 0, a 2-GiB ceiling and no swap; disable guest swap as +well. This profile retains a fresh disk-backed node volume, rather than tmpfs, +and records its filesystem. Node volumes and provider volumes remain retained. + +The separate support context needs at least six CPUs and 7 GiB usable RAM: +RustFS uses CPUs 0–1 and the client 2–5, with the same 2-GiB container limits. +All nodes must reach the provider at loopback port 9000. The owner and client +must reach follower ports 8090 and 8100, and the client must reach owner port +8080. This fixture keeps loopback-only application endpoints; prepare and record +those cross-VM routes before running. TCP tunnels retain end-to-end pinned mTLS +but add transport overhead that belongs in the reported configuration. Capture +guest memory, swap, boot identity and OOM evidence alongside host contention. +Existing delivery, debt, audit and 120-second drain requirements still apply. + +```sh +python3 scripts/perf/run.py cellule-fleet --context separate-support \ + --artifacts /absolute/external/candidate --resource-profile three-vm-1cpu \ + --node-contexts node-owner node-follower-1 node-follower-2 \ + --tag three-vms --cells 2000 --seconds 300 --warmup 30 \ + --repetitions 1 --write-rates 750 1000 1500 2000 +``` + The example authority's renewal and publication regressions use a generated TLS identity and an in-memory directory. From an isolated verification checkout, run them explicitly with the TLS directory produced by diff --git a/scripts/perf/run.py b/scripts/perf/run.py index 5d970251..77eac231 100644 --- a/scripts/perf/run.py +++ b/scripts/perf/run.py @@ -11,6 +11,7 @@ ARGS = None MANIFEST = None DOCKER_HOST = None +NODE_CONTEXTS = {} RUNNER_BYTES = Path(__file__).read_bytes() RUNNER_SHA256 = hashlib.sha256(RUNNER_BYTES).hexdigest() CREDS = ['-e', 'AWS_ACCESS_KEY_ID=benchmark_access', '-e', 'AWS_SECRET_ACCESS_KEY=benchmark_secret_private', '-e', 'AWS_DEFAULT_REGION=us-east-1'] @@ -18,10 +19,27 @@ def isolated_owner(): return getattr(ARGS, 'resource_profile', 'shared-vm') == 'isolated-owner' -def resource_limits(role): +def small_nodes(): + return getattr(ARGS, 'resource_profile', 'shared-vm') in ('three-node-1cpu', 'three-vm-1cpu') + +def separate_vms(): + return getattr(ARGS, 'resource_profile', 'shared-vm') == 'three-vm-1cpu' + +def resource_limits(role, node_index=None): if role not in ('owner', 'follower', 'provider', 'client'): raise ValueError('unknown benchmark resource role') - if isolated_owner(): + if separate_vms(): + cpus, gib = {'owner': (1, 2), 'follower': (1, 2), + 'provider': (2, 2), 'client': (4, 2)}[role] + cpuset = {'owner': '0', 'follower': '0', 'provider': '0-1', 'client': '2-5'}[role] + elif small_nodes(): + cpus, gib = {'owner': (1, 2), 'follower': (1, 2), + 'provider': (2, 2), 'client': (4, 2)}[role] + if role == 'follower' and node_index not in (1, 2): + raise ValueError('small-node follower requires its placement index') + cpuset = {'owner': '0', 'follower': str(node_index), + 'provider': '3-4', 'client': '5-8'}[role] + elif isolated_owner(): cpus, gib = {'owner': (8, 16), 'follower': (1, 2), 'provider': (2, 2), 'client': (4, 2)}[role] cpuset = '0-7' if role == 'owner' else '8-11' @@ -31,10 +49,10 @@ def resource_limits(role): cpuset = None return cpus, gib, cpuset -def resource_args(role): - cpus, gib, cpuset = resource_limits(role) +def resource_args(role, node_index=None): + cpus, gib, cpuset = resource_limits(role, node_index) args = ['--cpus', str(cpus), '--memory', f'{gib}g'] - if isolated_owner() or role != 'client': + if isolated_owner() or small_nodes() or role != 'client': args += ['--memory-swap', f'{gib}g'] if cpuset: args += ['--cpuset-cpus', cpuset] @@ -43,11 +61,20 @@ def resource_args(role): def validate_resource_host(info): if isolated_owner() and (info['NCPU'] < 12 or info['MemTotal'] < 23 * 1024**3): raise ValueError('isolated owner requires at least 12 guest CPUs and 23 GiB usable memory') + if separate_vms() and (info['NCPU'] < 6 or info['MemTotal'] < 7 * 1024**3): + raise ValueError('separate support VM requires at least 6 guest CPUs and 7 GiB usable memory') + if small_nodes() and not separate_vms() and (info['NCPU'] < 9 or info['MemTotal'] < 12 * 1024**3): + raise ValueError('three small nodes and separate support roles require at least 9 guest CPUs and 12 GiB usable memory') def verify_resource_role(name, role): - if not isolated_owner(): + if not isolated_owner() and not small_nodes(): return None - cpus, gib, cpuset = resource_limits(role) + node_index = None + if role == 'follower': + if not name.endswith(('-peer-1', '-peer-2')): + raise ValueError('follower name lacks its placement index') + node_index = int(name[-1]) + cpus, gib, cpuset = resource_limits(role, node_index) config = json.loads(docker('inspect', name).stdout)[0]['HostConfig'] expected = {'NanoCpus': cpus * 10**9, 'Memory': gib * 1024**3, 'MemorySwap': gib * 1024**3, 'CpusetCpus': cpuset} @@ -89,8 +116,16 @@ def provider_filesystem(): result = parse_provider_filesystem(docker('exec', STORE, 'sh', '-c', 'df -Pk /data && df -Pi /data').stdout) return dict(result, timestamp=time.time()) -def docker(*args, check=True, timeout=700): - p = subprocess.run(DOCKER + list(map(str, args)), capture_output=True, text=True, timeout=timeout) +def docker(*args, check=True, timeout=700, context=None): + if context is None: + name = None + if args[0] == 'run' and '--name' in args: + name = args[args.index('--name') + 1] + elif args[0] in ('exec', 'inspect', 'logs', 'kill', 'wait', 'rm', 'stop', 'restart'): + name = next((value for value in args[1:] if value in NODE_CONTEXTS), None) + context = NODE_CONTEXTS.get(name, CTX) + command = ['docker', '--context', context] if context is not None else DOCKER + p = subprocess.run(command + list(map(str, args)), capture_output=True, text=True, timeout=timeout) if check and p.returncode: raise RuntimeError(f'docker {args[0]} failed: {p.stdout} {p.stderr}') return p @@ -197,8 +232,12 @@ def snapshot(directory, label, names): data[name] = {'inspect': json.loads(docker('inspect', name).stdout)[0]} role = 'provider' if name == STORE else 'client' if name == CONTROL else 'follower' if name.endswith(('peer-1', 'peer-2')) else 'owner' data[name]['resource_placement'] = verify_resource_role(name, role) - p = docker('exec', name, 'sh', '-c', 'cat /proc/1/limits; cat /proc/1/status; ls /proc/1/fd | wc -l; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/cpu.stat', check=False) + p = docker('exec', name, 'sh', '-c', 'cat /proc/1/limits; cat /proc/1/status; ls /proc/1/fd | wc -l; cat /sys/fs/cgroup/memory.events; cat /sys/fs/cgroup/memory.current; cat /sys/fs/cgroup/memory.peak; cat /sys/fs/cgroup/cpu.stat', check=False) data[name]['process'] = p.stdout + p.stderr + if name in NODE_CONTEXTS: + guest = docker('exec', name, 'sh', '-c', 'cat /proc/sys/kernel/random/boot_id; cat /proc/meminfo; cat /proc/swaps; awk \'$1 == "oom_kill" {print}\' /proc/vmstat') + data[name]['guest'] = guest.stdout + data[name]['docker_context'] = NODE_CONTEXTS[name] state = data[name]['inspect']['State'] if state['OOMKilled'] or not state['Running']: failures.append(f'{name}: {state}') @@ -215,7 +254,10 @@ def sampler(directory, stop): with (directory / 'docker-stats.jsonl').open('w') as f: while not stop.is_set(): try: - p = docker('stats', '--no-stream', '--format', '{{json .}}', check=False, timeout=30) + stats = [] + for context in dict.fromkeys([CTX, *NODE_CONTEXTS.values()]): + p = docker('stats', '--no-stream', '--format', '{{json .}}', check=True, timeout=30, context=context) + stats.extend(dict(json.loads(line), DockerContext=context) for line in p.stdout.splitlines() if line) except Exception as error: f.write(json.dumps({'timestamp': time.time(), 'error': str(error)}) + '\n') f.flush() @@ -224,15 +266,27 @@ def sampler(directory, stop): filesystem = provider_filesystem() except Exception as error: filesystem = {'error': str(error), 'measurement_usable': False} - f.write(json.dumps({'timestamp': time.time(), 'provider_filesystem': filesystem, 'stats': [json.loads(x) for x in p.stdout.splitlines() if x]}) + '\n') + f.write(json.dumps({'timestamp': time.time(), 'provider_filesystem': filesystem, 'stats': stats}) + '\n') f.flush() stop.wait(3) def start_node(system, durability, index, prefix, name): port = 8080 + index * 10 - args = ['run', '-d', '--name', name, '--network', 'host', *resource_args('follower' if index else 'owner'), '--ulimit', 'nofile=65536:65536', '--tmpfs', '/scratch:rw,size=4g', *CREDS] + workers = 1 if small_nodes() else 8 + if separate_vms(): + NODE_CONTEXTS[name] = ARGS.node_contexts[index] + volume = name + '-state' + if docker('volume', 'inspect', volume, check=False, context=NODE_CONTEXTS[name]).returncode == 0: + raise RuntimeError('node state volume is not fresh: ' + volume) + docker('volume', 'create', '--label', 'cellule.perf.artifacts=' + str(BASE), volume, context=NODE_CONTEXTS[name]) + storage = ['-v', volume + ':/scratch'] + else: + storage = ['--tmpfs', '/scratch:rw,size=4g'] + args = ['run', '-d', '--name', name, '--network', 'host', *resource_args('follower' if index else 'owner', index), '--ulimit', 'nofile=65536:65536', *storage, *CREDS] + if small_nodes(): + args += ['-e', 'TOKIO_WORKER_THREADS=1'] if system == 'cellule': - args += ['-e', f'PARITY_RETAINED_BYTES={ARGS.retained_bytes}', '-e', f'PARITY_DISK_BYTES={ARGS.disk_bytes}', '-v', f'{BASE}:/work:ro', '-e', 'TMPDIR=/scratch', '-e', f'CELLULE_AXUM_CELLS={ARGS.cells}', '-e', 'CELLULE_AXUM_WORKERS=8', '-e', f'CELLULE_AXUM_BIND=127.0.0.1:{port}', '-e', 'CELLULE_TEST_ENDPOINT=http://127.0.0.1:9000', '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', f'CELLULE_TEST_PREFIX={prefix}'] + args += ['-e', f'PARITY_RETAINED_BYTES={ARGS.retained_bytes}', '-e', f'PARITY_DISK_BYTES={ARGS.disk_bytes}', '-v', f'{BASE}:/work:ro', '-e', 'TMPDIR=/scratch', '-e', f'CELLULE_AXUM_CELLS={ARGS.cells}', '-e', f'CELLULE_AXUM_WORKERS={workers}', '-e', f'CELLULE_AXUM_BIND=127.0.0.1:{port}', '-e', 'CELLULE_TEST_ENDPOINT=http://127.0.0.1:9000', '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', f'CELLULE_TEST_PREFIX={prefix}'] if durability == 'fleet' or index == 0: args += ['-e', 'CELLULE_AXUM_FLEET_DIR=/scratch/fleet'] if durability == 'bucket' and index == 0: @@ -241,9 +295,13 @@ def start_node(system, durability, index, prefix, name): args += ['-e', f'CELLULE_AXUM_FOLLOWER={index}'] args += [RUST, 'sh', '-c', f"mkdir -m 700 /scratch/fleet && cp /work/tls/ca.crt /work/tls/node-{index}.crt /work/tls/node-{index}.key /scratch/fleet/ && exec /work/{MANIFEST['binaries']['sql']['path']}"] else: - args += ['-e', f'CELLD_PLACEMENT_WEIGHT={(1000000000 if index == 0 else 1)}', '-e', 'CELLD_WATCH=/scratch', '-e', f'CELLD_DURABILITY={durability}', '-e', 'CELLD_SHUTDOWN_TOTAL_MS=600000', '-e', 'CELLD_TOKIO_THREADS=8', '-e', 'CELLD_REBALANCE_INTERVAL_MS=0', '-e', f"CELLD_NODE={name.replace('-cold', '-owner')}", '-e', 'RUST_LOG=warn,celld::node_log=info', CELLD, '--bucket', f's3://comparison/{prefix}', '--endpoint', 'http://127.0.0.1:9000', '--region', 'us-east-1', '--listen', f'127.0.0.1:{port}', '--internal-listen', f'127.0.0.1:{port + 1}', '--advertise', f'127.0.0.1:{port + 1}'] + args += ['-e', f'CELLD_PLACEMENT_WEIGHT={(1000000000 if index == 0 else 1)}', '-e', 'CELLD_WATCH=/scratch', '-e', f'CELLD_DURABILITY={durability}', '-e', 'CELLD_SHUTDOWN_TOTAL_MS=600000', '-e', f'CELLD_TOKIO_THREADS={workers}', '-e', 'CELLD_REBALANCE_INTERVAL_MS=0', '-e', f"CELLD_NODE={name.replace('-cold', '-owner')}", '-e', 'RUST_LOG=warn,celld::node_log=info', CELLD, '--bucket', f's3://comparison/{prefix}', '--endpoint', 'http://127.0.0.1:9000', '--region', 'us-east-1', '--listen', f'127.0.0.1:{port}', '--internal-listen', f'127.0.0.1:{port + 1}', '--advertise', f'127.0.0.1:{port + 1}'] docker(*args) verify_resource_role(name, 'follower' if index else 'owner') + if separate_vms(): + evidence = {'context': NODE_CONTEXTS[name], 'volume': volume, 'retained': True, + 'filesystem': docker('exec', name, 'sh', '-c', 'df -T /scratch; cat /proc/mounts').stdout} + put(BASE / (name + '-storage.json'), evidence) return wait_ready(name, 'Follower service:' if index else 'Orders service:') if system == 'cellule' else wait_ready(name, port=port) def stop_node(name, directory): @@ -332,8 +390,27 @@ def run_case(system, durability): directory.mkdir(exist_ok=False) (directory / 'runner.py').write_bytes(RUNNER_BYTES) prefix = label + '-' + uuid.uuid4().hex[:12] - put(directory / 'case.json', {'system': system, 'durability': durability, 'prefix': prefix, 'framework_commit': MANIFEST['framework_revision'] if system == 'cellule' else 'f2bf648663a610eefde71f3547ad61e9b896b1f0', 'resident_cells': ARGS.cells, 'concurrency': int(ARGS.concurrency), 'queue_capacity': int(ARGS.queue_capacity), 'candidate_binary': MANIFEST['binaries']['sql']['path'], 'celld_application': 'celld-app', 'retained_budget_bytes': int(ARGS.retained_bytes), 'managed_disk_budget_bytes': int(ARGS.disk_bytes), 'value_bytes': 96, 'owner_cpus': 8, 'owner_memory_bytes': 16 * 1024 ** 3, 'followers': 2 if durability == 'fleet' else 0, 'profile': 'isolated-owner-sql-ledger-96-v1' if isolated_owner() else 'shared-vm-sql-ledger-96', 'provider_storage': 'fresh Linux Docker volume', 'provider_filesystem_required': True, 'provider_lifecycle_required': True, 'telemetry': ARGS.telemetry, 'warmup_seconds': ARGS.warmup, 'seconds': ARGS.seconds, 'runner_sha256': RUNNER_SHA256, 'docker_host': DOCKER_HOST, 'diagnostic': ARGS.seconds < 300 or ARGS.warmup < 30 or ARGS.telemetry == 'off'}) + owner_cpus, owner_gib, _ = resource_limits('owner') + profile = ('three-vm-1cpu-sql-ledger-96-v1' if separate_vms() else + 'three-node-1cpu-sql-ledger-96-v1' if small_nodes() else + 'isolated-owner-sql-ledger-96-v1' if isolated_owner() else 'shared-vm-sql-ledger-96') + put(directory / 'case.json', {'system': system, 'durability': durability, 'prefix': prefix, 'framework_commit': MANIFEST['framework_revision'] if system == 'cellule' else 'f2bf648663a610eefde71f3547ad61e9b896b1f0', 'resident_cells': ARGS.cells, 'concurrency': int(ARGS.concurrency), 'queue_capacity': int(ARGS.queue_capacity), 'candidate_binary': MANIFEST['binaries']['sql']['path'], 'celld_application': 'celld-app', 'retained_budget_bytes': int(ARGS.retained_bytes), 'managed_disk_budget_bytes': int(ARGS.disk_bytes), 'value_bytes': 96, 'owner_cpus': owner_cpus, 'owner_memory_bytes': owner_gib * 1024 ** 3, 'followers': 2 if durability == 'fleet' else 0, 'profile': profile, 'provider_storage': 'fresh Linux Docker volume', 'provider_filesystem_required': True, 'provider_lifecycle_required': True, 'telemetry': ARGS.telemetry, 'warmup_seconds': ARGS.warmup, 'seconds': ARGS.seconds, 'runner_sha256': RUNNER_SHA256, 'docker_host': DOCKER_HOST, 'diagnostic': ARGS.seconds < 300 or ARGS.warmup < 30 or ARGS.telemetry == 'off'}) + if separate_vms(): + case = json.loads((directory / 'case.json').read_text()) + case['node_contexts'] = ARGS.node_contexts + case['node_storage'] = 'fresh retained Docker volume on each node VM disk' + put(directory / 'case.json', case) names = [] + if separate_vms(): + hosts = [] + for context in ARGS.node_contexts: + info = json.loads(docker('info', '--format', '{{json .}}', context=context).stdout) + if info['NCPU'] != 1 or not 1.75 * 1024**3 <= info['MemTotal'] <= 2 * 1024**3: + raise RuntimeError('node VM differs from 1-vCPU/2-GiB profile: ' + context) + hosts.append(dict(context=context, **{key: info[key] for key in ('ID', 'NCPU', 'MemTotal', 'Architecture', 'ServerVersion')})) + if len({host['ID'] for host in hosts}) != 3: + raise RuntimeError('three-node profile requires three distinct Docker hosts') + put(directory / 'node-hosts.json', hosts) stop = threading.Event() sampling = None summary = {'schema_version': 1, 'case': label, 'writes': [], 'reads': [], 'completed': False, 'failure': None, 'build_manifest_sha256': hashlib.sha256((BASE / 'build.json').read_bytes()).hexdigest()} @@ -542,8 +619,10 @@ def provision(): import argparse parser = argparse.ArgumentParser(description='Paired, pinned SQL durability verification; raw evidence stays outside the repository.') parser.add_argument('--context', required=True) - parser.add_argument('--resource-profile', choices=['shared-vm', 'isolated-owner'], default='shared-vm', - help='isolated-owner reserves guest CPUs 0-7 for the owner and 8-11 for support roles') + parser.add_argument('--resource-profile', choices=['shared-vm', 'isolated-owner', 'three-node-1cpu', 'three-vm-1cpu'], default='shared-vm', + help='isolated-owner reserves CPUs 0-7; three-node-1cpu caps nodes in one VM; three-vm-1cpu uses three separate 1-CPU/2-GiB Docker hosts') + parser.add_argument('--node-contexts', nargs=3, metavar=('OWNER', 'FOLLOWER1', 'FOLLOWER2'), + help='three separate node Docker hosts; loopback peer/provider routes must already be connected') parser.add_argument('--artifacts', type=Path, required=True) parser.add_argument('--tag', default=uuid.uuid4().hex[:10]) parser.add_argument('--seconds', type=int, default=300) @@ -564,6 +643,10 @@ def provision(): help='after steady points offer 1.5x this qualified capacity for 60s, then 0.5x for 30s') parser.add_argument('cases', nargs='*', choices=['cellule-fleet', 'celld-fleet', 'cellule-bucket', 'celld-bucket']) ARGS = parser.parse_args() + if separate_vms() != bool(ARGS.node_contexts): + parser.error('three-vm-1cpu requires exactly three node contexts, and other profiles do not use them') + if ARGS.node_contexts and (len(set(ARGS.node_contexts)) != 3 or ARGS.context in ARGS.node_contexts): + parser.error('owner, followers and support require distinct Docker contexts') BASE = ARGS.artifacts.resolve() CTX = ARGS.context if BASE == Path(__file__).resolve().parents[2] or Path(__file__).resolve().parents[2] in BASE.parents: diff --git a/scripts/verify-remote-follower-tail.py b/scripts/verify-remote-follower-tail.py new file mode 100755 index 00000000..10d71f7c --- /dev/null +++ b/scripts/verify-remote-follower-tail.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import pathlib +import subprocess +import shutil +import shlex +import time + +ROOT = pathlib.Path(__file__).resolve().parent.parent +parser = argparse.ArgumentParser(description='Verify a remote unpublished ACK after owner loss or follower TCP partition on three separate 1-vCPU/2-GiB Docker hosts.') +parser.add_argument('--artifacts', type=pathlib.Path, required=True) +parser.add_argument('--binary', type=pathlib.Path, required=True, help='release SQL example test executable, not the ordinary SQL server') +parser.add_argument('--context', required=True, help='separate provider/client support Docker context') +parser.add_argument('--node-contexts', nargs=3, required=True, metavar=('OWNER', 'FOLLOWER1', 'FOLLOWER2')) +parser.add_argument('--tag', required=True, help='new isolated object prefix and container-name suffix') +parser.add_argument('--fault', choices=['owner-process-loss', 'peer-partition'], default='owner-process-loss') +parser.add_argument('--owner-ssh-config', type=pathlib.Path, help='SSH configuration for the owner VM; required for peer-partition') +parser.add_argument('--owner-ssh-host', help='owner VM SSH alias; required for peer-partition') +args = parser.parse_args() +if args.fault == 'peer-partition' and (not args.owner_ssh_config or not args.owner_ssh_host): + parser.error('peer-partition requires owner VM SSH configuration and alias') +BASE = args.artifacts.resolve() +if BASE.is_relative_to(ROOT): + parser.error('evidence must be outside the source checkout') +BASE.mkdir(parents=True, exist_ok=True) +TAG = args.tag +if not TAG or any(c not in 'abcdefghijklmnopqrstuvwxyz0123456789-' for c in TAG): + parser.error('tag must contain lowercase letters, digits and hyphens') +CASE = BASE / TAG +CASE.mkdir(exist_ok=False) +SUPPORT = args.context +CONTEXTS = args.node_contexts +(BASE / 'bin').mkdir(exist_ok=True) +if args.binary.resolve() != BASE / 'bin/sql_fault': + shutil.copy2(args.binary, BASE / 'bin/sql_fault') +if not (BASE / 'tls').exists(): + subprocess.run(['python3', str(ROOT / 'scripts/generate-capacity-tls.py'), str(BASE / 'tls')], check=True) +for name in ['control.py', 'wait-store-ready.py']: + shutil.copy2(ROOT / 'scripts/perf' / name, BASE / name) +RUST = 'rust:1.98.1-bookworm@sha256:93ce27a88655056a51dbdd8f5f2d7ddc071c7b0070fb288a37b5a285fc83971e' +STORE = 'ghcr.io/rustfs/rustfs@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858' +SELECTOR = 'fleet::remote_fault_tests::remote_crash_fixture_child' +owned = [] +partition_rules = set() +status = dict(phase='setup', started_at=time.time(), performance_qualification=False, + fault=args.fault, + transport='original pinned mTLS + signed grants + sealed remote tail', + scope='three independent 1-CPU/2-GiB VMs, native ext4; not physical power loss', + joins=[], ack_count=0, audit_count=0) + + +def save(): + temporary = CASE / 'lifecycle.json.tmp' + temporary.write_text(json.dumps(status, indent=2) + '\n') + temporary.replace(CASE / 'lifecycle.json') + + +def docker(context, *arguments, check=True, timeout=25): + result = subprocess.run(['docker', '--context', context, *arguments], text=True, + capture_output=True, timeout=timeout) + if check and result.returncode: + raise RuntimeError(f'Docker {context} {arguments[:2]} failed: {result.stderr}') + return result + + +def wait_until(predicate, seconds, label): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.5) + raise RuntimeError('Fixture guard expired: ' + label) + + +def owner_ssh(*command): + result = subprocess.run(['ssh', '-F', str(args.owner_ssh_config.resolve()), + '-o', 'BatchMode=yes', args.owner_ssh_host, shlex.join(command)], + text=True, capture_output=True, timeout=25) + if result.returncode: + raise RuntimeError('Owner VM SSH command failed: ' + result.stderr) + return result.stdout + + +def partition_rule(port, install): + rule = ['OUTPUT', '-p', 'tcp', '-d', '127.0.0.1', '--dport', str(port), + '-m', 'comment', '--comment', 'cellule-' + TAG + '-partition', '-j', 'DROP'] + if install: + # Record before the command: an SSH reply loss may leave the rule installed. + partition_rules.add(port) + owner_ssh('sudo', '-n', 'iptables', '-I', *rule) + else: + owner_ssh('sudo', '-n', 'iptables', '-D', *rule) + partition_rules.remove(port) + + +def partition_snapshot(label, ports): + raw = owner_ssh('sudo', '-n', 'iptables', '-L', 'OUTPUT', '-n', '-v', '-x') + (CASE / (label + '-iptables.txt')).write_text(raw) + counters = {} + for line in raw.splitlines(): + if 'cellule-' + TAG + '-partition' not in line: + continue + for port in ports: + if 'dpt:' + str(port) in line: + counters[port] = int(line.split()[0]) + assert all(counters.get(port, 0) > 0 for port in ports), 'no actual dropped TCP packets' + status.setdefault('partition_observations', []).append(dict(label=label, dropped_packets=counters)) + save() + + +def signal(context, name, file): + code = "import os,pathlib; p=pathlib.Path('/scratch')/" + repr(file) + "; f=p.open('x'); f.write('ready\\n'); f.flush(); os.fsync(f.fileno()); f.close()" + docker(context, 'exec', name, 'python3', '-c', code) + + +def native_snapshot(context, name): + code = """import pathlib,json,subprocess +p=pathlib.Path('/proc') +print(json.dumps(dict(boot_id=(p/'sys/kernel/random/boot_id').read_text().strip(), +memory=(p/'meminfo').read_text(),swaps=(p/'swaps').read_text(), +oom_kill=dict(x.split() for x in (p/'vmstat').read_text().splitlines())['oom_kill'], +filesystem=subprocess.check_output(['df','-PT','/scratch'],text=True), +memory_max=pathlib.Path('/sys/fs/cgroup/memory.max').read_text().strip(), +cpu_max=pathlib.Path('/sys/fs/cgroup/cpu.max').read_text().strip(), +memory_events=pathlib.Path('/sys/fs/cgroup/memory.events').read_text()))) +""" + data = json.loads(docker(context, 'exec', name, 'python3', '-c', code).stdout) + assert ' ext4 ' in data['filesystem'] and 'virtiofs' not in data['filesystem'] + assert data['memory_max'] == str(2 << 30) + assert data['cpu_max'].split() == ['100000', '100000'] + assert len(data['swaps'].splitlines()) == 1 + assert 'oom_kill 0' in data['memory_events'] + return data + + +def start_role(role, index): + context = CONTEXTS[index] + name = f'cellule-{TAG}-{role}-{index}' + volume = name + '-native' + if docker(context, 'volume', 'inspect', volume, check=False).returncode == 0: + raise RuntimeError('Native state volume is not fresh: ' + volume) + docker(context, 'volume', 'create', '--label', 'cellule.fault.artifacts=' + str(CASE), volume) + run_arguments = ['run', '-d', '--name', name, '--label', 'cellule.fault.artifacts=' + str(CASE), + '--network', 'host', '--cpus', '1', '--memory', '2g', '--memory-swap', '2g', + '--ulimit', 'nofile=65536:65536', '-v', volume + ':/scratch', '-v', str(BASE) + ':/work:ro', + '-e', 'AWS_ACCESS_KEY_ID=benchmark_access', '-e', 'AWS_SECRET_ACCESS_KEY=benchmark_secret_private', + '-e', 'AWS_DEFAULT_REGION=us-east-1', '-e', 'CELLULE_TEST_ENDPOINT=http://127.0.0.1:9000', + '-e', 'CELLULE_TEST_BUCKET=comparison', '-e', 'CELLULE_TEST_PREFIX=' + TAG, + '-e', 'CELLULE_TEST_DEVICE_PATH=/scratch', '-e', 'TMPDIR=/scratch', + '-e', 'CELLULE_AXUM_FLEET_DIR=/scratch/fleet', '-e', 'CELLULE_REMOTE_FAULT_ROLE=' + role, + '-e', 'CELLULE_REMOTE_FAULT_KIND=' + args.fault, + '-e', 'TOKIO_WORKER_THREADS=1', '-e', 'RUST_BACKTRACE=1'] + if index: + run_arguments += ['-e', 'CELLULE_AXUM_FOLLOWER=' + str(index)] + if role == 'recover': + run_arguments += ['-e', 'CELLULE_REMOTE_FAULT_ACK=' + json.dumps(status['ack'])] + run_arguments += [RUST, 'sh', '-c', f'mkdir -m 700 /scratch/fleet && cp /work/tls/ca.crt /work/tls/node-{index}.crt /work/tls/node-{index}.key /scratch/fleet/ && exec /work/bin/sql_fault {SELECTOR} --exact --ignored --nocapture'] + # Track ownership before the RPC: a failed Docker response can be ambiguous. + owned.append((context, name)) + docker(context, *run_arguments) + status.setdefault('roles', []).append(dict(role=role, index=index, context=context, name=name, volume=volume)) + save() + return context, name + + +def marker(context, name, file): + result = docker(context, 'exec', name, 'cat', '/scratch/' + file, check=False) + if result.returncode == 0: + return json.loads(result.stdout) + state = json.loads(docker(context, 'inspect', name).stdout)[0]['State'] + if not state['Running']: + raise RuntimeError(f'Fixture role exited before {file}: {name}, exit {state["ExitCode"]}') + return None + + +def joined(context, name, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + info = json.loads(docker(context, 'inspect', name).stdout)[0] + if not info['State']['Running']: + status['joins'].append(dict(context=context, name=name, state=info['State'])) + assert not info['State']['OOMKilled'] + return info['State']['ExitCode'] + time.sleep(0.5) + raise RuntimeError('Process did not join: ' + name) + + +try: + save() + listed = docker(SUPPORT, 'run', '--rm', '-v', str(BASE) + ':/work:ro', RUST, '/work/bin/sql_fault', SELECTOR, '--exact', '--ignored', '--list').stdout + assert SELECTOR + ': test' in listed and '1 test, 0 benchmarks' in listed + status['binary_sha256'] = hashlib.sha256((BASE / 'bin/sql_fault').read_bytes()).hexdigest() + info = {context: json.loads(docker(context, 'info', '--format', '{{json .}}').stdout) + for context in CONTEXTS} + assert len({value['ID'] for value in info.values()}) == 3 + assert all(value['NCPU'] == 1 and 1.8 * (1 << 30) < value['MemTotal'] <= 2 << 30 + for value in info.values()) + (CASE / 'node-hosts.json').write_text(json.dumps(info, indent=2) + '\n') + store = 'cellule-' + TAG + '-store' + volume = store + '-native' + assert docker(SUPPORT, 'volume', 'inspect', volume, check=False).returncode != 0 + docker(SUPPORT, 'volume', 'create', '--label', 'cellule.fault.artifacts=' + str(CASE), volume) + owned.append((SUPPORT, store)) + docker(SUPPORT, 'run', '-d', '--name', store, '--label', 'cellule.fault.artifacts=' + str(CASE), + '--network', 'host', '--cpus', '2', '--cpuset-cpus', '0-1', '--memory', '2g', + '--memory-swap', '2g', '-v', volume + ':/data', '-e', 'RUSTFS_ACCESS_KEY=benchmark_access', + '-e', 'RUSTFS_SECRET_KEY=benchmark_secret_private', '-e', 'MIMALLOC_ALLOW_THP=0', STORE, '/data') + control = BASE / 'control.py' + wait_until(lambda: docker(SUPPORT, 'run', '--rm', '--network', 'host', '-v', + str(control) + ':/control.py:ro', RUST, 'python3', '/control.py', + 'bucket', check=False).returncode == 0, 60, 'real bucket creation') + ready = BASE / 'wait-store-ready.py' + result = docker(SUPPORT, 'run', '--rm', '--network', 'host', '-v', str(ready) + ':/ready.py:ro', + RUST, 'python3', '/ready.py', timeout=70) + (CASE / 'provider-ready.json').write_text(result.stdout) + followers = [start_role('follower', index) for index in (1, 2)] + for context, name in followers: + wait_until(lambda: 'Follower service:' in docker(context, 'logs', name).stdout, + 60, 'follower service ready') + owner = start_role('owner', 0) + status['phase'] = 'awaiting_unpublished_remote_ack' + save() + wait_until(lambda: marker(*owner, 'acknowledged.json') is not None, 45, 'remote ACK') + status['ack'] = marker(*owner, 'acknowledged.json') + assert status['ack']['commit_sequence'] == 1 and status['ack']['bucket_commit_sequence'] == 0 + assert status['ack']['required_followers'] == 2 and status['ack']['held_bundle_upload'] + status['ack_count'] = 1 + before = {name: native_snapshot(context, name) for context, name in [owner, *followers]} + assert len({value['boot_id'] for value in before.values()}) == 3 + (CASE / 'native-before-crash.json').write_text(json.dumps(before, indent=2) + '\n') + if args.fault == 'peer-partition': + assert owner_ssh('cat', '/proc/sys/kernel/random/boot_id').strip() == before[owner[1]]['boot_id'] + partition_rule(8090, True) + partition_rule(8100, True) + signal(*owner, 'partition-installed') + status['phase'] = 'checking_partition_output_gate' + save() + wait_until(lambda: marker(*owner, 'partition-gated.json') is not None, 10, 'partitioned output gate') + gated = marker(*owner, 'partition-gated.json') + assert gated['second_handler_executed'] and gated['second_capture_assigned'] + assert not gated['second_success'] and not gated['read_success'] + assert gated['bucket_commit_sequence'] == 0 + (CASE / 'partition-gated.json').write_text(json.dumps(gated, indent=2) + '\n') + status['phase'] = 'killing_owner' + save() + docker(*owner[:1], 'kill', '--signal', 'KILL', owner[1]) + assert joined(*owner, 10) == 137 + if args.fault == 'peer-partition': + partition_snapshot('both-followers-blocked', [8090, 8100]) + # The embedding example bounds its live inventory to the three-node fleet. + # Enroll the replacement only after actual old-boot lease expiry, using + # wall time; never fake a future recovery timestamp or relax that bound. + status['phase'] = 'waiting_real_owner_lease_expiry' + save() + wait_until(lambda: time.time() * 1000 > status['ack']['owner_expires_at_ms'] + 1000, + 40, 'real predecessor lease expiry before successor enrollment') + successor = start_role('recover', 0) + successor_native = native_snapshot(*successor) + assert successor_native['boot_id'] == before[owner[1]]['boot_id'] + assert successor_native['oom_kill'] == before[owner[1]]['oom_kill'] + (CASE / 'native-successor.json').write_text(json.dumps(successor_native, indent=2) + '\n') + status['phase'] = 'recovering_remote_tail' + save() + if args.fault == 'peer-partition': + wait_until(lambda: marker(*successor, 'recovery-refused.json') is not None, + 65, 'recovery refuses missing complete follower witness') + refused = marker(*successor, 'recovery-refused.json') + assert refused['missing_witness_refused'] and refused['no_takeover'] and refused['no_overlay'] + assert refused['bucket_commit_sequence'] == 0 + (CASE / 'recovery-refused.json').write_text(json.dumps(refused, indent=2) + '\n') + partition_snapshot('recovery-refused', [8090, 8100]) + partition_rule(8090, False) + signal(*successor, 'one-witness-reachable') + wait_until(lambda: marker(*successor, 'recovered-before-drain.json') is not None, + 45, 'one complete reachable follower recovers original ACK') + recovered = marker(*successor, 'recovered-before-drain.json') + assert recovered['value'] == recovered['commit_sequence'] == 1 + assert recovered['exact_retry'] and recovered['single_reachable_witness'] + (CASE / 'recovered-before-drain.json').write_text(json.dumps(recovered, indent=2) + '\n') + partition_snapshot('one-follower-still-blocked', [8100]) + partition_rule(8100, False) + signal(*successor, 'partition-healed') + assert joined(*successor, 100) == 0 + # Read the marker from the stopped container without reusing its database. + docker(successor[0], 'cp', successor[1] + ':/scratch/recovered.json', str(CASE / 'recovered.json')) + recovered = json.loads((CASE / 'recovered.json').read_text()) + assert recovered['commit_sequence'] == recovered['value'] == 1 + assert recovered['exact_retry'] and not recovered['reexecution'] + assert recovered['runtime_joined'] and recovered['owner_joined'] + status['audit_count'] = 1 + for context, name in followers: + after = native_snapshot(context, name) + assert after['boot_id'] == before[name]['boot_id'] + assert after['oom_kill'] == before[name]['oom_kill'] + status.setdefault('native_followers_after_recovery', {})[name] = after + docker(context, 'kill', '--signal', 'INT', name) + assert joined(context, name, 90) == 0 + status['phase'] = 'passed' +except Exception as error: + status.update(failure_phase=status['phase'], phase='failed', error=repr(error)) +finally: + # Even a failed scenario has to join every accepted process and retain logs. + cleanup_errors = [] + for port in sorted(partition_rules.copy()): + try: + partition_rule(port, False) + except Exception as error: + cleanup_errors.append(dict(partition_port=port, error=repr(error))) + # Keep the real provider alive until node shutdown has joined its I/O. + for context, name in reversed(owned): + try: + inspection = docker(context, 'inspect', name, check=False) + if inspection.returncode: + if 'No such object' in inspection.stderr or 'No such container' in inspection.stderr: + continue + raise RuntimeError(inspection.stderr) + info = json.loads(inspection.stdout)[0] + if info['State']['Running']: + docker(context, 'stop', '--time', '10', name, timeout=25) + result = docker(context, 'logs', name, check=False) + (CASE / (name + '.log')).write_text(result.stdout + result.stderr) + terminal = json.loads(docker(context, 'inspect', name).stdout)[0] + assert not terminal['State']['Running'] and not terminal['State']['OOMKilled'] + (CASE / (name + '-terminal.json')).write_text(json.dumps(terminal, indent=2) + '\n') + if status['phase'] == 'passed': + expected_exit = 137 if name == owner[1] else 0 + assert terminal['State']['ExitCode'] == expected_exit + except Exception as error: + cleanup_errors.append(dict(context=context, name=name, error=repr(error))) + if cleanup_errors: + status.update(phase='failed', cleanup_errors=cleanup_errors) + status.update(finished_at=time.time(), all_owned_containers_terminal=not cleanup_errors) + save() +print(json.dumps(status, indent=2)) +raise SystemExit(0 if status['phase'] == 'passed' else 1)