From 4a0c3bd0159c444891cd8d08bc32fe90f118f444 Mon Sep 17 00:00:00 2001 From: rain Date: Tue, 10 Nov 2026 00:42:24 +0800 Subject: [PATCH 1/2] test(547): restore the deleted daemon slot regression suite and publish the state machine (#547) Slice 1: restore the four deleted daemon assertions (the pre-#543 connection tests) as package integration tests, and make the pure Slots/RetainedFailures state machine public behind the #[doc(hidden)] internals module so it is unit-testable from tests/. No behavior change. - engine/crates/layerfs-daemon/tests/daemon_slots.rs: the four real-daemon custody assertions over TCP (undecodable records free their slot, the capacity notice on a full pool, and a concurrent failure storm). - engine/crates/layerfs-daemon/tests/slot_state.rs: the deleted owner/retained unit assertions plus the guard-drop release, via the public state machine. - application/internals.rs: #[doc(hidden)] re-export of the pub'd state-machine values; application/{owner,retained}.rs: Slots/RetainedFailures and their methods are now pub. --- .../src/application/internals.rs | 17 + .../layerfs-daemon/src/application/mod.rs | 2 + .../layerfs-daemon/src/application/owner.rs | 34 +- .../src/application/retained.rs | 20 +- .../layerfs-daemon/tests/daemon_slots.rs | 381 ++++++++++++++++++ .../crates/layerfs-daemon/tests/slot_state.rs | 318 +++++++++++++++ 6 files changed, 745 insertions(+), 27 deletions(-) create mode 100644 engine/crates/layerfs-daemon/src/application/internals.rs create mode 100644 engine/crates/layerfs-daemon/tests/daemon_slots.rs create mode 100644 engine/crates/layerfs-daemon/tests/slot_state.rs diff --git a/engine/crates/layerfs-daemon/src/application/internals.rs b/engine/crates/layerfs-daemon/src/application/internals.rs new file mode 100644 index 0000000000..dd7201b71c --- /dev/null +++ b/engine/crates/layerfs-daemon/src/application/internals.rs @@ -0,0 +1,17 @@ +//! Pure control-slot and retained-failure values, observed by the integration +//! tests in `tests/`. +//! +//! `#[doc(hidden)]` keeps this module out of rustdoc; it is not access +//! control. The values are the daemon's state machine with no I/O seam, so +//! the only place a test can reach them is a re-export: product `src/` may +//! not carry a test-only `cfg` attribute (the product-boundary guard rejects +//! it). Keep the documented `application` exports to the startup API +//! (`Application`, `ApplicationError`, `ConnectionFailure`); anything a test +//! reads belongs here, behind this module's one name. + +#[doc(hidden)] +pub use super::owner::{ + AdmissionClass, Finished, SlotClass, SlotCounts, SlotTicket, Slots, RESERVED_SLOTS, +}; +#[doc(hidden)] +pub use super::retained::{RetainedFailure, RetainedFailures, RetainedSnapshot, LIMIT, TTL}; diff --git a/engine/crates/layerfs-daemon/src/application/mod.rs b/engine/crates/layerfs-daemon/src/application/mod.rs index 65427ebb16..78d0dd6073 100644 --- a/engine/crates/layerfs-daemon/src/application/mod.rs +++ b/engine/crates/layerfs-daemon/src/application/mod.rs @@ -6,6 +6,8 @@ mod connection; mod diagnostics; mod failure; mod filesystem; +#[doc(hidden)] +pub mod internals; mod owner; mod retained; mod serve; diff --git a/engine/crates/layerfs-daemon/src/application/owner.rs b/engine/crates/layerfs-daemon/src/application/owner.rs index a8181601e6..3f7956d2a8 100644 --- a/engine/crates/layerfs-daemon/src/application/owner.rs +++ b/engine/crates/layerfs-daemon/src/application/owner.rs @@ -25,11 +25,11 @@ use std::{ /// Slots reserved for the installer (and, later, a presence lease); ordinary /// controls never consume them. -pub(super) const RESERVED_SLOTS: usize = 1; +pub const RESERVED_SLOTS: usize = 1; /// The fixed class of one slot index. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum SlotClass { +pub enum SlotClass { /// Ordinary authenticated control connections. Pool, /// Installer today, a presence lease later. @@ -37,7 +37,7 @@ pub(super) enum SlotClass { } impl SlotClass { - pub(super) fn as_str(self) -> &'static str { + pub fn as_str(self) -> &'static str { match self { Self::Pool => "pool", Self::Reserved => "reserved", @@ -47,7 +47,7 @@ impl SlotClass { /// What a caller is admitted for; maps to the slot class it may use. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum AdmissionClass { +pub enum AdmissionClass { /// An ordinary control connection. ControlPool, /// The one-time Store installer. @@ -77,14 +77,14 @@ pub(super) enum SlotState { /// exactly one `Slots::finish` with this ticket; a stale ticket changes /// nothing, so a duplicate or late finish cannot free a newer worker. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) struct SlotTicket { - pub(super) index: usize, - pub(super) generation: u64, - pub(super) class: SlotClass, +pub struct SlotTicket { + pub index: usize, + pub generation: u64, + pub class: SlotClass, } /// What finishing one connection did with its failure, if any. -pub(super) enum Finished { +pub enum Finished { /// The worker ended without a failure. Clean, /// The failure was written to the daemon log and needs no record. @@ -111,7 +111,7 @@ pub struct SlotCounts { /// (#540). The layout is `connections` `Pool` entries followed by /// [`RESERVED_SLOTS`] `Reserved` entries; a class only ever admits within its /// own range. -pub(super) struct Slots { +pub struct Slots { slots: Vec, pool: usize, next_generation: u64, @@ -119,7 +119,7 @@ pub(super) struct Slots { } impl Slots { - pub(super) fn new(count: usize) -> Self { + pub fn new(count: usize) -> Self { let pool = count.saturating_sub(RESERVED_SLOTS); let mut slots = Vec::with_capacity(pool + RESERVED_SLOTS); slots.resize(pool, SlotState::Vacant); @@ -142,7 +142,7 @@ impl Slots { /// Claims one vacant slot of `class`, or `None` when that class is full. /// A pool admission never takes a reserved slot and vice versa. - pub(super) fn admit(&mut self, class: AdmissionClass) -> Option { + pub fn admit(&mut self, class: AdmissionClass) -> Option { let wanted = class.slot_class(); let index = self.slots.iter().enumerate().find_map(|(index, value)| { (self.class_of(index) == wanted && matches!(value, SlotState::Vacant)).then_some(index) @@ -164,7 +164,7 @@ impl Slots { /// Ends one worker: release its slot and, for a retained failure, record /// the already-built record. A ticket whose generation or class no longer /// matches changes nothing. - pub(super) fn finish( + pub fn finish( &mut self, ticket: SlotTicket, result: Result<(), ConnectionFailure>, @@ -193,12 +193,12 @@ impl Slots { } } - pub(super) fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { + pub fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { self.retained.snapshot(now) } /// Admission counts for each class. - pub(super) fn counts(&self) -> SlotCounts { + pub fn counts(&self) -> SlotCounts { let mut counts = SlotCounts { pool_vacant: 0, pool_running: 0, @@ -217,12 +217,12 @@ impl Slots { } /// Visible retained records, for the admission notice. - pub(super) fn visible(&mut self) -> usize { + pub fn visible(&mut self) -> usize { self.retained.len() } /// Drops expired records; the accept loop reaps when it refuses admission. - pub(super) fn reap(&mut self, now: Instant) { + pub fn reap(&mut self, now: Instant) { self.retained.reap(now); } } diff --git a/engine/crates/layerfs-daemon/src/application/retained.rs b/engine/crates/layerfs-daemon/src/application/retained.rs index f7697d4030..5de1641ca9 100644 --- a/engine/crates/layerfs-daemon/src/application/retained.rs +++ b/engine/crates/layerfs-daemon/src/application/retained.rs @@ -13,9 +13,9 @@ use std::{ }; /// Most retained records kept at once. The oldest is dropped first. -pub(super) const LIMIT: usize = 64; +pub const LIMIT: usize = 64; /// How long a retained record stays operator-visible. -pub(super) const TTL: Duration = Duration::from_secs(60 * 60); +pub const TTL: Duration = Duration::from_secs(60 * 60); /// Most bytes a retained textual cause keeps. const CAUSE_BYTES: usize = 512; @@ -55,7 +55,7 @@ pub struct RetainedFailure { impl RetainedFailure { /// The bounded record for `failure`, built outside the state lock. - pub(super) fn describe( + pub fn describe( failure: &ConnectionFailure, class: super::owner::SlotClass, generation: u64, @@ -129,7 +129,7 @@ struct Entry { /// A fixed-capacity, TTL-pruned ring of retained connection failures. Its /// capacity is reserved when it is built, so recording under the state lock /// never allocates. -pub(super) struct RetainedFailures { +pub struct RetainedFailures { entries: VecDeque, dropped: u64, recorded: u64, @@ -144,7 +144,7 @@ impl Default for RetainedFailures { } impl RetainedFailures { - pub(super) fn new(limit: usize, ttl: Duration) -> Self { + pub fn new(limit: usize, ttl: Duration) -> Self { let limit = limit.max(1); Self { entries: VecDeque::with_capacity(limit), @@ -158,7 +158,7 @@ impl RetainedFailures { /// Records one already-described failure, prunes expired records and /// evicts the oldest beyond the limit. Returns the exact record that now /// is visible, with its store position filled in. - pub(super) fn record(&mut self, mut record: RetainedFailure, now: Instant) -> RetainedFailure { + pub fn record(&mut self, mut record: RetainedFailure, now: Instant) -> RetainedFailure { self.prune(now); while self.entries.len() >= self.limit { self.entries.pop_front(); @@ -177,7 +177,7 @@ impl RetainedFailures { } /// Drops records whose deadline passed; a stale record is never returned. - pub(super) fn prune(&mut self, now: Instant) { + pub fn prune(&mut self, now: Instant) { while let Some(front) = self.entries.front() { if now < front.deadline { break; @@ -188,12 +188,12 @@ impl RetainedFailures { } /// The lazy reaper: the accept loop calls it when it refuses admission. - pub(super) fn reap(&mut self, now: Instant) { + pub fn reap(&mut self, now: Instant) { self.prune(now); } /// A copy of the visible records, oldest first, with their ages. - pub(super) fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { + pub fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { self.prune(now); RetainedSnapshot { failures: self @@ -213,7 +213,7 @@ impl RetainedFailures { } } - pub(super) fn len(&self) -> usize { + pub fn len(&self) -> usize { self.entries.len() } } diff --git a/engine/crates/layerfs-daemon/tests/daemon_slots.rs b/engine/crates/layerfs-daemon/tests/daemon_slots.rs new file mode 100644 index 0000000000..c2a1ee6cef --- /dev/null +++ b/engine/crates/layerfs-daemon/tests/daemon_slots.rs @@ -0,0 +1,381 @@ +//! #547 regression coverage for the real daemon's control-slot custody. +//! +//! #543 deleted the inline tests that proved #540: connections whose received +//! record the daemon cannot decode must not remove admission capacity, no +//! vacant slot must answer with the advisory capacity notice instead of a +//! bare EOF, and a failure storm must neither deadlock nor leak a slot. +//! +//! Each test starts the package's own `layerfs-daemon` executable with a +//! protected configuration record and talks to it over the authenticated +//! Noise channel exactly as a client does. No product `Service` is composed +//! inside the test process, and no test-only product API is used. + +#![cfg(any(target_os = "linux", target_os = "macos"))] + +use layerfs_bridge::{ + control::{ + Answer, Call, CapacityModel, DaemonPhase, HelloRequest, Reply, Request, SdkCall, SdkValue, + CAPACITY_OPERATION, + }, + daemon_setup::{DaemonLimits, DaemonSetup}, + duplex::{self, ChannelError, Connection}, + DaemonStore, KernelRoute, +}; +use std::{ + fs::{self, File}, + io::{BufRead, BufReader, Read}, + net::{SocketAddr, TcpStream}, + os::unix::fs::{MetadataExt, PermissionsExt}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::mpsc, + thread, + time::{Duration, Instant}, +}; + +const CLIENT_PRIVATE: [u8; 32] = [71; 32]; +const SERVER_PRIVATE: [u8; 32] = [72; 32]; +const CONNECTIONS: u16 = 8; +/// Deaths past every slot (the pool plus the reserved one), so a daemon that +/// retains a failed slot runs out of admission capacity. +const ROUNDS: usize = CONNECTIONS as usize + 4; +/// A record that is received but fails `InitialRecord::decode`, so the worker +/// retains the exact failure. It is not a decoded request: the daemon executes +/// nothing for it. +const DIE: &[u8] = b"client-died-before-reading-reply"; + +/// One real daemon executable and the directory holding its protected files. +struct Daemon { + child: Child, + address: SocketAddr, + stderr: PathBuf, + directory: PathBuf, +} + +impl Drop for Daemon { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = fs::remove_dir_all(&self.directory); + } +} + +fn private_directory(path: &Path) { + fs::create_dir_all(path).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap(); +} + +/// Starts the actual executable and returns it once it reports its listener. +fn start(label: &str, handshake_ms: u32) -> Daemon { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + let directory = std::env::temp_dir().join(format!( + "layerfs-547-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&directory); + private_directory(&directory); + let mounts = directory.join("mounts"); + private_directory(&mounts); + let identity = fs::metadata(&directory).unwrap(); + let setup = DaemonSetup { + listen: "127.0.0.1:0".into(), + private_key: SERVER_PRIVATE, + control_peer: duplex::public_key(&CLIENT_PRIVATE).unwrap(), + // The native NFS route needs no FUSE device, on either platform. + route: KernelRoute::Nfs, + overlay: directory.join("overlay.sqlite").display().to_string(), + mounts: mounts.display().to_string(), + command_uid: identity.uid(), + command_gid: identity.gid(), + limits: DaemonLimits { + connections: CONNECTIONS, + handshake_ms, + cache_bytes: 1024 * 1024, + namespaces: 16, + pager_kib: 1024, + serial_low_water: 256, + }, + // No installed Store: the daemon serves startup control only, which + // is enough to prove admission custody and needs no Store service. + store: DaemonStore::Sqlite { + locator: directory.join("store.sqlite").display().to_string(), + install: None, + }, + }; + let config = directory.join("daemon.config"); + fs::write(&config, setup.encode().unwrap()).unwrap(); + fs::set_permissions(&config, fs::Permissions::from_mode(0o600)).unwrap(); + let stderr = directory.join("daemon.stderr"); + let child = Command::new(env!("CARGO_BIN_EXE_layerfs-daemon")) + .arg("--config") + .arg(&config) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::from(File::create(&stderr).unwrap())) + .spawn() + .expect("spawn layerfs-daemon"); + let mut child = Daemon { + child, + address: "127.0.0.1:0".parse().unwrap(), + stderr, + directory, + }; + let stdout = child.child.stdout.take().unwrap(); + let (send, receive) = mpsc::sync_channel(1); + let reader = thread::spawn(move || { + let mut line = String::new(); + let result = BufReader::new(stdout).read_line(&mut line).map(|_| line); + let _ = send.send(result); + }); + let line = match receive.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(line)) => line, + _ => { + let output = fs::read_to_string(&child.stderr).unwrap_or_default(); + panic!("daemon never reported a listener; stderr:\n{output}"); + } + }; + reader.join().unwrap(); + child.address = line + .strip_prefix("LAYERFS_DAEMON_LISTEN ") + .unwrap_or_else(|| panic!("unexpected daemon startup line: {line:?}")) + .trim() + .parse() + .unwrap(); + child +} + +fn connect(address: SocketAddr) -> Connection { + let stream = TcpStream::connect_timeout(&address, Duration::from_secs(2)).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + duplex::initiate( + stream, + &CLIENT_PRIVATE, + duplex::public_key(&SERVER_PRIVATE).unwrap(), + ) + .unwrap() +} + +/// One authenticated client that reaches its Hello reply and then holds the +/// connection idle; the worker stays blocked in `receive()` and keeps its slot. +fn hello(address: SocketAddr) -> Connection { + let mut connection = connect(address); + let call = Call { + id: 1, + request: Request::Hello(HelloRequest { + expected_instance: None, + wait_for_store: false, + }), + }; + connection.send.send(&call.encode().unwrap()).unwrap(); + match Answer::decode(connection.receive.receive().unwrap()) + .unwrap() + .reply + { + Reply::Hello(status) => assert_eq!(status.phase, DaemonPhase::InstallPending), + other => panic!("unexpected Hello reply: {other:?}"), + } + connection +} + +/// The daemon's own control-capacity read model. +fn capacity(address: SocketAddr) -> CapacityModel { + let mut connection = connect(address); + let call = Call { + id: 7, + request: Request::Sdk(SdkCall { + method: CAPACITY_OPERATION.to_owned(), + args: SdkValue::Null, + }), + }; + connection.send.send(&call.encode().unwrap()).unwrap(); + match Answer::decode(connection.receive.receive().unwrap()) + .unwrap() + .reply + { + Reply::Sdk(value) => CapacityModel::parse(&value), + other => panic!("capacity query was not answered: {other:?}"), + } +} + +fn wait_for_log(path: &Path, needle: &str, count: usize) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let text = fs::read_to_string(path).unwrap_or_default(); + let found = text.lines().filter(|line| line.starts_with(needle)).count(); + if found >= count { + return; + } + assert!( + Instant::now() < deadline, + "timed out waiting for {count} {needle:?} line(s) in {}; saw {found}\n{text}", + path.display() + ); + thread::sleep(Duration::from_millis(20)); + } +} + +/// Clients send a record the daemon cannot decode and then disappear. Every +/// retained failure must free its slot: a fresh authenticated client is +/// admitted, the pool is whole, and one log line names each failure (#540). +#[test] +fn undecodable_records_do_not_consume_control_slots() { + let daemon = start("slots", 2000); + for round in 0..ROUNDS { + let mut connection = connect(daemon.address); + connection.send.send(DIE).unwrap(); + // The daemon records the failure and closes its end; wait for that + // close so the failure is retained rather than only seen as EOF. + assert!( + connection.receive.receive().is_err(), + "round {round}: the daemon kept the connection after a recorded failure" + ); + drop(connection); + } + + // A fresh authenticated client is admitted and reaches its Hello reply + // after every failed worker released its slot. + drop(hello(daemon.address)); + + let model = capacity(daemon.address); + // The query connection is the only worker running at this observation. + assert_eq!(model.admission.pool_running, 1); + assert_eq!(model.admission.pool_vacant, i64::from(CONNECTIONS) - 1); + assert_eq!( + model.admission.pool_vacant + model.admission.pool_running, + i64::from(CONNECTIONS) + ); + assert_eq!(model.retained.limit, 64); + assert_eq!(model.retained.recorded, ROUNDS as i64); + assert_eq!(model.retained.count, ROUNDS as i64); + assert_eq!(model.retained.dropped, 0); + for record in &model.retained.failures { + assert_eq!(record.class, "pool"); + assert!(record.generation > 0); + assert!( + record.owner.contains("layerfs-control-"), + "{}", + record.owner + ); + assert_eq!(record.received_bytes, Some(DIE.len() as i64)); + assert_eq!(record.request, None); + assert!(!record.reply_sent); + assert!(record.cause.contains("kernel request"), "{}", record.cause); + } + + wait_for_log(&daemon.stderr, "ConnectionFailure {", ROUNDS); +} + +/// With every pool slot held and the reserved slot occupied by a connection +/// that never completes its handshake, the next caller gets one 24-byte +/// `LAYERFS-CAPACITY` body instead of a bare EOF (#540). +#[test] +fn no_vacant_slot_writes_the_capacity_notice() { + let daemon = start("notice", 10_000); + let mut holders = Vec::new(); + for _ in 0..CONNECTIONS { + holders.push(hello(daemon.address)); + } + // Take the reserved slot with a raw connection that never speaks: its + // worker blocks in the Noise handshake and keeps the last slot. + let reserved = TcpStream::connect_timeout(&daemon.address, Duration::from_secs(2)).unwrap(); + reserved + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + thread::sleep(Duration::from_millis(300)); + // This one must be refused before any handshake. + let mut probe = TcpStream::connect_timeout(&daemon.address, Duration::from_secs(2)).unwrap(); + probe + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + + let mut framed = [0u8; 26]; + probe.read_exact(&mut framed).unwrap(); + assert_eq!( + u16::from_be_bytes([framed[0], framed[1]]) as usize, + layerfs_bridge::duplex::admission::BODY_LEN + ); + assert_eq!(&framed[2..2 + 16], layerfs_bridge::duplex::admission::MAGIC); + let notice = layerfs_bridge::duplex::admission::classify(&framed[2..]) + .expect("the body is not a capacity notice"); + // No Store is installed, so the daemon is still starting. + assert_eq!( + notice.reason, + layerfs_bridge::duplex::admission::AdmissionReason::Starting + ); + assert_eq!(notice.pool_vacant, 0); + assert_eq!(notice.reserved_vacant, 0); + + drop(probe); + drop(reserved); + holders.clear(); +} + +/// Sixteen threads each die repeatedly. The accept loop must not deadlock, +/// every worker must finish, the ring stays bounded and exact, and a fresh +/// authenticated client is still admitted afterwards. +#[test] +fn concurrent_failures_do_not_deadlock_or_leak() { + let daemon = start("concurrent", 3000); + let address = daemon.address; + let mut threads = Vec::new(); + for _ in 0..16 { + threads.push(thread::spawn(move || { + for _ in 0..4 { + loop { + let Ok(stream) = TcpStream::connect_timeout(&address, Duration::from_secs(2)) + else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + match duplex::initiate( + stream, + &CLIENT_PRIVATE, + duplex::public_key(&SERVER_PRIVATE).unwrap(), + ) { + Ok(mut connection) => { + connection.send.send(DIE).unwrap(); + let _ = connection.receive.receive(); + break; + } + Err(ChannelError::Admission(_)) | Err(ChannelError::Io(_)) => { + // Full pool or a connection lost in the storm: + // neither attempted anything, so retry is safe. + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("unexpected open failure: {error:?}"), + } + } + } + })); + } + for thread in threads { + thread.join().unwrap(); + } + + let model = capacity(address); + assert_eq!(model.admission.pool_running, 1); + assert_eq!(model.admission.pool_vacant, i64::from(CONNECTIONS) - 1); + assert!(model.retained.count >= 1, "{model:?}"); + assert!(model.retained.count <= model.retained.limit); + assert_eq!( + model.retained.count + model.retained.dropped, + model.retained.recorded + ); + + // A fresh authenticated client is admitted after the storm. + drop(hello(address)); + wait_for_log(&daemon.stderr, "ConnectionFailure {", 1); +} diff --git a/engine/crates/layerfs-daemon/tests/slot_state.rs b/engine/crates/layerfs-daemon/tests/slot_state.rs new file mode 100644 index 0000000000..c9699147cb --- /dev/null +++ b/engine/crates/layerfs-daemon/tests/slot_state.rs @@ -0,0 +1,318 @@ +//! #547 regression coverage for the pure control-slot state machine. +//! +//! #543 deleted the inline `#[cfg(test)]` modules that proved the #540 +//! invariants: a retained failure never owns a slot, a stale or duplicate +//! finish changes nothing, and the retained ring stays bounded and timed. +//! These tests live outside product `src/`; they reach only the pure +//! `Slots`/`RetainedFailures` values, no socket and no daemon process. + +use layerfs_bridge::control::{Call, Reply, Request}; +use layerfs_bridge::duplex::{ChannelError, ChannelWork, PeerEndFailure, PeerEndPhase}; +use layerfs_daemon::application::internals::{ + AdmissionClass, Finished, RetainedFailure, RetainedFailures, SlotClass, Slots, LIMIT, + RESERVED_SLOTS, TTL, +}; +use layerfs_daemon::application::{ApplicationError, ConnectionFailure}; +use layerfs_daemon::control::{Served, Success}; +use std::time::{Duration, Instant}; + +/// A failure after a decoded request: retained, never released. +fn failure(slot: usize) -> ConnectionFailure { + ConnectionFailure { + slot, + received: None, + original: None, + cause: ApplicationError::Poisoned, + fence_error: None, + diagnostic_error: None, + } +} + +fn describe(slot: usize, class: SlotClass, generation: u64) -> RetainedFailure { + RetainedFailure::describe( + &failure(slot), + class, + generation, + "daemon:pid:layerfs-control-0", + ) +} + +/// A pool admission never takes the reserved slot and vice versa. +#[test] +fn admit_uses_only_its_class() { + let mut slots = Slots::new(8 + RESERVED_SLOTS); + for _ in 0..8 { + slots.admit(AdmissionClass::ControlPool).unwrap(); + } + assert_eq!( + slots.admit(AdmissionClass::ControlPool), + None, + "a pool admission took the reserved slot" + ); + let reserved = slots.admit(AdmissionClass::Install).unwrap(); + assert_eq!(reserved.index, 8); + assert_eq!(reserved.class, SlotClass::Reserved); + assert_eq!(slots.admit(AdmissionClass::Lease), None); + let counts = slots.counts(); + assert_eq!(counts.pool_vacant, 0); + assert_eq!(counts.pool_running, 8); + assert_eq!(counts.reserved_vacant, 0); + assert_eq!(counts.reserved_running, 1); +} + +/// A retained failure is a record, not a slot: finishing the worker frees the +/// slot before the record is stored (#540). +#[test] +fn finish_frees_even_for_a_retained_failure() { + let mut slots = Slots::new(8 + RESERVED_SLOTS); + let now = Instant::now(); + let ticket = slots.admit(AdmissionClass::ControlPool).unwrap(); + let prebuilt = describe(ticket.index, ticket.class, ticket.generation); + match slots.finish(ticket, Err(failure(ticket.index)), Some(prebuilt), now) { + Finished::Retained(record) => { + assert_eq!(record.class, "pool"); + assert_eq!(record.generation, ticket.generation); + } + _ => panic!("a decoded failure was not retained"), + } + assert_eq!(slots.counts().pool_vacant, 8); + assert_eq!(slots.snapshot(now).failures.len(), 1); +} + +/// Every running slot ends exactly once; a duplicate or stale finish releases +/// nothing and cannot free a newer worker (invariant 1). +#[test] +fn stale_or_duplicate_finish_changes_nothing() { + let mut slots = Slots::new(8 + RESERVED_SLOTS); + let now = Instant::now(); + let first = slots.admit(AdmissionClass::ControlPool).unwrap(); + slots.finish( + first, + Err(failure(first.index)), + Some(describe(first.index, first.class, first.generation)), + now, + ); + // A duplicate finish of the same ticket changes nothing. + assert!(matches!( + slots.finish(first, Ok(()), None, now), + Finished::Stale + )); + let second = slots.admit(AdmissionClass::ControlPool).unwrap(); + assert_eq!(second.index, first.index); + assert!(second.generation > first.generation); + // The old ticket cannot free the newer worker. + assert!(matches!( + slots.finish(first, Ok(()), None, now), + Finished::Stale + )); + assert_eq!(slots.counts().pool_running, 1); + slots.finish(second, Ok(()), None, now); + assert_eq!(slots.counts().pool_vacant, 8); +} + +/// The retained ring is bounded: the 65th record evicts the oldest and each +/// eviction increments `dropped` (invariant 3). +#[test] +fn ring_limit_evicts_oldest_and_counts_drops() { + let mut store = RetainedFailures::new(3, TTL); + let now = Instant::now(); + for slot in 0..5 { + store.record(describe(slot, SlotClass::Pool, slot as u64 + 1), now); + } + assert_eq!(store.len(), 3); + let snapshot = store.snapshot(now); + assert_eq!(snapshot.dropped, 2); + assert_eq!(snapshot.recorded, 5); + assert_eq!( + snapshot.failures.iter().map(|f| f.slot).collect::>(), + vec![2, 3, 4] + ); + assert_eq!(snapshot.limit, 3); + assert_eq!(snapshot.ttl, TTL); + + let mut full = RetainedFailures::new(LIMIT, TTL); + for slot in 0..LIMIT + 1 { + full.record(describe(slot % 8, SlotClass::Pool, slot as u64 + 1), now); + } + let snapshot = full.snapshot(now); + assert_eq!(snapshot.failures.len(), LIMIT); + assert_eq!(snapshot.dropped, 1); + assert_eq!(snapshot.recorded as usize, LIMIT + 1); +} + +/// Records past their TTL are never returned and count as dropped. +#[test] +fn ttl_prunes_expired_records() { + let mut store = RetainedFailures::new(8, Duration::from_secs(10)); + let now = Instant::now(); + store.record(describe(1, SlotClass::Pool, 1), now); + store.record( + describe(2, SlotClass::Pool, 2), + now + Duration::from_secs(5), + ); + let snapshot = store.snapshot(now + Duration::from_secs(9)); + assert_eq!(snapshot.failures.len(), 2); + assert_eq!(snapshot.failures[0].expires_in, Duration::from_secs(1)); + let snapshot = store.snapshot(now + Duration::from_secs(16)); + assert_eq!(snapshot.failures.len(), 0); + assert_eq!(snapshot.dropped, 2); +} + +/// The accept loop's lazy reaper frees expired records before it refuses. +#[test] +fn retained_store_reaps_expired_records() { + let mut slots = Slots::new(1 + RESERVED_SLOTS); + let now = Instant::now(); + let ticket = slots.admit(AdmissionClass::ControlPool).unwrap(); + slots.finish( + ticket, + Err(failure(ticket.index)), + Some(describe(ticket.index, ticket.class, ticket.generation)), + now, + ); + slots.reap(now + TTL + Duration::from_secs(1)); + assert_eq!( + slots + .snapshot(now + TTL + Duration::from_secs(1)) + .failures + .len(), + 0 + ); +} + +/// The textual cause is bounded and cut on a character boundary. +#[test] +fn cause_is_bounded_and_char_safe() { + let mut failure = failure(0); + failure.cause = ApplicationError::Worker("é".repeat(4096)); + let record = RetainedFailure::describe(&failure, SlotClass::Pool, 1, "owner"); + assert!(record.cause.len() <= 512 + "…".len()); + assert!(record.cause.ends_with('…')); + assert!(record.cause.is_char_boundary(record.cause.len())); +} + +/// A record names the slot, class, generation, owner, request, reply state and +/// cause the operator needs — not just a count. +#[test] +fn record_names_the_cause_request_and_reply_state() { + let served = Served { + call: Call { + id: 1, + request: Request::EndSession, + }, + outcome: Ok(Success { + reply: Reply::SessionEnded, + commit: None, + observation_failure: None, + kernel: None, + }), + }; + let failure = ConnectionFailure { + slot: 3, + received: Some(vec![0xde, 0xad, 0xbe, 0xef]), + original: None, + cause: ApplicationError::SessionEnd { + served: Box::new(served), + ending: Box::new(PeerEndFailure { + cause: ChannelError::Invalid("peer reset after final reply"), + phase: PeerEndPhase::Read, + fence_attempted: false, + received: None, + work: ChannelWork::default(), + }), + }, + fence_error: None, + diagnostic_error: None, + }; + let record = RetainedFailure::describe( + &failure, + SlotClass::Pool, + 9, + "a1b2c3d4:812:layerfs-control-3", + ); + assert_eq!(record.slot, 3); + assert_eq!(record.class, "pool"); + assert_eq!(record.generation, 9); + assert_eq!(record.received_bytes, Some(4)); + assert_eq!(record.request, Some("end-session")); + assert!(record.reply_sent); + assert!(record.cause.contains("session"), "{}", record.cause); + let line = record.to_string(); + assert!(line.starts_with("ConnectionFailure {"), "{line}"); + assert!( + line.contains("original_kind: Some(\"end-session\")"), + "{line}" + ); + assert!(line.contains("reply: \"sent\""), "{line}"); + assert!(line.contains("class: \"pool\""), "{line}"); + assert!(line.contains("owner:"), "{line}"); + assert!(line.contains("expires_in_ms:"), "{line}"); +} + +/// A record names the slot, class, generation and owner that a retained +/// record reports, with its deadline visible. +#[test] +fn owner_and_deadline_are_visible() { + let mut store = RetainedFailures::new(8, TTL); + let now = Instant::now(); + let record = store.record(describe(3, SlotClass::Pool, 9), now); + assert_eq!(record.class, "pool"); + assert_eq!(record.generation, 9); + assert!( + record.owner.contains("layerfs-control-"), + "{}", + record.owner + ); + assert_eq!(record.retained, 1); + let snapshot = store.snapshot(now + Duration::from_secs(1)); + assert_eq!( + snapshot.failures[0].expires_in, + TTL - Duration::from_secs(1) + ); +} + +/// A worker whose guard is dropped without an explicit finish releases its +/// slot and records the abandonment exactly once, like any other retained +/// failure (invariant 1). +#[test] +fn guard_drop_releases_the_slot_and_records_once() { + let mut slots = Slots::new(1 + RESERVED_SLOTS); + let now = Instant::now(); + let ticket = slots.admit(AdmissionClass::ControlPool).unwrap(); + let failure = ConnectionFailure { + slot: ticket.index, + received: None, + original: None, + cause: ApplicationError::Worker("dropped before completion".to_owned()), + fence_error: None, + diagnostic_error: None, + }; + let record = RetainedFailure::describe( + &failure, + ticket.class, + ticket.generation, + "daemon:pid:layerfs-control-0", + ); + match slots.finish(ticket, Err(failure), Some(record), now) { + Finished::Retained(record) => { + assert_eq!(record.slot, ticket.index); + assert_eq!(record.class, "pool"); + assert!(!record.reply_sent); + assert!( + record.cause.contains("dropped before completion"), + "{}", + record.cause + ); + } + _ => panic!("an abandoned slot was not recorded"), + } + let counts = slots.counts(); + assert_eq!(counts.pool_running, 0); + assert_eq!(counts.pool_vacant, 1); + // A second finish of the same ticket is stale: the record is never doubled. + assert!(matches!( + slots.finish(ticket, Ok(()), None, now), + Finished::Stale + )); + assert_eq!(slots.snapshot(now).failures.len(), 1); +} From 50243b9c47a3aa7ef52205652fb5a0f1fc0609cf Mon Sep 17 00:00:00 2001 From: rain Date: Tue, 10 Nov 2026 00:54:23 +0800 Subject: [PATCH 2/2] refactor(547): structure the daemon slot custody for the recovery work (#547) Slice 2 (no behavior change): move RetainedFailures out of Slots into State, extract the pure disposition() classifier and capacity_report() read model, pass SlotClass instead of the reserved slot index to control_operation/hello, and delete the dead AdmissionClass::Lease, AdmissionReason::ReservedBusy and the three CAPACITY_OPERATION aliases (the bridge canonical OPERATION now stands alone). - owner.rs: Slots keeps only the slot table; State gains the retained store. A free disposition(&ConnectionFailure, SlotTicket, &str, Instant) -> Disposition builds the bounded record outside the lock; Slots::finish applies it. - connection.rs: SlotGuard records the retained record into state.retained; capacity_report(SlotCounts, RetainedSnapshot) -> CapacityModel is extracted; control_operation/hello take SlotClass; the CONNECTION_FAILURES alias is gone. - bridge/control/sandbox/control: AdmissionReason::ReservedBusy and the CAPACITY_OPERATION aliases are deleted; callers use OPERATION. - tests/slot_state.rs updated to the split Slots + RetainedFailures API. --- .../layerfs-api/tests/sandbox_capacity.rs | 4 +- .../crates/layerfs-control/tests/doctor.rs | 4 +- .../layerfs-sandbox/src/daemon/capacity.rs | 2 +- .../layerfs-sandbox/src/daemon/operator.rs | 2 +- .../crates/layerfs-sandbox/src/daemon/pool.rs | 2 +- .../crates/layerfs-sandbox/src/host/call.rs | 2 +- .../layerfs-sandbox/tests/capacity_read.rs | 6 +- engine/crates/layerfs-bridge/src/control.rs | 3 +- .../layerfs-bridge/src/duplex/admission.rs | 4 - .../src/application/connection.rs | 136 +++++++++--------- .../src/application/diagnostics.rs | 5 +- .../src/application/internals.rs | 3 +- .../layerfs-daemon/src/application/owner.rs | 88 +++++++----- .../src/application/retained.rs | 2 +- .../layerfs-daemon/tests/daemon_slots.rs | 4 +- .../crates/layerfs-daemon/tests/slot_state.rs | 78 +++++----- 16 files changed, 171 insertions(+), 174 deletions(-) diff --git a/api/crates/layerfs-api/tests/sandbox_capacity.rs b/api/crates/layerfs-api/tests/sandbox_capacity.rs index feb60971a2..b802f060c0 100644 --- a/api/crates/layerfs-api/tests/sandbox_capacity.rs +++ b/api/crates/layerfs-api/tests/sandbox_capacity.rs @@ -9,7 +9,7 @@ use layerfs_api::home::{self, paths, write_record}; use layerfs_bridge::control::{ AdmissionModel, Answer, Call, DaemonPhase, DaemonStatus, Reply, Request, RetainedModel, - SdkValue, CAPACITY_OPERATION, + SdkValue, OPERATION, }; use layerfs_bridge::duplex; use std::net::{SocketAddr, TcpListener}; @@ -107,7 +107,7 @@ fn serve(stream: std::net::TcpStream, model: &SdkValue, stall: bool) { Request::Sdk(sdk) if sdk.method == "sandbox.list_workspaces" => { Reply::Sdk(no_workspaces()) } - Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + Request::Sdk(sdk) if sdk.method == OPERATION => { if stall { // read but never answered: the read model's own fence, not // a closed socket, ends its wait diff --git a/control/crates/layerfs-control/tests/doctor.rs b/control/crates/layerfs-control/tests/doctor.rs index 7b3eeb9990..ab3a1bf223 100644 --- a/control/crates/layerfs-control/tests/doctor.rs +++ b/control/crates/layerfs-control/tests/doctor.rs @@ -6,7 +6,7 @@ //! error. use layerfs_bridge::control::{ AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request, - RetainedModel, SdkRefusal, SdkValue, CAPACITY_OPERATION, + RetainedModel, SdkRefusal, SdkValue, OPERATION, }; use layerfs_bridge::duplex; use layerfs_control::doctor::doctor; @@ -727,7 +727,7 @@ fn capacity_daemon(reply: Reply) -> (SocketAddr, std::thread::JoinHandle<()>) { let Request::Sdk(sdk) = call.request else { panic!("expected the capacity read model"); }; - assert_eq!(sdk.method, CAPACITY_OPERATION); + assert_eq!(sdk.method, OPERATION); channel .send .send(&Answer { id, reply }.encode().unwrap()) diff --git a/control/crates/layerfs-sandbox/src/daemon/capacity.rs b/control/crates/layerfs-sandbox/src/daemon/capacity.rs index 31b41b1556..d7cf617d45 100644 --- a/control/crates/layerfs-sandbox/src/daemon/capacity.rs +++ b/control/crates/layerfs-sandbox/src/daemon/capacity.rs @@ -24,7 +24,7 @@ use std::time::{Duration, Instant}; /// The daemon's operator read model of control admission and retained /// connection failures. -pub const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; +pub const OPERATION: &str = layerfs_bridge::control::OPERATION; /// How long one daemon query may take: connecting, its Hello and its one /// exchange. Short on purpose: a reader must not stall behind a daemon that diff --git a/control/crates/layerfs-sandbox/src/daemon/operator.rs b/control/crates/layerfs-sandbox/src/daemon/operator.rs index 325c506b1c..36f8ee74cd 100644 --- a/control/crates/layerfs-sandbox/src/daemon/operator.rs +++ b/control/crates/layerfs-sandbox/src/daemon/operator.rs @@ -11,7 +11,7 @@ pub fn call(control: &mut Control, method: &str, args: SdkValue) -> SdkResult { match method { // The daemon's control admission and bounded retained-failure store // (#540): an operator read model, never a product operation. - layerfs_bridge::control::CAPACITY_OPERATION => { + layerfs_bridge::control::OPERATION => { crate::daemon::call::exchange_call(control, method, args) } "sandbox.list_workspaces" => crate::daemon::call::exchange_call(control, method, args), diff --git a/control/crates/layerfs-sandbox/src/daemon/pool.rs b/control/crates/layerfs-sandbox/src/daemon/pool.rs index e7eab3a0be..ad03cb7001 100644 --- a/control/crates/layerfs-sandbox/src/daemon/pool.rs +++ b/control/crates/layerfs-sandbox/src/daemon/pool.rs @@ -108,7 +108,7 @@ fn not_admitted(refusal: &SdkRefusal) -> Option<(NotAdmitted, &'static str)> { /// precise form: it names the resource and the retained record count. fn notice_refusal(notice: AdmissionNotice) -> SdkRefusal { match notice.reason { - AdmissionReason::NoPoolSlot | AdmissionReason::ReservedBusy => SdkRefusal { + AdmissionReason::NoPoolSlot => SdkRefusal { code: "Capacity".to_owned(), message: format!( "the Sandbox's daemon has no vacant control slot: {} pool slot(s) and {} reserved slot(s) are vacant, with {} retained failure record(s). The call was not admitted; retrying is safe", diff --git a/control/crates/layerfs-sandbox/src/host/call.rs b/control/crates/layerfs-sandbox/src/host/call.rs index 92ced5f1d5..d5d1cf6989 100644 --- a/control/crates/layerfs-sandbox/src/host/call.rs +++ b/control/crates/layerfs-sandbox/src/host/call.rs @@ -42,7 +42,7 @@ pub const OPS: [(Op, &str); 10] = [ (Op::SandboxCreate, "sandbox.create"), (Op::SandboxClose, "sandbox.close"), (Op::ListWorkspaces, "sandbox.list_workspaces"), - (Op::Capacity, layerfs_bridge::control::CAPACITY_OPERATION), + (Op::Capacity, layerfs_bridge::control::OPERATION), (Op::Mount, "workspace.mount"), (Op::Exec, "workspace.exec"), (Op::ExecCancel, "workspace.exec_cancel"), diff --git a/control/crates/layerfs-sandbox/tests/capacity_read.rs b/control/crates/layerfs-sandbox/tests/capacity_read.rs index b90709a375..05b133b006 100644 --- a/control/crates/layerfs-sandbox/tests/capacity_read.rs +++ b/control/crates/layerfs-sandbox/tests/capacity_read.rs @@ -8,7 +8,7 @@ //! accepts, and counts what it saw. use layerfs_bridge::control::{ AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request, - RetainedModel, SdkValue, CAPACITY_OPERATION, + RetainedModel, SdkValue, OPERATION, }; use layerfs_bridge::duplex; use layerfs_home::home::paths; @@ -138,7 +138,7 @@ fn serve(stream: TcpStream, answer: Behavior, seen: &Seen) { seen.ended.store(true, Ordering::SeqCst); Reply::SessionEnded } - Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + Request::Sdk(sdk) if sdk.method == OPERATION => { seen.queries.fetch_add(1, Ordering::SeqCst); if matches!(answer, Behavior::Trickle) { if let Some(raw) = raw.as_mut() { @@ -337,7 +337,7 @@ fn a_quarantined_connection_attempts_nothing() { lease.close_handle().unwrap().close().unwrap(); // the call a pooled lease would make next: refused before sending, so it // is not an attempt whose effect is unknown - let error = operator::call(&mut lease, CAPACITY_OPERATION, SdkValue::Null).unwrap_err(); + let error = operator::call(&mut lease, OPERATION, SdkValue::Null).unwrap_err(); match error { SdkError::Operation(failure) => match failure.cause { OperationCause::Exchange(channel) => { diff --git a/engine/crates/layerfs-bridge/src/control.rs b/engine/crates/layerfs-bridge/src/control.rs index cb8dd79c5c..3a63b48147 100644 --- a/engine/crates/layerfs-bridge/src/control.rs +++ b/engine/crates/layerfs-bridge/src/control.rs @@ -165,8 +165,7 @@ fn boolean(value: u8) -> Result { } pub use crate::control_capacity::{ - AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, - OPERATION as CAPACITY_OPERATION, + AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, OPERATION, }; pub use crate::control_forced::{ AbortDisposition, CommitKnowledge, DetachDisposition, ForceUnmounted, ForcedCleanup, diff --git a/engine/crates/layerfs-bridge/src/duplex/admission.rs b/engine/crates/layerfs-bridge/src/duplex/admission.rs index 820ef6e88a..ee500342da 100644 --- a/engine/crates/layerfs-bridge/src/duplex/admission.rs +++ b/engine/crates/layerfs-bridge/src/duplex/admission.rs @@ -25,8 +25,6 @@ pub enum AdmissionReason { NoPoolSlot = 1, /// The daemon is still starting and admits no control yet. Starting = 2, - /// The reserved slot is in use and only pool capacity was requested. - ReservedBusy = 3, } impl AdmissionReason { @@ -34,7 +32,6 @@ impl AdmissionReason { match byte { 1 => Some(Self::NoPoolSlot), 2 => Some(Self::Starting), - 3 => Some(Self::ReservedBusy), _ => None, } } @@ -44,7 +41,6 @@ impl AdmissionReason { match self { Self::NoPoolSlot => "no-pool-slot", Self::Starting => "starting", - Self::ReservedBusy => "reserved-busy", } } } diff --git a/engine/crates/layerfs-daemon/src/application/connection.rs b/engine/crates/layerfs-daemon/src/application/connection.rs index 57dc1e941a..2b0a15a6be 100644 --- a/engine/crates/layerfs-daemon/src/application/connection.rs +++ b/engine/crates/layerfs-daemon/src/application/connection.rs @@ -1,7 +1,7 @@ //! Bounded real authenticated connection workers and retained original failures. use super::{ - owner::{AdmissionClass, Finished, SlotClass, SlotTicket}, - retained::RetainedFailure, + owner::{disposition, AdmissionClass, Finished, SlotClass, SlotCounts, SlotTicket}, + retained::RetainedSnapshot, Application, ApplicationError, ConnectionFailure, }; use layerfs_bridge::{ @@ -80,15 +80,12 @@ impl SlotGuard { return; } self.done = true; - // Build the bounded record before taking the state lock. - let prebuilt = result.as_ref().err().map(|failure| { - RetainedFailure::describe( - failure, - self.ticket.class, - self.ticket.generation, - &self.owner, - ) - }); + let now = Instant::now(); + // Classify and build the bounded record before taking the state lock. + let classified = result + .as_ref() + .err() + .map(|failure| disposition(failure, self.ticket, &self.owner, now)); // Poison recovery here only releases the slot and records; normal // admission still reports `Poisoned` rather than trusting corrupt state. let finished = { @@ -97,9 +94,14 @@ impl SlotGuard { .state .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let finished = state - .slots - .finish(self.ticket, result, prebuilt, Instant::now()); + let finished = state.slots.finish(self.ticket, result, classified); + let finished = match finished { + Finished::Retained(record) => { + let recorded = state.retained.record(*record, now); + Finished::Retained(Box::new(recorded)) + } + other => other, + }; self.application.changed.notify_all(); finished }; @@ -161,8 +163,8 @@ impl Application { Some(ticket) => ticket, None => { let counts = state.slots.counts(); - let retained = state.slots.visible(); - state.slots.reap(Instant::now()); + let retained = state.retained.len(); + state.retained.reap(Instant::now()); let reason = if state.startup.phase != DaemonPhase::ControlReady { AdmissionReason::Starting } else { @@ -332,9 +334,9 @@ impl Application { let bodies = super::bodies::Bodies::of(&call.request, &mut connection.receive); let (outcome, diagnostic_error, telemetry) = if matches!(call.request, Request::Observed { .. }) { - self.observed_control(&call, slot) + self.observed_control(&call, slot, ticket.class) } else { - (self.control(&call, slot), None, None) + (self.control(&call, ticket.class), None, None) }; if let Err(cause) = bodies.finish() { return Err(ConnectionFailure { @@ -446,9 +448,9 @@ impl Application { fn control( &self, call: &Call, - slot: usize, + class: SlotClass, ) -> Result { - self.control_operation(&call.request, slot) + self.control_operation(&call.request, class) } /// After the daemon reported ready (a `Hello` answered ControlReady was /// sent), opens one idle Store reader on a thread of its own, so the @@ -494,7 +496,7 @@ impl Application { pub(super) fn control_operation( &self, request: &Request, - slot: usize, + class: SlotClass, ) -> Result { use crate::control::{Failure, Success}; use layerfs_bridge::control::{ControlCode, DaemonPhase}; @@ -503,13 +505,11 @@ impl Application { } if let Request::Hello(request) = request { return self - .hello(*request, slot) + .hello(*request, class) .map(|status| Success::reply(Reply::Hello(status))); } let state = self.state.lock().map_err(|_| Failure::Poisoned)?; - if slot == usize::from(self.setup.limits.connections) - && state.startup.phase != DaemonPhase::ControlReady - { + if class == SlotClass::Reserved && state.startup.phase != DaemonPhase::ControlReady { return Err(Failure::Rejected( ControlCode::Capacity, "reserved installer slot", @@ -521,7 +521,7 @@ impl Application { // Operator-visible retained failures live with the application, not // with the Store control Service, so answer this one here. if let Request::Sdk(call) = request { - if call.method == CONNECTION_FAILURES { + if call.method == layerfs_bridge::control::OPERATION { return self .connection_failures() .map(|value| Success::reply(Reply::Sdk(value))); @@ -545,47 +545,53 @@ impl Application { &self, ) -> Result { use crate::control::Failure; - use layerfs_bridge::control::{ - AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, - }; let counts = self.connection_slots().map_err(|_| Failure::Poisoned)?; let snapshot = self .retained_connection_failures() .map_err(|_| Failure::Poisoned)?; - let failures = snapshot - .failures - .iter() - .map(|failure| RetainedFailureModel { - slot: failure.slot as i64, - class: failure.class.to_owned(), - generation: failure.generation as i64, - owner: failure.owner.clone(), - received_bytes: failure.received_bytes.map(|bytes| bytes as i64), - request: failure.request.map(str::to_owned), - reply_sent: failure.reply_sent, - cause: failure.cause.clone(), - fence_error: failure.fence_error.clone(), - diagnostic_error: failure.diagnostic_error.clone(), - age_ms: failure.age.as_millis() as i64, - expires_in_ms: failure.expires_in.as_millis() as i64, - }); - Ok(CapacityModel { - admission: AdmissionModel { - pool_vacant: counts.pool_vacant as i64, - pool_running: counts.pool_running as i64, - reserved_vacant: counts.reserved_vacant as i64, - reserved_running: counts.reserved_running as i64, - }, - retained: RetainedModel { - limit: snapshot.limit as i64, - ttl_ms: snapshot.ttl.as_millis() as i64, - count: snapshot.failures.len() as i64, - dropped: snapshot.dropped as i64, - recorded: snapshot.recorded as i64, - failures: failures.collect(), - }, - } - .to_sdk()) + Ok(capacity_report(counts, snapshot).to_sdk()) + } +} + +/// The daemon's control-capacity read model from its slot counts and its +/// bounded retained-failure snapshot (#540). Pure: no lock and no store. +fn capacity_report( + counts: SlotCounts, + snapshot: RetainedSnapshot, +) -> layerfs_bridge::control::CapacityModel { + use layerfs_bridge::control::{AdmissionModel, RetainedFailureModel, RetainedModel}; + let failures = snapshot + .failures + .iter() + .map(|failure| RetainedFailureModel { + slot: failure.slot as i64, + class: failure.class.to_owned(), + generation: failure.generation as i64, + owner: failure.owner.clone(), + received_bytes: failure.received_bytes.map(|bytes| bytes as i64), + request: failure.request.map(str::to_owned), + reply_sent: failure.reply_sent, + cause: failure.cause.clone(), + fence_error: failure.fence_error.clone(), + diagnostic_error: failure.diagnostic_error.clone(), + age_ms: failure.age.as_millis() as i64, + expires_in_ms: failure.expires_in.as_millis() as i64, + }); + layerfs_bridge::control::CapacityModel { + admission: AdmissionModel { + pool_vacant: counts.pool_vacant as i64, + pool_running: counts.pool_running as i64, + reserved_vacant: counts.reserved_vacant as i64, + reserved_running: counts.reserved_running as i64, + }, + retained: RetainedModel { + limit: snapshot.limit as i64, + ttl_ms: snapshot.ttl.as_millis() as i64, + count: snapshot.failures.len() as i64, + dropped: snapshot.dropped as i64, + recorded: snapshot.recorded as i64, + failures: failures.collect(), + }, } } @@ -605,7 +611,3 @@ fn reported_ready(outcome: &Result, ); impl Application { - pub(super) fn observed_control(&self, call: &Call, slot: usize) -> Observed { + pub(super) fn observed_control(&self, call: &Call, slot: usize, class: SlotClass) -> Observed { let operation = match call.request.without_observation() { Ok(operation) => operation, Err(_) => { @@ -195,7 +196,7 @@ impl Application { let saves_before = saves.as_ref().map(|store| store.save_counters()); let observed_before = observing.elapsed(); let recording = traced.then(crate::telemetry::Recording::start); - let outcome = self.control_operation(operation, slot); + let outcome = self.control_operation(operation, class); let reports = recording.map(crate::telemetry::Recording::finish); let observed_after = std::time::Instant::now(); let sql = sessions diff --git a/engine/crates/layerfs-daemon/src/application/internals.rs b/engine/crates/layerfs-daemon/src/application/internals.rs index dd7201b71c..19b0d5f6c2 100644 --- a/engine/crates/layerfs-daemon/src/application/internals.rs +++ b/engine/crates/layerfs-daemon/src/application/internals.rs @@ -11,7 +11,8 @@ #[doc(hidden)] pub use super::owner::{ - AdmissionClass, Finished, SlotClass, SlotCounts, SlotTicket, Slots, RESERVED_SLOTS, + disposition, AdmissionClass, Disposition, Finished, SlotClass, SlotCounts, SlotTicket, Slots, + RESERVED_SLOTS, }; #[doc(hidden)] pub use super::retained::{RetainedFailure, RetainedFailures, RetainedSnapshot, LIMIT, TTL}; diff --git a/engine/crates/layerfs-daemon/src/application/owner.rs b/engine/crates/layerfs-daemon/src/application/owner.rs index 3f7956d2a8..f3418d5a93 100644 --- a/engine/crates/layerfs-daemon/src/application/owner.rs +++ b/engine/crates/layerfs-daemon/src/application/owner.rs @@ -52,16 +52,13 @@ pub enum AdmissionClass { ControlPool, /// The one-time Store installer. Install, - /// A future presence lease. - #[allow(dead_code)] - Lease, } impl AdmissionClass { fn slot_class(self) -> SlotClass { match self { Self::ControlPool => SlotClass::Pool, - Self::Install | Self::Lease => SlotClass::Reserved, + Self::Install => SlotClass::Reserved, } } } @@ -104,10 +101,11 @@ pub struct SlotCounts { pub reserved_running: usize, } -/// The fixed control-slot table and the bounded retained-failure store beside it. +/// The fixed control-slot table; the bounded retained-failure store lives +/// beside it in [`State`]. /// /// A retained failure never owns a slot: `finish` always leaves it `Vacant` -/// and only records the failure. Admission capacity therefore cannot leak +/// and only describes the failure. Admission capacity therefore cannot leak /// (#540). The layout is `connections` `Pool` entries followed by /// [`RESERVED_SLOTS`] `Reserved` entries; a class only ever admits within its /// own range. @@ -115,7 +113,6 @@ pub struct Slots { slots: Vec, pool: usize, next_generation: u64, - retained: RetainedFailures, } impl Slots { @@ -128,7 +125,6 @@ impl Slots { slots, pool, next_generation: 0, - retained: RetainedFailures::default(), } } @@ -161,15 +157,14 @@ impl Slots { }) } - /// Ends one worker: release its slot and, for a retained failure, record - /// the already-built record. A ticket whose generation or class no longer - /// matches changes nothing. + /// Ends one worker: releases its slot and, for a retained failure, returns + /// the already-described record for the caller to store. A ticket whose + /// generation or class no longer matches changes nothing. pub fn finish( &mut self, ticket: SlotTicket, result: Result<(), ConnectionFailure>, - prebuilt: Option, - now: Instant, + disposition: Option, ) -> Finished { let running = matches!( self.slots.get(ticket.index), @@ -182,21 +177,14 @@ impl Slots { self.slots[ticket.index] = SlotState::Vacant; match result { Ok(()) => Finished::Clean, - Err(failure) if releases_after_logging(&failure) => { - Finished::Released(Box::new(failure)) - } - Err(_) => { - let record = prebuilt.expect("a retained failure is described before the lock"); - let record = self.retained.record(record, now); - Finished::Retained(Box::new(record)) - } + Err(failure) => match disposition { + Some(Disposition::Released) => Finished::Released(Box::new(failure)), + Some(Disposition::Retained(record)) => Finished::Retained(record), + None => unreachable!("a finished failure is classified before the lock"), + }, } } - pub fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { - self.retained.snapshot(now) - } - /// Admission counts for each class. pub fn counts(&self) -> SlotCounts { let mut counts = SlotCounts { @@ -215,16 +203,6 @@ impl Slots { } counts } - - /// Visible retained records, for the admission notice. - pub fn visible(&mut self) -> usize { - self.retained.len() - } - - /// Drops expired records; the accept loop reaps when it refuses admission. - pub fn reap(&mut self, now: Instant) { - self.retained.reap(now); - } } /// A failure with no decoded product attempt or custody: the existing daemon @@ -237,6 +215,38 @@ fn releases_after_logging(failure: &ConnectionFailure) -> bool { ApplicationError::Channel(_) | ApplicationError::Io(_) ) } + +/// The terminal disposition of one finished connection failure. +#[derive(Debug)] +pub enum Disposition { + /// Logged only: the slot is already free and no record is kept. + Released, + /// A bounded retained record is kept in the store. + Retained(Box), +} + +/// Classifies one finished failure and builds the bounded record for a +/// retained one. Pure: no lock and no store access. `owner` names the worker +/// in a retained record; `now` is reserved for the in-flight deadline a later +/// slice adds. +pub fn disposition( + failure: &ConnectionFailure, + ticket: SlotTicket, + owner: &str, + _now: Instant, +) -> Disposition { + if releases_after_logging(failure) { + Disposition::Released + } else { + Disposition::Retained(Box::new(RetainedFailure::describe( + failure, + ticket.class, + ticket.generation, + owner, + ))) + } +} + pub(super) struct Startup { pub phase: DaemonPhase, pub service: Option>, @@ -247,6 +257,7 @@ pub(super) struct Startup { pub(super) struct State { pub startup: Startup, pub slots: Slots, + pub retained: RetainedFailures, } /// One real daemon application; Store/Overlay are initialized once and reused. pub struct Application { @@ -413,6 +424,7 @@ impl Application { install_failure: None, }, slots: Slots::new(count), + retained: RetainedFailures::default(), }), changed: Condvar::new(), workers: Mutex::new((0..count).map(|_| None).collect()), @@ -447,12 +459,12 @@ impl Application { pub(super) fn hello( &self, request: HelloRequest, - slot: usize, + class: SlotClass, ) -> Result { use crate::control::Failure; use layerfs_bridge::control::ControlCode; let mut state = self.state.lock().map_err(|_| Failure::Poisoned)?; - if slot == usize::from(self.setup.limits.connections) + if class == SlotClass::Reserved && matches!( state.startup.phase, DaemonPhase::InstallPending | DaemonPhase::Installing @@ -485,7 +497,7 @@ impl Application { /// The bounded retained-failure store, pruned at this observation. pub fn retained_connection_failures(&self) -> Result { let mut state = self.state.lock().map_err(|_| ApplicationError::Poisoned)?; - Ok(state.slots.snapshot(Instant::now())) + Ok(state.retained.snapshot(Instant::now())) } /// Vacant and in-use control slots, by class. diff --git a/engine/crates/layerfs-daemon/src/application/retained.rs b/engine/crates/layerfs-daemon/src/application/retained.rs index 5de1641ca9..c08a350cc7 100644 --- a/engine/crates/layerfs-daemon/src/application/retained.rs +++ b/engine/crates/layerfs-daemon/src/application/retained.rs @@ -213,7 +213,7 @@ impl RetainedFailures { } } - pub fn len(&self) -> usize { + pub(super) fn len(&self) -> usize { self.entries.len() } } diff --git a/engine/crates/layerfs-daemon/tests/daemon_slots.rs b/engine/crates/layerfs-daemon/tests/daemon_slots.rs index c2a1ee6cef..5655480ec5 100644 --- a/engine/crates/layerfs-daemon/tests/daemon_slots.rs +++ b/engine/crates/layerfs-daemon/tests/daemon_slots.rs @@ -15,7 +15,7 @@ use layerfs_bridge::{ control::{ Answer, Call, CapacityModel, DaemonPhase, HelloRequest, Reply, Request, SdkCall, SdkValue, - CAPACITY_OPERATION, + OPERATION, }, daemon_setup::{DaemonLimits, DaemonSetup}, duplex::{self, ChannelError, Connection}, @@ -190,7 +190,7 @@ fn capacity(address: SocketAddr) -> CapacityModel { let call = Call { id: 7, request: Request::Sdk(SdkCall { - method: CAPACITY_OPERATION.to_owned(), + method: OPERATION.to_owned(), args: SdkValue::Null, }), }; diff --git a/engine/crates/layerfs-daemon/tests/slot_state.rs b/engine/crates/layerfs-daemon/tests/slot_state.rs index c9699147cb..9eac8cb342 100644 --- a/engine/crates/layerfs-daemon/tests/slot_state.rs +++ b/engine/crates/layerfs-daemon/tests/slot_state.rs @@ -9,13 +9,15 @@ use layerfs_bridge::control::{Call, Reply, Request}; use layerfs_bridge::duplex::{ChannelError, ChannelWork, PeerEndFailure, PeerEndPhase}; use layerfs_daemon::application::internals::{ - AdmissionClass, Finished, RetainedFailure, RetainedFailures, SlotClass, Slots, LIMIT, - RESERVED_SLOTS, TTL, + disposition, AdmissionClass, Finished, RetainedFailure, RetainedFailures, SlotClass, Slots, + LIMIT, RESERVED_SLOTS, TTL, }; use layerfs_daemon::application::{ApplicationError, ConnectionFailure}; use layerfs_daemon::control::{Served, Success}; use std::time::{Duration, Instant}; +const OWNER: &str = "daemon:pid:layerfs-control-0"; + /// A failure after a decoded request: retained, never released. fn failure(slot: usize) -> ConnectionFailure { ConnectionFailure { @@ -29,12 +31,7 @@ fn failure(slot: usize) -> ConnectionFailure { } fn describe(slot: usize, class: SlotClass, generation: u64) -> RetainedFailure { - RetainedFailure::describe( - &failure(slot), - class, - generation, - "daemon:pid:layerfs-control-0", - ) + RetainedFailure::describe(&failure(slot), class, generation, OWNER) } /// A pool admission never takes the reserved slot and vice versa. @@ -52,7 +49,6 @@ fn admit_uses_only_its_class() { let reserved = slots.admit(AdmissionClass::Install).unwrap(); assert_eq!(reserved.index, 8); assert_eq!(reserved.class, SlotClass::Reserved); - assert_eq!(slots.admit(AdmissionClass::Lease), None); let counts = slots.counts(); assert_eq!(counts.pool_vacant, 0); assert_eq!(counts.pool_running, 8); @@ -61,22 +57,24 @@ fn admit_uses_only_its_class() { } /// A retained failure is a record, not a slot: finishing the worker frees the -/// slot before the record is stored (#540). +/// slot, and the bounded record is stored beside the slots (#540). #[test] fn finish_frees_even_for_a_retained_failure() { let mut slots = Slots::new(8 + RESERVED_SLOTS); + let mut store = RetainedFailures::default(); let now = Instant::now(); let ticket = slots.admit(AdmissionClass::ControlPool).unwrap(); - let prebuilt = describe(ticket.index, ticket.class, ticket.generation); - match slots.finish(ticket, Err(failure(ticket.index)), Some(prebuilt), now) { + let classified = disposition(&failure(ticket.index), ticket, OWNER, now); + match slots.finish(ticket, Err(failure(ticket.index)), Some(classified)) { Finished::Retained(record) => { assert_eq!(record.class, "pool"); assert_eq!(record.generation, ticket.generation); + store.record(*record, now); } _ => panic!("a decoded failure was not retained"), } assert_eq!(slots.counts().pool_vacant, 8); - assert_eq!(slots.snapshot(now).failures.len(), 1); + assert_eq!(store.snapshot(now).failures.len(), 1); } /// Every running slot ends exactly once; a duplicate or stale finish releases @@ -86,27 +84,17 @@ fn stale_or_duplicate_finish_changes_nothing() { let mut slots = Slots::new(8 + RESERVED_SLOTS); let now = Instant::now(); let first = slots.admit(AdmissionClass::ControlPool).unwrap(); - slots.finish( - first, - Err(failure(first.index)), - Some(describe(first.index, first.class, first.generation)), - now, - ); + let classified = disposition(&failure(first.index), first, OWNER, now); + slots.finish(first, Err(failure(first.index)), Some(classified)); // A duplicate finish of the same ticket changes nothing. - assert!(matches!( - slots.finish(first, Ok(()), None, now), - Finished::Stale - )); + assert!(matches!(slots.finish(first, Ok(()), None), Finished::Stale)); let second = slots.admit(AdmissionClass::ControlPool).unwrap(); assert_eq!(second.index, first.index); assert!(second.generation > first.generation); // The old ticket cannot free the newer worker. - assert!(matches!( - slots.finish(first, Ok(()), None, now), - Finished::Stale - )); + assert!(matches!(slots.finish(first, Ok(()), None), Finished::Stale)); assert_eq!(slots.counts().pool_running, 1); - slots.finish(second, Ok(()), None, now); + slots.finish(second, Ok(()), None); assert_eq!(slots.counts().pool_vacant, 8); } @@ -119,8 +107,8 @@ fn ring_limit_evicts_oldest_and_counts_drops() { for slot in 0..5 { store.record(describe(slot, SlotClass::Pool, slot as u64 + 1), now); } - assert_eq!(store.len(), 3); let snapshot = store.snapshot(now); + assert_eq!(snapshot.failures.len(), 3); assert_eq!(snapshot.dropped, 2); assert_eq!(snapshot.recorded, 5); assert_eq!( @@ -162,17 +150,18 @@ fn ttl_prunes_expired_records() { #[test] fn retained_store_reaps_expired_records() { let mut slots = Slots::new(1 + RESERVED_SLOTS); + let mut store = RetainedFailures::default(); let now = Instant::now(); let ticket = slots.admit(AdmissionClass::ControlPool).unwrap(); - slots.finish( - ticket, - Err(failure(ticket.index)), - Some(describe(ticket.index, ticket.class, ticket.generation)), - now, - ); - slots.reap(now + TTL + Duration::from_secs(1)); + let classified = disposition(&failure(ticket.index), ticket, OWNER, now); + if let Finished::Retained(record) = + slots.finish(ticket, Err(failure(ticket.index)), Some(classified)) + { + store.record(*record, now); + } + store.reap(now + TTL + Duration::from_secs(1)); assert_eq!( - slots + store .snapshot(now + TTL + Duration::from_secs(1)) .failures .len(), @@ -277,6 +266,7 @@ fn owner_and_deadline_are_visible() { #[test] fn guard_drop_releases_the_slot_and_records_once() { let mut slots = Slots::new(1 + RESERVED_SLOTS); + let mut store = RetainedFailures::default(); let now = Instant::now(); let ticket = slots.admit(AdmissionClass::ControlPool).unwrap(); let failure = ConnectionFailure { @@ -287,13 +277,8 @@ fn guard_drop_releases_the_slot_and_records_once() { fence_error: None, diagnostic_error: None, }; - let record = RetainedFailure::describe( - &failure, - ticket.class, - ticket.generation, - "daemon:pid:layerfs-control-0", - ); - match slots.finish(ticket, Err(failure), Some(record), now) { + let classified = disposition(&failure, ticket, OWNER, now); + match slots.finish(ticket, Err(failure), Some(classified)) { Finished::Retained(record) => { assert_eq!(record.slot, ticket.index); assert_eq!(record.class, "pool"); @@ -303,6 +288,7 @@ fn guard_drop_releases_the_slot_and_records_once() { "{}", record.cause ); + store.record(*record, now); } _ => panic!("an abandoned slot was not recorded"), } @@ -311,8 +297,8 @@ fn guard_drop_releases_the_slot_and_records_once() { assert_eq!(counts.pool_vacant, 1); // A second finish of the same ticket is stale: the record is never doubled. assert!(matches!( - slots.finish(ticket, Ok(()), None, now), + slots.finish(ticket, Ok(()), None), Finished::Stale )); - assert_eq!(slots.snapshot(now).failures.len(), 1); + assert_eq!(store.snapshot(now).failures.len(), 1); }