diff --git a/api/crates/layerfs-api/src/sandbox/get.rs b/api/crates/layerfs-api/src/sandbox/get.rs index f1e05b8467..2839ca84b8 100644 --- a/api/crates/layerfs-api/src/sandbox/get.rs +++ b/api/crates/layerfs-api/src/sandbox/get.rs @@ -5,7 +5,7 @@ use crate::calls::Context; use layerfs_bridge::control::SdkValue; use layerfs_home::home::with_field; use layerfs_home::outcome::{Outcome, View}; -use layerfs_sandbox::family::{get, GetRequest, Page}; +use layerfs_sandbox::family::{get, GetRequest, Got, Page}; /// Its table entry. pub(crate) const CALL: Call = call( @@ -17,9 +17,11 @@ pub(crate) const CALL: Call = call( run, ); -/// Runs `sandbox.get`: its record with its derived `state`, one page of -/// its `workspaces` (each with its `path`, `name` and `ws`), why its -/// machine did not answer (`down_reason`) and `next_page_pos`. +/// Runs `sandbox.get`: its record with its derived `state`, one page of its +/// `workspaces` (each with its `path`, `name` and `ws`), why its machine did +/// not answer (`down_reason`), the daemon's `daemon_capacity` read model when +/// the Sandbox answered it (#540), and `next_page_pos`. The library reads it +/// best effort and bounded, so this layer only shows what it got. fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { let request = GetRequest { name: args.get_text("name").unwrap_or("").to_owned(), @@ -29,18 +31,28 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { }, }; let (home, events) = context.parts(); - let got = get(home, events, &request)?; - let workspaces = got.workspaces.into_iter().map(|listed| { + let Got { + record, + state, + workspaces, + down_reason, + next, + daemon_capacity, + } = get(home, events, &request)?; + let workspaces = workspaces.into_iter().map(|listed| { let item = with_field(listed.item, "path", SdkValue::Text(listed.path)); let item = with_field(item, "name", SdkValue::opt_text(listed.name)); with_field(item, "ws", SdkValue::Text(listed.ws)) }); - let shown = got.record.to_sdk(View::Full); - let shown = with_field(shown, "state", SdkValue::text(got.state)); + let shown = record.to_sdk(View::Full); + let shown = with_field(shown, "state", SdkValue::text(state)); let mut shown = with_field(shown, "workspaces", SdkValue::List(workspaces.collect())); - if let Some(why) = got.down_reason { + if let Some(why) = down_reason { shown = with_field(shown, "down_reason", SdkValue::Text(why)); } - let next = super::list::next_page_pos(got.next); + if let Some(model) = daemon_capacity { + shown = with_field(shown, "daemon_capacity", model.to_sdk()); + } + let next = super::list::next_page_pos(next); Ok(with_field(shown, "next_page_pos", next)) } diff --git a/api/crates/layerfs-api/tests/sandbox_capacity.rs b/api/crates/layerfs-api/tests/sandbox_capacity.rs new file mode 100644 index 0000000000..feb60971a2 --- /dev/null +++ b/api/crates/layerfs-api/tests/sandbox_capacity.rs @@ -0,0 +1,365 @@ +//! `sandbox.get`'s `daemon_capacity` (#540, #543): the daemon's +//! `backend.daemon.connection_failures` read model, shown best-effort and +//! bounded on the connection the Workspace list lent from this process's +//! pool. Nothing runs: a stand-in daemon answers the Sandbox's Workspace list +//! and its capacity model on the same connection, a second one answers the +//! list and then says nothing, and a Sandbox whose machine or forward is +//! already down is not dialed at all. +#![cfg(unix)] +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, DaemonPhase, DaemonStatus, Reply, Request, RetainedModel, + SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; +use std::net::{SocketAddr, TcpListener}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("lfs-scap-{tag}-{}-{sequence}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +/// What the stand-in daemon answers `sandbox.list_workspaces` with: no +/// Workspaces, like a Sandbox nothing has mounted in. +fn no_workspaces() -> SdkValue { + SdkValue::map([ + ("kind", t("workspaces")), + ("version", SdkValue::Int(1)), + ("workspaces", SdkValue::List(vec![])), + ]) +} + +/// A stand-in daemon at its own loopback port, and how many connections it +/// served: it answers this Sandbox's Workspace list and its capacity model +/// with `model` on the connection it was given. With `stall` it answers the +/// Workspace list, reads the capacity query and never answers it, which is +/// what a daemon stuck behind its own slots looks like. +fn daemon(model: SdkValue, stall: bool) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let model = Arc::new(model); + let served = Arc::new(AtomicUsize::new(0)); + let counted = served.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + counted.fetch_add(1, Ordering::SeqCst); + let model = model.clone(); + // one worker per connection: the pool keeps one and reuses it + std::thread::spawn(move || serve(stream, &model, stall)); + } + }); + (endpoint, served) +} + +/// One stand-in connection: its Hello, then each call's answer until the +/// caller closes it. +fn serve(stream: std::net::TcpStream, model: &SdkValue, stall: bool) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + let Ok(mut channel) = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) else { + return; + }; + while let Ok(bytes) = channel.receive.receive() { + let Ok(call) = Call::decode(bytes) else { break }; + let id = call.id; + let reply = match call.request { + Request::Hello(_) => Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + Request::Sdk(sdk) if sdk.method == "sandbox.list_workspaces" => { + Reply::Sdk(no_workspaces()) + } + Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + if stall { + // read but never answered: the read model's own fence, not + // a closed socket, ends its wait + std::thread::sleep(Duration::from_secs(6)); + break; + } + Reply::Sdk(model.clone()) + } + _ => continue, + }; + if channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .is_err() + { + break; + } + } +} + +/// A record from `fields`, in order. +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +/// The home's machine, Store `shop` and project `api`; the machine's `worker` +/// record only while it runs (its absence is a stopped machine, #535 D11). +fn home_records(home: &Path, worker: bool) { + write_record( + &paths::machine_record(home, "native"), + &map(&[("name", t("native")), ("platform", t("native"))]), + ) + .unwrap(); + if worker { + // the forward's machine answers: a forward with no recorded process is + // not checked against one + write_record( + &paths::worker_record(home, "native"), + &map(&[("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + } + write_record( + &paths::store_record(home, "shop"), + &map(&[ + ("name", t("shop")), + ("kind", t("sqlite")), + ("binding", SdkValue::Bytes(vec![7; 16])), + ("cursor", SdkValue::Bytes(vec![8; 32])), + ("profile", t("disposable")), + ("machine", t("native")), + ]), + ) + .unwrap(); + write_record( + &paths::project_record(home, "shop", "api"), + &map(&[("name", t("api"))]), + ) + .unwrap(); +} + +/// Sandbox `s1` on `native`, recorded with its `id` (`None`: gone, its create +/// never recorded one) whose daemon this host reaches through the forward to +/// `endpoint`, carried by forward process `pid` (`None`: a port nothing +/// carries). +fn sandbox_record(home: &Path, endpoint: SocketAddr, id: Option<&str>, pid: Option) { + let mut forward = vec![ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]; + if let Some(pid) = pid { + forward.push(("pid", SdkValue::Int(pid))); + } + let mut fields = vec![ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ]; + if let Some(id) = id { + fields.push(("id", t(id))); + } + fields.push(( + "daemon", + map(&[("name", t("lfs-x-s1")), ("forward", map(&forward))]), + )); + write_record(&paths::sandbox_record(home, "s1"), &map(&fields)).unwrap(); + write_record( + &paths::sandbox_control(home, "s1"), + &map(&[ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]), + ) + .unwrap(); +} + +/// A model with no vacant pool slot and one retained failure. +fn busy() -> SdkValue { + let model = layerfs_bridge::control::CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel { + limit: 64, + ttl_ms: 3_600_000, + count: 1, + dropped: 0, + recorded: 1, + failures: vec![layerfs_bridge::control::RetainedFailureModel { + slot: 0, + class: "pool".into(), + generation: 6, + owner: "daemon instance, pid 4, thread layerfs-control-0".into(), + received_bytes: None, + request: Some("unmount".into()), + reply_sent: false, + cause: "control: the kernel control did not answer".into(), + fence_error: None, + diagnostic_error: None, + age_ms: 12, + expires_in_ms: 3_599_988, + }], + }, + }; + model.to_sdk() +} + +#[test] +fn a_running_sandbox_s_get_shows_its_daemon_capacity() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("shown"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, Some("s-1"), None); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + let capacity = got + .get("daemon_capacity") + .unwrap_or_else(|| panic!("{got:?}")); + let admission = capacity.get("admission").unwrap(); + assert_eq!(admission.get_int("pool_vacant"), Some(0), "{capacity:?}"); + assert_eq!(admission.get_int("pool_running"), Some(8), "{capacity:?}"); + assert_eq!( + admission.get_int("reserved_vacant"), + Some(1), + "{capacity:?}" + ); + let retained = capacity.get("retained").unwrap(); + assert_eq!(retained.get_int("count"), Some(1), "{capacity:?}"); + assert_eq!(retained.get_int("limit"), Some(64), "{capacity:?}"); + let Some(SdkValue::List(failures)) = retained.get("failures") else { + panic!("{capacity:?}") + }; + assert_eq!(failures.len(), 1, "{capacity:?}"); + assert_eq!( + failures[0].get_text("cause"), + Some("control: the kernel control did not answer") + ); + assert_eq!(failures[0].get_int("generation"), Some(6)); + assert_eq!(failures[0].get_text("request"), Some("unmount")); + // the Workspace list and the capacity read share the one pooled + // connection (#543): a get takes no second control slot of its own + assert_eq!(served.load(Ordering::SeqCst), 1, "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_daemon_that_does_not_answer_leaves_daemon_capacity_out() { + let (endpoint, _served) = daemon(busy(), true); + let home = tempdir("stalled"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, Some("s-1"), None); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + // the read model's own bound ends the wait; the get still answers + assert!( + elapsed < Duration::from_secs(4), + "sandbox.get stalled for {elapsed:?}" + ); + assert!( + elapsed >= Duration::from_secs(1), + "the bounded query did not wait: {elapsed:?}" + ); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_stopped_machine_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("stopped"); + home::open(&home).unwrap(); + // the machine records no worker: nothing starts it (#535 D11) + home_records(&home, false); + sandbox_record(&home, endpoint, Some("s-1"), None); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + assert!(got.get("down_reason").is_some(), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + // a down machine is answered from its records: no port is dialed and + // nothing waits for a daemon that is not there + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn an_ended_forward_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("forward-ended"); + home::open(&home).unwrap(); + home_records(&home, true); + // a recorded forward process that is not alive: its port may be reused by + // another process, so it is never dialed + sandbox_record(&home, endpoint, Some("s-1"), Some(3_999_999)); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + assert!(got.get("down_reason").is_some(), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_sandbox_with_no_id_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("gone"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, None, None); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(got.get_text("state"), Some("gone"), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-control/src/doctor/capacity.rs b/control/crates/layerfs-control/src/doctor/capacity.rs new file mode 100644 index 0000000000..f69dda51a1 --- /dev/null +++ b/control/crates/layerfs-control/src/doctor/capacity.rs @@ -0,0 +1,113 @@ +//! A running Sandbox's daemon control capacity (#540): the +//! `backend.daemon.connection_failures` read model, queried best-effort for +//! each Sandbox whose record says it runs and whose daemon endpoint is +//! known. Nothing is started: a daemon that does not answer is a warning +//! naming its endpoint and the operation, never an error, because the +//! doctor changes nothing and a stopped daemon is a normal state. +//! Two more warnings come from the model itself: retained connection +//! failures, and a pool with no vacant control slot. The one bounded query +//! itself is `layerfs_sandbox::daemon::capacity`, shared with the +//! `daemon_capacity` field `sandbox.get` shows. +use super::report::Findings; +use super::scan::Scanned; +use layerfs_bridge::control::{CapacityModel, SdkRefusal}; +use layerfs_sandbox::daemon::capacity::{endpoint, read, OPERATION, WAIT}; +use std::net::SocketAddr; +use std::path::Path; + +/// Queries the daemon of every running Sandbox whose endpoint is known. +pub(super) fn check(home: &Path, scanned: &Scanned, findings: &mut Findings) { + for sandbox in &scanned.sandboxes { + // a Sandbox whose create never recorded an id is gone; a creating, + // closing or unresolved one has no steady daemon to ask + if sandbox.id().is_none() || sandbox.state(false) != "running" { + continue; + } + let name = sandbox.name(); + let Some(endpoint) = endpoint(home, sandbox) else { + // no known endpoint: the record does not say where its daemon + // listens, and the doctor starts nothing to find out + continue; + }; + match read(home, sandbox, endpoint) { + Ok(model) => capacity(findings, name, endpoint, &model), + Err(refusal) => unanswered(findings, name, endpoint, &refusal), + } + } +} + +/// A daemon that did not answer: a warning naming its endpoint and the +/// operation. An exhausted pool is reported from the refusal's admission +/// notice, which names its resource and vacant slots; the daemon never +/// admitted the query, so it is still not an error. +fn unanswered(findings: &mut Findings, name: &str, endpoint: SocketAddr, refusal: &SdkRefusal) { + let detail = &refusal.detail; + if refusal.code == "Capacity" && detail.get_text("resource") == Some("daemon_control_slot") { + findings.warning( + ("sandbox", name), + format!( + "Sandbox {name}'s daemon at {endpoint} refused `{OPERATION}` with Capacity on resource daemon_control_slot (pool_vacant {}, reserved_vacant {}, retained {}): no control slot is vacant", + detail.get_int("pool_vacant").unwrap_or(0), + detail.get_int("reserved_vacant").unwrap_or(0), + detail.get_int("retained").unwrap_or(0) + ), + format!( + "end the calls that hold its control slots, or wait for them to end; `layerfs sandbox get {name}` shows the daemon's daemon_capacity, the `{OPERATION}` read model" + ), + ); + return; + } + findings.warning( + ("sandbox", name), + format!( + "Sandbox {name}'s daemon at {endpoint} did not answer `{OPERATION}` within {}s: {}", + WAIT.as_secs(), + refusal.message + ), + format!( + "`layerfs sandbox get {name}` shows whether it still runs; `layerfs sandbox close {name}` closes one whose daemon is gone" + ), + ); +} + +/// The warnings the daemon's capacity model gives: retained connection +/// failures, and a pool with no vacant control slot. +fn capacity(findings: &mut Findings, name: &str, endpoint: SocketAddr, model: &CapacityModel) { + let retained = &model.retained; + if retained.count > 0 { + findings.warning( + ("sandbox", name), + format!( + "Sandbox {name}'s daemon at {endpoint} retained {} of {} connection failure record(s) ({} dropped): calls it refused before Hello are kept there for {:.1}s", + retained.count, + retained.limit, + retained.dropped, + retained.ttl_ms as f64 / 1000.0 + ), + capacity_fix(name), + ); + } + let admission = &model.admission; + if admission.exhausted() { + findings.warning( + ("sandbox", name), + format!( + "Sandbox {name}'s daemon at {endpoint} has no vacant pool control slot (pool_vacant {}, pool_running {}, reserved_vacant {}, reserved_running {}); a call that needs one is refused Capacity on resource daemon_control_slot", + admission.pool_vacant, + admission.pool_running, + admission.reserved_vacant, + admission.reserved_running + ), + format!( + "end the calls that hold its control slots, or wait for them to end; `layerfs sandbox get {name}` shows the daemon's daemon_capacity, the `{OPERATION}` read model" + ), + ); + } +} + +/// Where the operator finds the model a capacity warning came from. +fn capacity_fix(name: &str) -> String { + format!( + "`layerfs sandbox get {name}` shows the daemon's daemon_capacity, the `{OPERATION}` read model" + ) +} diff --git a/control/crates/layerfs-control/src/doctor/mod.rs b/control/crates/layerfs-control/src/doctor/mod.rs index 5dc1d8f5e1..742451bb62 100644 --- a/control/crates/layerfs-control/src/doctor/mod.rs +++ b/control/crates/layerfs-control/src/doctor/mod.rs @@ -2,9 +2,11 @@ //! changes nothing and starts no machine. It reads the records, their modes //! and references, the `op` intents, the Store leases, a layout 2 home's //! conversion blockers, partial Store copies on `native` and the build of -//! each running machine's worker (its hello alone: no request is sent). +//! each running machine's worker (its hello alone: no request is sent), and +//! a running Sandbox's daemon capacity (a best-effort query, bounded to 2 s). //! Each finding names the call that fixes it. It takes no lock: a home //! that changes meanwhile may be reported as it was part way. +mod capacity; mod checks; mod convert; mod report; diff --git a/control/crates/layerfs-control/src/doctor/report.rs b/control/crates/layerfs-control/src/doctor/report.rs index 4c5aeafc8e..88fcd17ed5 100644 --- a/control/crates/layerfs-control/src/doctor/report.rs +++ b/control/crates/layerfs-control/src/doctor/report.rs @@ -109,6 +109,7 @@ pub fn doctor(home: &Path) -> Outcome { Ok(Layout::Current(_)) => { let scanned = super::scan::scan(home, &mut findings)?; super::checks::check(home, &scanned, &mut findings)?; + super::capacity::check(home, &scanned, &mut findings); super::workers::check(home, &scanned, &mut findings); } } diff --git a/control/crates/layerfs-control/tests/doctor.rs b/control/crates/layerfs-control/tests/doctor.rs index 911ba1874b..7b3eeb9990 100644 --- a/control/crates/layerfs-control/tests/doctor.rs +++ b/control/crates/layerfs-control/tests/doctor.rs @@ -1,13 +1,19 @@ //! `layerfs doctor` (#536 H7): a home with one of each problem gives one //! finding each, naming the call that fixes it; a clean home is `ok` with //! no findings; a layout 2 home reports what refuses its conversion; and -//! the doctor writes nothing in the home. -use layerfs_bridge::control::SdkValue; +//! the doctor writes nothing in the home. A live fake daemon's capacity model +//! is explained as warnings, and its Capacity refusal is a warning, not an +//! error. +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request, + RetainedModel, SdkRefusal, SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; use layerfs_control::doctor::doctor; use layerfs_home::home::{paths, private_dir, write_entity, write_record}; use layerfs_machine::worker::{read_frame, read_frame_within, write_frame, MAX_HELLO}; use std::collections::BTreeMap; -use std::net::TcpListener; +use std::net::{SocketAddr, TcpListener}; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::sync::mpsc; @@ -576,3 +582,295 @@ fn exec_records_are_checked_and_left_as_they_are() { .any(|f| f.0 == "error" && f.3.contains("not an exec record"))); std::fs::remove_dir_all(home).unwrap(); } + +#[test] +fn a_stalled_daemon_endpoint_is_a_bounded_warning_and_starts_nothing() { + let home = new_home("stalled"); + layerfs_control::migrate::open(&home).unwrap(); + store(&home, "shop", sqlite("shop", "native")); + let project = SdkValue::map([("name", t("api"))]); + write_entity(&paths::project_record(&home, "shop", "api"), &project).unwrap(); + let native = SdkValue::map([("name", t("native")), ("platform", t("native"))]); + write_entity(&paths::machine_record(&home, "native"), &native).unwrap(); + let sandbox = SdkValue::map([ + ("name", t("sb")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("sb")), + ]); + write_entity(&paths::sandbox_record(&home, "sb"), &sandbox).unwrap(); + // a daemon endpoint that accepts and then says nothing: the connect + // succeeds and only the bounded handshake read ends the query + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let (dialed, dialed_rx) = mpsc::channel(); + std::thread::spawn(move || { + if let Ok((stream, _)) = listener.accept() { + let _ = dialed.send(()); + std::thread::sleep(Duration::from_secs(6)); + drop(stream); + } + }); + let keys = SdkValue::map([ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(vec![1; 32])), + ("peer", SdkValue::Bytes(vec![2; 32])), + ("instance", SdkValue::Bytes(vec![3; 32])), + ]); + write_record(&paths::sandbox_control(&home, "sb"), &keys).unwrap(); + + let before = tree(&home); + let at = std::time::Instant::now(); + let report = doctor(&home).unwrap(); + let elapsed = at.elapsed(); + assert!( + elapsed < Duration::from_secs(4), + "the doctor stalled for {elapsed:?}" + ); + assert!( + elapsed >= Duration::from_secs(1), + "the doctor did not wait for its bounded query: {elapsed:?}" + ); + dialed_rx + .recv_timeout(Duration::from_secs(2)) + .expect("the doctor never dialed the endpoint"); + assert_eq!(before, tree(&home), "the doctor wrote in the home"); + // a daemon that does not answer is a warning, not an error: the doctor + // stays read-only and does not fail on an idle or stopped daemon + assert_eq!(report.get("ok"), Some(&SdkValue::Bool(true)), "{report:?}"); + let found = findings(&report); + assert_eq!(found.len(), 1, "{found:#?}"); + let (severity, kind, name, message, fix) = &found[0]; + assert_eq!( + (severity.as_str(), kind.as_str(), name.as_str()), + ("warning", "sandbox", "sb") + ); + assert!(message.contains(&endpoint.to_string()), "{message}"); + assert!( + message.contains("backend.daemon.connection_failures"), + "{message}" + ); + assert!(message.contains("within 2s"), "{message}"); + assert!(fix.contains("sandbox get sb"), "{fix}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_sandbox_without_a_recorded_endpoint_is_not_queried() { + let home = new_home("noendpoint"); + layerfs_control::migrate::open(&home).unwrap(); + store(&home, "shop", sqlite("shop", "native")); + let project = SdkValue::map([("name", t("api"))]); + write_entity(&paths::project_record(&home, "shop", "api"), &project).unwrap(); + let sandbox = SdkValue::map([ + ("name", t("sb")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("sb")), + ]); + write_entity(&paths::sandbox_record(&home, "sb"), &sandbox).unwrap(); + // daemon keys without an endpoint: nothing tells the doctor where to + // dial, and it starts nothing to find out + let keys = SdkValue::map([ + ("secret", SdkValue::Bytes(vec![1; 32])), + ("peer", SdkValue::Bytes(vec![2; 32])), + ("instance", SdkValue::Bytes(vec![3; 32])), + ]); + write_record(&paths::sandbox_control(&home, "sb"), &keys).unwrap(); + let at = std::time::Instant::now(); + let report = doctor(&home).unwrap(); + assert!(at.elapsed() < Duration::from_secs(1)); + assert_eq!(report.get("ok"), Some(&SdkValue::Bool(true))); + assert!(findings(&report).is_empty(), "{report:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +/// A fake daemon that answers the doctor's two calls: its Hello, then the +/// control-capacity read model with `reply`. +fn capacity_daemon(reply: Reply) -> (SocketAddr, std::thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let worker = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut channel = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) + .unwrap(); + let hello = Call::decode(channel.receive.receive().unwrap()).unwrap(); + assert!(matches!(hello.request, Request::Hello(_))); + channel + .send + .send( + &Answer { + id: hello.id, + reply: Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + } + .encode() + .unwrap(), + ) + .unwrap(); + let call = Call::decode(channel.receive.receive().unwrap()).unwrap(); + let id = call.id; + let Request::Sdk(sdk) = call.request else { + panic!("expected the capacity read model"); + }; + assert_eq!(sdk.method, CAPACITY_OPERATION); + channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .unwrap(); + }); + (endpoint, worker) +} + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// A home with one running Sandbox whose daemon keys name `endpoint`. +fn running_sandbox(home: &Path, endpoint: SocketAddr) { + layerfs_control::migrate::open(home).unwrap(); + store(home, "shop", sqlite("shop", "native")); + let project = SdkValue::map([("name", t("api"))]); + write_entity(&paths::project_record(home, "shop", "api"), &project).unwrap(); + let native = SdkValue::map([("name", t("native")), ("platform", t("native"))]); + write_entity(&paths::machine_record(home, "native"), &native).unwrap(); + let sandbox = SdkValue::map([ + ("name", t("sb")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("sb")), + ]); + write_entity(&paths::sandbox_record(home, "sb"), &sandbox).unwrap(); + let keys = SdkValue::map([ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]); + write_record(&paths::sandbox_control(home, "sb"), &keys).unwrap(); +} + +#[test] +fn capacity_model_warnings_name_retained_failures_and_an_exhausted_pool() { + let model = layerfs_bridge::control::CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel { + limit: 64, + ttl_ms: 3_600_000, + count: 3, + dropped: 2, + recorded: 3, + failures: Vec::new(), + }, + }; + let (endpoint, worker) = capacity_daemon(Reply::Sdk(model.to_sdk())); + let home = new_home("capacity"); + running_sandbox(&home, endpoint); + let report = doctor(&home).unwrap(); + worker.join().unwrap(); + assert_eq!(report.get("ok"), Some(&SdkValue::Bool(true)), "{report:?}"); + let found = findings(&report); + assert_eq!(found.len(), 2, "{found:#?}"); + let text: Vec = found + .iter() + .map(|(_, _, _, message, fix)| format!("{message} {fix}")) + .collect(); + assert!(text[0].contains("retained 3 of 64"), "{text:#?}"); + assert!(text[0].contains("2 dropped"), "{text:#?}"); + assert!( + text[0].contains("backend.daemon.connection_failures"), + "{text:#?}" + ); + assert!(text[1].contains("pool_vacant 0"), "{text:#?}"); + assert!(text[1].contains("pool_running 8"), "{text:#?}"); + assert!(text[1].contains("Capacity"), "{text:#?}"); + assert!(text[1].contains("daemon_control_slot"), "{text:#?}"); + for (severity, kind, name, _, _) in &found { + assert_eq!( + (severity.as_str(), kind.as_str(), name.as_str()), + ("warning", "sandbox", "sb") + ); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_healthy_capacity_model_gives_no_finding() { + let model = CapacityModel { + admission: AdmissionModel { + pool_vacant: 7, + pool_running: 1, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel { + limit: 64, + ttl_ms: 3_600_000, + count: 0, + dropped: 0, + recorded: 0, + failures: Vec::new(), + }, + }; + let (endpoint, worker) = capacity_daemon(Reply::Sdk(model.to_sdk())); + let home = new_home("capacity-healthy"); + running_sandbox(&home, endpoint); + let report = doctor(&home).unwrap(); + worker.join().unwrap(); + assert_eq!(report.get("ok"), Some(&SdkValue::Bool(true))); + assert!(findings(&report).is_empty(), "{report:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_capacity_refusal_is_a_warning_not_an_error() { + let refusal = SdkRefusal { + code: "Capacity".to_owned(), + message: "the Sandbox's daemon has no vacant control slot".to_owned(), + detail: SdkValue::map([ + ("resource", t("daemon_control_slot")), + ("retained", SdkValue::Int(3)), + ("pool_vacant", SdkValue::Int(0)), + ("reserved_vacant", SdkValue::Int(1)), + ("retryable", SdkValue::Bool(true)), + ]), + }; + let (endpoint, worker) = capacity_daemon(Reply::SdkRefused(refusal)); + let home = new_home("capacity-refused"); + running_sandbox(&home, endpoint); + let report = doctor(&home).unwrap(); + worker.join().unwrap(); + assert_eq!(report.get("ok"), Some(&SdkValue::Bool(true))); + let found = findings(&report); + assert_eq!(found.len(), 1, "{found:#?}"); + let message = &found[0].3; + assert!(message.contains("Capacity"), "{message}"); + assert!(message.contains("daemon_control_slot"), "{message}"); + assert!(message.contains("pool_vacant 0"), "{message}"); + assert!(message.contains("reserved_vacant 1"), "{message}"); + assert!(message.contains("retained 3"), "{message}"); + assert_eq!(found[0].0, "warning"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-control/tests/doctor_machine_target.rs b/control/crates/layerfs-control/tests/doctor_machine_target.rs index 67a0e08723..fa1c426e02 100644 --- a/control/crates/layerfs-control/tests/doctor_machine_target.rs +++ b/control/crates/layerfs-control/tests/doctor_machine_target.rs @@ -106,6 +106,7 @@ fn machine_target_reports_dependencies_and_preserves_home() { "fail", ); MachineRecord::built_in("native").write(&f.home).unwrap(); + #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] let native = f.report("native", Duration::from_secs(1)); #[cfg(target_os = "macos")] outcome(&native, "machine.provider", "pass"); diff --git a/control/crates/layerfs-home/src/home/capacity.rs b/control/crates/layerfs-home/src/home/capacity.rs index 6e01258759..d57e1354bc 100644 --- a/control/crates/layerfs-home/src/home/capacity.rs +++ b/control/crates/layerfs-home/src/home/capacity.rs @@ -5,7 +5,11 @@ use std::path::Path; /// Uses filesystem allocation units, with checked arithmetic; never writes. pub fn filesystem_available(path: &Path) -> std::io::Result { let filesystem = nix::sys::statvfs::statvfs(path).map_err(std::io::Error::from)?; - u64::from(filesystem.blocks_available()) - .checked_mul(filesystem.fragment_size() as u64) - .ok_or_else(|| std::io::Error::other("filesystem available-byte count overflow")) + // `fsblkcnt_t` is `u32` on macOS but `u64` on Linux, so `u64::from` + // is a useless conversion there; widen through `u128` (the same shape + // every other statvfs consumer uses) and narrow back, checked. + let available = + u128::from(filesystem.blocks_available()) * u128::from(filesystem.fragment_size()); + u64::try_from(available) + .map_err(|_| std::io::Error::other("filesystem available-byte count overflow")) } diff --git a/control/crates/layerfs-sandbox/src/daemon/capacity.rs b/control/crates/layerfs-sandbox/src/daemon/capacity.rs new file mode 100644 index 0000000000..31b41b1556 --- /dev/null +++ b/control/crates/layerfs-sandbox/src/daemon/capacity.rs @@ -0,0 +1,141 @@ +//! A running Sandbox daemon's control-capacity read model (#540): its +//! `backend.daemon.connection_failures` answer, read once, best-effort and +//! bounded. The doctor explains it and `sandbox.get` shows it as +//! `daemon_capacity`; both ask through here so one place decides where the +//! daemon is, how long an answer may take, and what an unanswered query is. +//! +//! Both reads end at [`WAIT`], and their connection is fenced at that +//! deadline, so a daemon that trickles its handshake or its one reply keeps +//! neither a thread, a connection nor a control slot past it. +use crate::daemon::fence::Fence; +use crate::daemon::pool::{Controls, Dial}; +use crate::daemon::{operator, trace}; +use crate::direct; +use crate::internal::Control; +use crate::record::SandboxRecord; +use layerfs_bridge::control::{CapacityModel, Request, SdkRefusal, SdkValue}; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused}; +use layerfs_home::secrets; +use std::net::SocketAddr; +use std::path::Path; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// The daemon's operator read model of control admission and retained +/// connection failures. +pub const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; + +/// How long one daemon query may take: connecting, its Hello and its one +/// exchange. Short on purpose: a reader must not stall behind a daemon that +/// is unreachable, as an ordinary 60 s admission wait would. +pub const WAIT: Duration = Duration::from_secs(2); + +/// The model [`read`] gives: the daemon's answer, parsed once. +pub fn read( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let home = home.to_owned(); + let sandbox = sandbox.clone(); + bounded(WAIT, move || query(&home, &sandbox, endpoint)) + .map(|answer| CapacityModel::parse(&answer)) +} + +/// The daemon's answer to [`OPERATION`] on a connection the Sandbox's pool +/// lends (`sandbox.get`, #543): at most [`WAIT`] for admission and for the +/// exchange, which is fenced at the same deadline, so this read takes no +/// second connection and no second slot of its own. A window the admission +/// wait used up refuses the read before it is sent, and a connection the +/// fence ends stays quarantined: the pool closes it instead of lending it +/// again. Nothing is sent again. +pub fn lent(controls: &Controls) -> Result { + let deadline = Instant::now() + WAIT; + let mut control = trace::control_within(controls, WAIT)?; + if deadline.saturating_duration_since(Instant::now()).is_zero() { + // the admission wait used the whole window. The connection is healthy + // and returns to the pool; arming a fence for a deadline that already + // passed would shut it down for nothing, so nothing is sent. + return Err(refused( + "Unavailable", + format!( + "no control connection came free within {}s, so the capacity read was not sent", + WAIT.as_secs() + ), + )); + } + let fence = Fence::channel((control.close_handle().map_err(failed))?, deadline); + let answer = exchange(&mut control); + drop(fence); + answer +} + +/// The address of a running Sandbox's daemon when its record names one to +/// this host: a forwarded machine's forward, else the endpoint the start +/// recorded in `control.json`. Nothing is observed or started. +pub fn endpoint(home: &Path, sandbox: &SandboxRecord) -> Option { + if let Some(forward) = sandbox.daemon().and_then(direct::forward_of) { + return Some(forward); + } + let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name())) + .ok() + .flatten()?; + keys.get_text("endpoint")?.parse().ok() +} + +/// [`query`] under a hard wall: a daemon that accepts a connection and then +/// says nothing must not hold the caller past [`WAIT`], even where the fence +/// itself cannot be armed. +fn bounded( + within: Duration, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result { + let (sent, received) = mpsc::channel(); + std::thread::spawn(move || { + let _ = sent.send(work()); + }); + match received.recv_timeout(within) { + Ok(result) => result, + Err(_) => Err(refused( + "Unavailable", + format!("the daemon did not answer within {}s", within.as_secs()), + )), + } +} + +/// One daemon query on a connection dialed for it: its recorded endpoint with +/// its keys, for at most [`WAIT`], fenced at the same deadline, then read the +/// capacity model once. +fn query( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name()))? + .ok_or_else(|| refused("NoSuchSandbox", "the Sandbox has no daemon keys"))?; + let dial = Dial { + endpoint, + secret: secrets::key(&keys, "secret")?, + peer: secrets::key(&keys, "peer")?, + instance: secrets::key(&keys, "instance")?, + }; + let (mut control, fence) = dial.open_within(WAIT)?; + let answer = exchange(&mut control); + // this conversation ends here: a connection dropped instead leaves the + // daemon reading a closed socket, which it logs as an unexpected end + let _ = control.call(Request::EndSession); + drop(fence); + answer +} + +/// The one exchange: the daemon's answer, or the refusal it gave, kept exact. +fn exchange(control: &mut Control) -> Result { + operator::call(control, OPERATION, SdkValue::Null).map_err(|error| match error { + crate::SdkError::Refused(refusal) => refusal, + crate::SdkError::Operation(failure) => { + crate::daemon::values::refusal(&failure).unwrap_or_else(|| failed(failure)) + } + crate::SdkError::Mount(failure) => failed(failure), + }) +} diff --git a/control/crates/layerfs-sandbox/src/daemon/channel.rs b/control/crates/layerfs-sandbox/src/daemon/channel.rs index 9853146c60..49c161ef81 100644 --- a/control/crates/layerfs-sandbox/src/daemon/channel.rs +++ b/control/crates/layerfs-sandbox/src/daemon/channel.rs @@ -1,7 +1,7 @@ //! One attempted authenticated control exchange, retaining the original unknown. use layerfs_bridge::{ control::{Answer, Call, ControlError, Reply, Request, SdkValue}, - duplex::{ChannelError, ChannelWork, Connection}, + duplex::{ChannelError, ChannelWork, CloseHandle, Connection}, }; use std::fmt; /// One sequential control connection. Concurrent callers may use separate channels. @@ -74,10 +74,13 @@ impl Control { pub fn take_telemetry(&mut self) -> Vec { std::mem::take(&mut self.telemetry) } - /// True once an exchange on this connection failed: it is quarantined, - /// and every later call on it is refused before sending. + /// True once this connection cannot carry another call: an exchange + /// failed on it, its session ended, or an application fence + /// ([`Control::close_handle`]) shut its channel down. The pool closes it + /// instead of lending it again, and a later call is refused before + /// sending, so a quarantined connection is never reported as an attempt. pub fn is_failed(&self) -> bool { - self.failed + self.failed || self.connection.is_quarantined() } /// Sends once and awaits the original reply once. Typed remote refusals are /// replies; failed delivery retains an unknown, never replays the command. @@ -118,7 +121,10 @@ impl Control { let mut attempted = false; let mut received = None; let result = (|| { - if self.failed { + if self.is_failed() { + // a fence or an ended session may have quarantined the channel + // after this owner's last call: nothing is sent on it + self.failed = true; return Err(ControlCause::Channel(ChannelError::Quarantined)); } let record = call.encode().map_err(ControlCause::Protocol)?; @@ -219,7 +225,7 @@ impl Control { manifest: &layerfs_bridge::provision::StoreManifest, ) -> Result> { - if self.failed { + if self.is_failed() { return Err(Box::new(layerfs_project::store::InstallFailure { source: store.path.clone(), manifest: manifest.clone(), @@ -243,6 +249,12 @@ impl Control { pub fn clear_io_waits(&self) -> Result<(), ChannelError> { self.connection.clear_io_waits() } + /// A clone of this channel's socket for an application-owned fence + /// (`daemon::fence`): it carries no send or replay capability, and + /// shutting it down quarantines the channel. + pub fn close_handle(&self) -> Result { + self.connection.close_handle() + } /// Actual cumulative duplex send/receive work for this connection. pub fn channel_work(&self) -> (ChannelWork, ChannelWork) { (self.connection.send.work(), self.connection.receive.work()) diff --git a/control/crates/layerfs-sandbox/src/daemon/fence.rs b/control/crates/layerfs-sandbox/src/daemon/fence.rs new file mode 100644 index 0000000000..fdbbf4345b --- /dev/null +++ b/control/crates/layerfs-sandbox/src/daemon/fence.rs @@ -0,0 +1,82 @@ +//! One deadline a control connection cannot outlive (#540, #543). +//! +//! A bounded read that only sets socket timeouts bounds each read and write +//! system call, not the exchange: a daemon that trickles one byte just +//! inside the timeout can hold the reader, its connection and its pool slot +//! far past the time the caller promised. A fence instead holds a clone of +//! the connection's socket and shuts both directions down at one absolute +//! deadline, which ends the blocked call immediately. The doctor's bounded +//! observation (`doctor/sandbox_channel.rs` outside this crate) uses the same +//! pattern. +use layerfs_bridge::duplex::CloseHandle; +use std::net::{Shutdown, TcpStream}; +use std::sync::mpsc; +use std::time::Instant; + +/// What this fence ends at its deadline: a socket cloned from a connection +/// being opened, or the application-owned close handle of an established +/// channel. +enum End { + Socket(TcpStream), + Channel(CloseHandle), +} + +impl End { + fn shut(&self) { + match self { + Self::Socket(socket) => { + let _ = socket.shutdown(Shutdown::Both); + } + // the handle quarantines the channel itself, and the pool reads + // that flag, so it closes the connection instead of lending it + // again + Self::Channel(handle) => { + let _ = handle.close(); + } + } + } +} + +/// A socket fence, armed at one absolute deadline; its `Drop` cancels the +/// deadline and joins the thread that waits for it. +pub struct Fence { + stop: mpsc::Sender<()>, + thread: Option>, +} + +impl Fence { + /// Arms `socket` (a clone of the connection's) at `deadline`. + pub fn on(socket: TcpStream, deadline: Instant) -> Self { + Self::armed(End::Socket(socket), deadline) + } + + /// Arms established channel `handle` at `deadline`. + pub fn channel(handle: CloseHandle, deadline: Instant) -> Self { + Self::armed(End::Channel(handle), deadline) + } + + fn armed(end: End, deadline: Instant) -> Self { + let (stop, wait) = mpsc::channel(); + let thread = std::thread::spawn(move || { + if matches!( + wait.recv_timeout(deadline.saturating_duration_since(Instant::now())), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + end.shut(); + } + }); + Self { + stop, + thread: Some(thread), + } + } +} + +impl Drop for Fence { + fn drop(&mut self) { + let _ = self.stop.send(()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/control/crates/layerfs-sandbox/src/daemon/mod.rs b/control/crates/layerfs-sandbox/src/daemon/mod.rs index 93718cd2d4..a9a3eb2000 100644 --- a/control/crates/layerfs-sandbox/src/daemon/mod.rs +++ b/control/crates/layerfs-sandbox/src/daemon/mod.rs @@ -3,10 +3,14 @@ #[doc(hidden)] pub mod call; #[doc(hidden)] +pub mod capacity; +#[doc(hidden)] pub mod channel; #[doc(hidden)] pub mod failure; #[doc(hidden)] +pub mod fence; +#[doc(hidden)] pub mod internal; #[doc(hidden)] pub mod operator; diff --git a/control/crates/layerfs-sandbox/src/daemon/operator.rs b/control/crates/layerfs-sandbox/src/daemon/operator.rs index 57fe3ac49e..325c506b1c 100644 --- a/control/crates/layerfs-sandbox/src/daemon/operator.rs +++ b/control/crates/layerfs-sandbox/src/daemon/operator.rs @@ -2,16 +2,23 @@ use crate::{internal::Control, SdkError, SdkResult}; use layerfs_bridge::control::{SdkRefusal, SdkValue}; -/// Calls the daemon's read model that lists its Workspaces -/// (`sandbox.list_workspaces`), the one daemon backend method the host -/// Sandbox code sends; any other is refused before it is sent. +/// Calls one daemon backend method: the read model that lists its +/// Workspaces (`sandbox.list_workspaces`), or the operator view of control +/// admission and retained connection failures +/// (`backend.daemon.connection_failures`, #540). Any other method is refused +/// before it is sent. pub fn call(control: &mut Control, method: &str, args: SdkValue) -> SdkResult { - if method != "sandbox.list_workspaces" { - return Err(SdkError::Refused(SdkRefusal { + match method { + // The daemon's control admission and bounded retained-failure store + // (#540): an operator read model, never a product operation. + layerfs_bridge::control::CAPACITY_OPERATION => { + crate::daemon::call::exchange_call(control, method, args) + } + "sandbox.list_workspaces" => crate::daemon::call::exchange_call(control, method, args), + _ => Err(SdkError::Refused(SdkRefusal { code: "UnknownMethod".into(), message: format!("no daemon backend method {method}"), detail: SdkValue::Null, - })); + })), } - crate::daemon::call::exchange_call(control, method, args) } diff --git a/control/crates/layerfs-sandbox/src/daemon/pool.rs b/control/crates/layerfs-sandbox/src/daemon/pool.rs index ea7ccb6ccb..e7eab3a0be 100644 --- a/control/crates/layerfs-sandbox/src/daemon/pool.rs +++ b/control/crates/layerfs-sandbox/src/daemon/pool.rs @@ -5,15 +5,20 @@ //! every call wait for the one before it, a 1 ms read behind a whole Commit. A //! Sandbox keeps up to [`CONNECTIONS`] connections and opens one only when //! every open one is busy. Each exchange runs whole on one connection, sent -//! once. A connection whose exchange failed is quarantined by its `Control` -//! and is closed rather than reused; the failed call is never sent again. +//! once. A connection whose exchange failed, or whose channel an application +//! fence (`daemon::fence`) shut down, is quarantined by its `Control` and is +//! closed rather than reused; the failed call is never sent again. //! When the daemon admits no more connections, the Sandbox keeps the ones it //! has and callers take turns on them, each waiting at most [`LEND_WAIT`] //! before it is refused `Busy`. Opening a connection is not an attempt //! of any operation: nothing is sent before the connection is ready. +use crate::daemon::fence::Fence; use crate::internal::Control; -use layerfs_bridge::control::{DaemonPhase, HelloRequest, Reply, Request, SdkRefusal}; -use layerfs_bridge::duplex::initiate_observed; +use layerfs_bridge::control::{DaemonPhase, HelloRequest, Reply, Request, SdkRefusal, SdkValue}; +use layerfs_bridge::duplex::{ + admission::{AdmissionNotice, AdmissionReason}, + initiate_observed, ChannelError, +}; use layerfs_home::outcome::{failed, refused}; use std::net::SocketAddr; use std::ops::{Deref, DerefMut}; @@ -46,47 +51,216 @@ pub const ADMISSION_WAIT: Duration = Duration::from_secs(60); /// before it ends, and its `Busy` refusal is not retried. pub const LEND_WAIT: Duration = ADMISSION_WAIT; -/// Whether a failed open is the daemon closing a connection it did not admit: -/// the stream ended before the Hello reply. Through a port proxy the client -/// reads the end (`UnexpectedEof`); dialing the daemon directly, its handshake -/// write can meet the closed socket first (`BrokenPipe`, `ConnectionReset`). -/// Nothing but Hello was sent, so no operation was attempted and opening -/// again is safe. -fn not_admitted(refusal: &SdkRefusal) -> bool { - refusal.code == "Failed" - && ["UnexpectedEof", "BrokenPipe", "ConnectionReset"] - .iter() - .any(|kind| refusal.message.contains(kind)) +/// Why an open ended before the daemon's Hello reply. Every one means the +/// call was not admitted and nothing was attempted, so opening again is +/// safe; they name different observable causes (#540). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum NotAdmitted { + /// The daemon had no vacant pool slot. `retained` is exact when the + /// advisory capacity notice arrived, and unknown when only EOF was seen. + NoVacantSlot { retained: Option }, + /// The connection itself failed before Hello (`BrokenPipe`, + /// `ConnectionReset`). + ConnectionFailed, +} + +impl NotAdmitted { + fn code(self) -> &'static str { + match self { + Self::NoVacantSlot { .. } => "Capacity", + Self::ConnectionFailed => "Unavailable", + } + } + + fn admission(self) -> &'static str { + match self { + Self::NoVacantSlot { .. } => "no-vacant-slot", + Self::ConnectionFailed => "connection-failed", + } + } +} + +/// Classify a failed open that never reached the Hello reply. Through a port +/// proxy the client reads the end (`UnexpectedEof`, a clean close by the +/// daemon's accept loop); dialing the daemon directly, its handshake write can +/// meet the closed socket first (`BrokenPipe`, `ConnectionReset`). The old +/// message claimed "other processes hold all 8 connections"; that is not +/// knowledge the client has (#540). +fn not_admitted(refusal: &SdkRefusal) -> Option<(NotAdmitted, &'static str)> { + if refusal.code != "Failed" { + return None; + } + if refusal.message.contains("UnexpectedEof") { + return Some(( + NotAdmitted::NoVacantSlot { retained: None }, + "UnexpectedEof", + )); + } + for kind in ["BrokenPipe", "ConnectionReset"] { + if refusal.message.contains(kind) { + return Some((NotAdmitted::ConnectionFailed, kind)); + } + } + None +} + +/// The exact refusal for a decoded advisory capacity notice. This is the +/// precise form: it names the resource and the retained record count. +fn notice_refusal(notice: AdmissionNotice) -> SdkRefusal { + match notice.reason { + AdmissionReason::NoPoolSlot | AdmissionReason::ReservedBusy => SdkRefusal { + code: "Capacity".to_owned(), + message: format!( + "the Sandbox's daemon has no vacant control slot: {} pool slot(s) and {} reserved slot(s) are vacant, with {} retained failure record(s). The call was not admitted; retrying is safe", + notice.pool_vacant, notice.reserved_vacant, notice.retained + ), + detail: SdkValue::map([ + ("resource", SdkValue::text("daemon_control_slot")), + ("retained", SdkValue::Int(i64::from(notice.retained))), + ("pool_vacant", SdkValue::Int(i64::from(notice.pool_vacant))), + ( + "reserved_vacant", + SdkValue::Int(i64::from(notice.reserved_vacant)), + ), + ("retryable", SdkValue::Bool(true)), + ("admission", SdkValue::text(notice.reason.as_str())), + ]), + }, + AdmissionReason::Starting => SdkRefusal { + code: "Unavailable".to_owned(), + message: "the Sandbox's daemon is still starting and admits no control connection yet. The call was not admitted; retrying is safe" + .to_owned(), + detail: SdkValue::map([ + ("resource", SdkValue::Null), + ("retained", SdkValue::Null), + ("retryable", SdkValue::Bool(true)), + ("admission", SdkValue::text("starting")), + ]), + }, + } +} + +/// The exact refusal after `wait`: what the client observed, not a guess at +/// what another process is doing. +fn admission_refusal(wait: Duration, admission: NotAdmitted, kind: &str) -> SdkRefusal { + let message = match admission { + NotAdmitted::NoVacantSlot { retained } => format!( + "the Sandbox's daemon admitted no control connection for {}s: it closed the connection before Hello ({kind}), which it does when it has no vacant control slot{}. The call was not admitted; retrying is safe", + wait.as_secs(), + retained.map_or(String::new(), |count| format!( + " (it reported {count} retained failure record(s))" + )) + ), + NotAdmitted::ConnectionFailed => format!( + "the connection to the Sandbox's daemon failed before Hello for {}s ({kind}): the call was not admitted; retrying is safe", + wait.as_secs() + ), + }; + let retained = match admission { + NotAdmitted::NoVacantSlot { retained } => { + retained.map_or(SdkValue::Null, |count| SdkValue::Int(i64::from(count))) + } + _ => SdkValue::Null, + }; + SdkRefusal { + code: admission.code().to_owned(), + message, + detail: SdkValue::map([ + ( + "resource", + match admission { + NotAdmitted::NoVacantSlot { .. } => SdkValue::text("daemon_control_slot"), + _ => SdkValue::Null, + }, + ), + ("retained", retained), + ("retryable", SdkValue::Bool(true)), + ("admission", SdkValue::text(admission.admission())), + ("observed", SdkValue::text(kind)), + ]), + } +} + +/// Whether this refusal says nothing was attempted and a later open may +/// succeed. Only these exact refusals are retried. +fn retryable(refusal: &SdkRefusal) -> bool { + matches!(refusal.code.as_str(), "Capacity" | "Unavailable") + && refusal.detail.get_bool("retryable") } impl Dial { /// The first connection a process opens to a Sandbox. While the daemon - /// admits no more connections (other processes hold all of them), waits - /// for one to be released, up to [`ADMISSION_WAIT`], instead of failing - /// the caller's operation with a bare `UnexpectedEof`. + /// refuses admission, waits for one to be released, up to + /// [`ADMISSION_WAIT`], instead of failing the caller's operation with a + /// bare `UnexpectedEof`. Its final refusal names what the client actually + /// observed: no vacant control slot, or a failed connection (#540). pub fn open_waiting(&self) -> Result { self.open_admitted(ADMISSION_WAIT) } + /// [`Dial::open_within`], with the fence that ends this connection at + /// the same deadline: connecting, the handshake and its Hello must finish + /// inside that time, and a daemon that trickles any of them is shut down + /// at `deadline` rather than kept waiting. The connection keeps those I/O + /// waits for the one bounded call made on it. Nothing is retried and no + /// admission wait applies: a daemon that refuses or does not answer is + /// reported as it came. + pub fn open_within(&self, within: Duration) -> Result<(Control, Fence), SdkRefusal> { + let deadline = Instant::now() + within; + let remaining = || { + deadline + .saturating_duration_since(Instant::now()) + .max(Duration::from_millis(1)) + }; + let stream = + std::net::TcpStream::connect_timeout(&self.endpoint, remaining()).map_err(failed)?; + stream.set_read_timeout(Some(remaining())).map_err(failed)?; + stream + .set_write_timeout(Some(remaining())) + .map_err(failed)?; + // armed before the handshake: a trickling far side cannot outlive the + // deadline this open promised, and neither can the one call made on it + let fence = Fence::on(stream.try_clone().map_err(failed)?, deadline); + let connection = match initiate_observed(stream, &self.secret, self.peer).result { + Ok(connection) => connection, + Err(ChannelError::Admission(notice)) => return Err(notice_refusal(notice)), + Err(error) => return Err(failed(error)), + }; + // the waits stay: the caller makes exactly one call and drops it + let mut control = Control::new(connection); + let hello = control + .call(Request::Hello(HelloRequest { + expected_instance: Some(self.instance), + wait_for_store: false, + })) + .map_err(failed)?; + match hello { + Reply::Hello(status) if status.phase == DaemonPhase::ControlReady => { + Ok((control, fence)) + } + _ => Err(refused("StaleSandbox", "saved daemon is not ControlReady")), + } + } + /// [`Dial::open_waiting`], waiting at most `wait`. fn open_admitted(&self, wait: Duration) -> Result { let started = Instant::now(); let mut pause = Duration::from_millis(20); loop { match self.open() { - Err(refusal) if not_admitted(&refusal) && started.elapsed() < wait => { - std::thread::sleep(pause); - pause = (pause * 2).min(Duration::from_millis(500)); - } - Err(refusal) if not_admitted(&refusal) => { - return Err(refused( - "Busy", - format!( - "the Sandbox's daemon admitted no connection for {}s: other processes hold all {} of its connections", - wait.as_secs(), - crate::lifecycle::DAEMON_CONNECTIONS - ), - )) + Err(refusal) => { + let legacy = not_admitted(&refusal); + let may_retry = retryable(&refusal) || legacy.is_some(); + if may_retry && started.elapsed() < wait { + std::thread::sleep(pause); + pause = (pause * 2).min(Duration::from_millis(500)); + continue; + } + if let Some((admission, kind)) = legacy { + return Err(admission_refusal(wait, admission, kind)); + } + // A notice-based or transport refusal is already exact. + return Err(refusal); } other => return other, } @@ -103,9 +277,11 @@ impl Dial { stream .set_write_timeout(Some(Duration::from_secs(10))) .map_err(failed)?; - let connection = initiate_observed(stream, &self.secret, self.peer) - .result - .map_err(failed)?; + let connection = match initiate_observed(stream, &self.secret, self.peer).result { + Ok(connection) => connection, + Err(ChannelError::Admission(notice)) => return Err(notice_refusal(notice)), + Err(error) => return Err(failed(error)), + }; connection.clear_io_waits().map_err(failed)?; let mut control = Control::new(connection); match control @@ -159,6 +335,14 @@ impl Controls { /// A connection for one call, and how long the call waited for it. pub fn lend(&self) -> Result<(Lease<'_>, Duration), SdkRefusal> { + self.lend_within(LEND_WAIT) + } + + /// [`Controls::lend`], waiting at most `wait` for admission and for an + /// in-use connection. A caller that must not hold an operation intent + /// while a daemon refuses connections (an explicit close) uses a short + /// wait and reports the exact refusal (#540). + pub fn lend_within(&self, wait: Duration) -> Result<(Lease<'_>, Duration), SdkRefusal> { let started = Instant::now(); let poisoned = || refused("Failed", "binding state poisoned"); let mut pool = self.pool.lock().map_err(|_| poisoned())?; @@ -172,7 +356,7 @@ impl Controls { let only = pool.open == 1; drop(pool); let opened = if only { - self.dial.open_waiting() + self.dial.open_admitted(wait) } else { self.dial.open() }; @@ -190,14 +374,14 @@ impl Controls { } } } - let remaining = LEND_WAIT.saturating_sub(started.elapsed()); + let remaining = wait.saturating_sub(started.elapsed()); if remaining.is_zero() { return Err(refused( "Busy", format!( "all {} of this process's connections to the Sandbox stayed in use for {}s; nothing was attempted", pool.open, - LEND_WAIT.as_secs() + wait.as_secs() ), )); } @@ -228,7 +412,8 @@ impl Drop for Lease<'_> { .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if control.is_failed() { - // Quarantined: closed, and a later call may open another. + // Quarantined: its own failure or its channel's fence ended it, + // and a later call may open another. pool.open -= 1; drop(control); } else { diff --git a/control/crates/layerfs-sandbox/src/daemon/trace.rs b/control/crates/layerfs-sandbox/src/daemon/trace.rs index a1cf2620b5..4b0d161215 100644 --- a/control/crates/layerfs-sandbox/src/daemon/trace.rs +++ b/control/crates/layerfs-sandbox/src/daemon/trace.rs @@ -17,6 +17,19 @@ pub fn control(controls: &Controls) -> Result, SdkRefusal> { internal::trace(&mut lease, scope()); Ok(Traced(lease)) } + +/// [`control`], waiting at most `wait` for admission and for an in-use +/// connection. A short-wait caller reports the daemon's exact refusal instead +/// of holding its operation intent for the whole admission wait (#540). +pub fn control_within( + controls: &Controls, + wait: std::time::Duration, +) -> Result, SdkRefusal> { + let (mut lease, waited) = controls.lend_within(wait)?; + add_waited(waited); + internal::trace(&mut lease, scope()); + Ok(Traced(lease)) +} impl Drop for Traced<'_> { fn drop(&mut self) { let telemetry = internal::take_telemetry(&mut self.0); diff --git a/control/crates/layerfs-sandbox/src/direct.rs b/control/crates/layerfs-sandbox/src/direct.rs index 280f7ef3de..f6188b3306 100644 --- a/control/crates/layerfs-sandbox/src/direct.rs +++ b/control/crates/layerfs-sandbox/src/direct.rs @@ -285,22 +285,26 @@ pub fn host(home: &Path, record: &SandboxRecord, target: &Target, message: Messa /// `sandbox.get`'s Workspace list of forwarded Sandbox `record`: a daemon /// that cannot be reached is [`Unreached`], which the get reports as down. pub fn list(home: &Path, record: &SandboxRecord) -> std::result::Result { - match list_workspaces(home, record) { + match served(home, record, Call::ListWorkspaces { path: None }) { Err(refusal) if refusal.code == "Unavailable" => Err(Unreached(refusal)), other => Ok(other), } } -/// `sandbox.list_workspaces` on forwarded Sandbox `record` in this process, -/// under the Store's lease (the Sandbox half of `workspace::direct::call`). -fn list_workspaces(home: &Path, record: &SandboxRecord) -> Outcome { +/// The daemon's control-capacity read model of forwarded Sandbox `record` +/// (#540), on the connection the Workspace list lent from this process's +/// pool for it (#543). A refused or unanswered read is the caller's to omit. +pub fn capacity(home: &Path, record: &SandboxRecord) -> Outcome { + served(home, record, Call::Capacity) +} + +/// Host `call` on forwarded Sandbox `record` in this process, under the +/// Store's lease (the Sandbox half of `workspace::direct::call`). +fn served(home: &Path, record: &SandboxRecord, call: Call) -> Outcome { let home = home.to_path_buf(); let target = resolve::target(&home, record.project().unwrap_or(""))?; let _lease = lease::hold(&home, &target.store)?; - dispatch::call( - host(&home, record, &target, Message::Absent), - Call::ListWorkspaces { path: None }, - ) + dispatch::call(host(&home, record, &target, Message::Absent), call) } /// The Workspaces forwarded Sandbox `record` holds, which its close sends diff --git a/control/crates/layerfs-sandbox/src/dispatch.rs b/control/crates/layerfs-sandbox/src/dispatch.rs index 09f3a68ccf..f610b3bf54 100644 --- a/control/crates/layerfs-sandbox/src/dispatch.rs +++ b/control/crates/layerfs-sandbox/src/dispatch.rs @@ -19,6 +19,9 @@ pub fn call(host: Host, call: Call) -> Outcome { let directory = crate::files::sandbox_dir(&host)?; listed(daemon_backend(&host, &call)?, &directory) } + // the operator view of control admission and retained failures + // (#540): a bounded read model, never a product operation + Call::Capacity => capacity(&host), Call::Create { platform } => crate::lifecycle::create(&host, platform.as_deref()), Call::Close { force, held } => { crate::lifecycle::close(&host, force, held.as_deref().unwrap_or_default()) @@ -103,6 +106,15 @@ fn daemon_backend(host: &Host, call: &Call) -> Outcome { }) } +/// The daemon's `backend.daemon.connection_failures` read model (#540): the +/// connection the Workspace list lent from the Sandbox's pool, under one +/// bound for admission and the exchange (#543). Nothing starts: the caller +/// asks only a Sandbox this process still reaches. +fn capacity(host: &Host) -> Outcome { + let state = registry::sandbox_live(host)?; + crate::daemon::capacity::lent(&state.controls) +} + /// Refuses an invalid Commit `message` before any effect. pub fn checked(message: &Message) -> Result<(), layerfs_bridge::control::SdkRefusal> { match message { diff --git a/control/crates/layerfs-sandbox/src/files.rs b/control/crates/layerfs-sandbox/src/files.rs index e120409b6f..f952590740 100644 --- a/control/crates/layerfs-sandbox/src/files.rs +++ b/control/crates/layerfs-sandbox/src/files.rs @@ -229,10 +229,29 @@ fn recorded_identity(value: &DaemonRecord) -> CommandIdentity { pub fn refresh(state: &SandboxState) -> Result<()> { refresh_except(state, &[]) } +/// [`refresh`], waiting at most `wait` in total for all status probes. An +/// unmount uses a short bound so a refusing daemon returns its exact refusal +/// instead of blocking behind the folder guard (#540). +pub fn refresh_within(state: &SandboxState, wait: std::time::Duration) -> Result<()> { + refresh_except_within(state, &[], wait) +} /// Explicit Sandbox close may dispose retained unmounts without asking their /// daemon for a Ready receipt that no longer exists. Other records still /// undergo the ordinary refresh and MountsActive admission. pub fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { + refresh_except_within(state, retained, crate::daemon::pool::ADMISSION_WAIT) +} +/// [`refresh_except`], waiting at most `wait` in total for all Workspace +/// status probes. One deadline is shared by every Workspace, so N records +/// cannot multiply the wait (#540). An explicit close uses a short wait so its +/// `op` intent is not held for the whole admission wait while the daemon +/// refuses connections. +pub fn refresh_except_within( + state: &SandboxState, + retained: &[String], + wait: std::time::Duration, +) -> Result<()> { + let deadline = std::time::Instant::now() + wait; let mut mounts = std::collections::BTreeMap::new(); for (_, value) in workspaces(&state.directory)? { let path = helpers::required(value.path(), "path")?; @@ -240,13 +259,18 @@ pub fn refresh_except(state: &SandboxState, retained: &[String]) -> Result<()> { continue; } let token = token_of(&value)?; - let status = - match workspace(&mut *crate::daemon::trace::control(&state.controls)?).status(token) { - Ok(status) => status, - // Unmounted by the daemon; its owner removes the record next. - Err(failure) if absent(&failure) => continue, - Err(failure) => return Err(failed(failure)), - }; + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + let status = match workspace(&mut *crate::daemon::trace::control_within( + &state.controls, + remaining, + )?) + .status(token) + { + Ok(status) => status, + // Unmounted by the daemon; its owner removes the record next. + Err(failure) if absent(&failure) => continue, + Err(failure) => return Err(failed(failure)), + }; let ready = status .kernel .and_then(|k| k.ready) diff --git a/control/crates/layerfs-sandbox/src/get.rs b/control/crates/layerfs-sandbox/src/get.rs index 3dbb7b4445..cb8485bd77 100644 --- a/control/crates/layerfs-sandbox/src/get.rs +++ b/control/crates/layerfs-sandbox/src/get.rs @@ -4,7 +4,7 @@ use super::family::{page, sandbox_of, Page}; use super::record::SandboxRecord; use crate::host::Call; use crate::route::reach; -use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::control::{CapacityModel, SdkRefusal, SdkValue}; use layerfs_home::events::Events; use layerfs_machine::worker::Unreached; use std::path::Path; @@ -44,6 +44,9 @@ pub struct Got { pub down_reason: Option, /// The next page's position; `None` after the last. pub next: Option, + /// Its daemon's control-capacity read model (#540), when a running + /// Sandbox answered the bounded read (#543). + pub daemon_capacity: Option, } /// Its record with one page of its Workspaces. Reading never clears an @@ -80,15 +83,47 @@ pub fn get(home: &Path, events: &mut Events<'_>, request: &GetRequest) -> Result Err(Unreached(why)) => (Vec::new(), Some(why.message), false), }; let (workspaces, next) = page(all, request.page); + let state = record.state(gone); + // best effort, bounded, and only for a Sandbox its daemon still serves: + // this read never fails a get (#540, #543) + let down = down_reason.is_some(); + let daemon_capacity = capacity(&home, events, &record, gone, state, down); Ok(Got { - state: record.state(gone), + state, record, workspaces, down_reason, next, + daemon_capacity, }) } +/// The daemon's `backend.daemon.connection_failures` read model of a Sandbox +/// this process still reaches (#540): read once, best effort, and omitted +/// when the Sandbox is gone, is not running, or its machine or forward is +/// already down — a get dials nothing for one of those (#543). +fn capacity( + home: &Path, + events: &mut Events<'_>, + record: &SandboxRecord, + gone: bool, + state: &str, + down: bool, +) -> Option { + if gone || down || state != "running" { + return None; + } + record.id()?; + // the forwarded daemon is dialed from here, its machine's from its worker: + // either route reads it with the connection the Workspace list lent + let answer = if direct::forwarded(record) { + direct::capacity(home, record) + } else { + reach(home, events, record, Call::Capacity, crate::dispatch::call).ok()? + }; + Some(CapacityModel::parse(&answer.ok()?)) +} + /// Workspace `item` of Sandbox `sandbox`, as its daemon answered it. fn listed_workspace(sandbox: &str, item: &SdkValue) -> ListedWorkspace { let path = (item.get_text("path")) diff --git a/control/crates/layerfs-sandbox/src/host/call.rs b/control/crates/layerfs-sandbox/src/host/call.rs index f3263bcb67..92ced5f1d5 100644 --- a/control/crates/layerfs-sandbox/src/host/call.rs +++ b/control/crates/layerfs-sandbox/src/host/call.rs @@ -19,6 +19,9 @@ pub enum Op { SandboxClose, /// `sandbox.list_workspaces`: its daemon's Workspace list. ListWorkspaces, + /// `backend.daemon.connection_failures`: its daemon's control-capacity + /// read model (#540), a bounded operator read. + Capacity, /// `workspace.mount`. Mount, /// `workspace.exec`. @@ -35,10 +38,11 @@ pub enum Op { /// Every host operation and the name a request carries (`op`). A /// machine's worker runs exactly these. -pub const OPS: [(Op, &str); 9] = [ +pub const OPS: [(Op, &str); 10] = [ (Op::SandboxCreate, "sandbox.create"), (Op::SandboxClose, "sandbox.close"), (Op::ListWorkspaces, "sandbox.list_workspaces"), + (Op::Capacity, layerfs_bridge::control::CAPACITY_OPERATION), (Op::Mount, "workspace.mount"), (Op::Exec, "workspace.exec"), (Op::ExecCancel, "workspace.exec_cancel"), @@ -100,6 +104,9 @@ pub enum Call { }, /// `sandbox.list_workspaces`, with the Workspace `path` it may name. ListWorkspaces { path: Option }, + /// `backend.daemon.connection_failures`: its daemon's control-capacity + /// read model (#540). + Capacity, /// `workspace.mount(branch, { name })`. Mount { branch: Option, @@ -149,6 +156,7 @@ impl Call { Self::Create { .. } => Op::SandboxCreate, Self::Close { .. } => Op::SandboxClose, Self::ListWorkspaces { .. } => Op::ListWorkspaces, + Self::Capacity => Op::Capacity, Self::Mount { .. } => Op::Mount, Self::Exec { .. } => Op::Exec, Self::ExecCancel { .. } => Op::ExecCancel, diff --git a/control/crates/layerfs-sandbox/src/host/frame.rs b/control/crates/layerfs-sandbox/src/host/frame.rs index afce72800c..6848360ddc 100644 --- a/control/crates/layerfs-sandbox/src/host/frame.rs +++ b/control/crates/layerfs-sandbox/src/host/frame.rs @@ -81,6 +81,8 @@ fn fields(call: &Call, sent: &Message, sandbox: Option<&str>) -> Vec<(&'static s Call::ListWorkspaces { path } | Call::Status { path } => { fields.extend(path.as_deref().map(|p| ("path", text(p)))) } + // a bounded operator read: it sends no field of its own + Call::Capacity => {} Call::Mount { branch, name } => { fields.push(("branch", opt(branch))); fields.extend(name.as_deref().map(|n| ("name", text(n)))); @@ -248,6 +250,7 @@ fn call(op: Op, more: &SdkValue) -> Call { }, }, Op::ListWorkspaces => Call::ListWorkspaces { path: text("path") }, + Op::Capacity => Call::Capacity, Op::Mount => Call::Mount { branch: text("branch"), name: text("name"), diff --git a/control/crates/layerfs-sandbox/src/lifecycle.rs b/control/crates/layerfs-sandbox/src/lifecycle.rs index 0dd64af826..034496b5e8 100644 --- a/control/crates/layerfs-sandbox/src/lifecycle.rs +++ b/control/crates/layerfs-sandbox/src/lifecycle.rs @@ -27,6 +27,10 @@ pub const STARTUP: Duration = Duration::from_secs(30); /// A failed create's report keeps its daemon's last log lines, up to this many bytes. pub const LOG_LINES: u16 = 20; pub const LOG_BYTES: usize = 2048; +/// How long an explicit close waits for the daemon's Workspace status before +/// it returns the exact admission refusal. The close's `op` intent must not +/// be held for the whole admission wait (#540). +const CLOSE_STATUS_WAIT: Duration = Duration::from_secs(2); pub fn operation(failure: Box) -> SdkRefusal { values::refusal(&failure).unwrap_or_else(|| failed(failure)) @@ -271,7 +275,16 @@ pub fn close(host: &Host, force: bool, held: &[String]) -> Outcome { let unreached = &mut crate::exec::cancel::unreached; crate::exec::cancel::settle(&state.directory, &id, None, force, unreached)?; let recoverable = crate::unmount_guard::check_all(&state)?; - crate::files::refresh_except(&state, &recoverable)?; + if !force { + // A daemon that refuses all control connections must refuse this close + // with its exact cause in bounded time, not hold the `op` intent while + // the SDK's ordinary 60 s admission wait runs out. One shared deadline + // bounds all Workspace probes; N Workspaces cannot multiply it. + crate::files::refresh_except_within(&state, &recoverable, CLOSE_STATUS_WAIT)?; + } + // `force` is the recovery path: it skips the daemon status probe and stops + // the runtime anyway, reporting every Workspace it could not drain as + // `unknown` below. It is still never blocked by the daemon it stops. // Explicit close can take over an orphaned/retained unmount, but must // still refuse every unrelated live Workspace. Stop detaches before // killing the daemon; Delete independently checks physical detach. diff --git a/control/crates/layerfs-sandbox/src/workspace/serve.rs b/control/crates/layerfs-sandbox/src/workspace/serve.rs index b4378700ad..457cfb1356 100644 --- a/control/crates/layerfs-sandbox/src/workspace/serve.rs +++ b/control/crates/layerfs-sandbox/src/workspace/serve.rs @@ -236,6 +236,11 @@ pub fn commit(host: &Host, path: Option<&str>, force: bool) -> Outcome { values::commit(&reply) } +/// How long unmount's Workspace status probe waits for the daemon. After +/// slot reclamation the ordinary path succeeds; a truly exhausted daemon +/// returns `Capacity` in bounded time instead of blocking behind the guard. +const UNMOUNT_STATUS_WAIT: std::time::Duration = std::time::Duration::from_secs(5); + /// `workspace.unmount(ws, { discard?, force? })` of the Workspace at /// `path`: the daemon's answer. It is refused `Busy` while a command of the /// Workspace runs; `force` cancels them first (D-10). A stale command @@ -258,7 +263,10 @@ pub fn unmount(host: &Host, path: Option<&str>, discard: bool, force: bool) -> O let wsid = layerfs_bridge::sdk_values::hex(&token.workspace.to_bytes()); let unreached = &mut crate::exec::cancel::unreached; crate::exec::cancel::settle(&state.directory, &state.id, Some(&wsid), force, unreached)?; - crate::files::refresh(&state)?; + // One short deadline, not the full admission wait: a Workspace unmount + // must not hold the folder guard while a refusing daemon keeps the + // ordinary 60 s control wait running (#540). + crate::files::refresh_within(&state, UNMOUNT_STATUS_WAIT)?; // the daemon's kernel probe answers Busy for a process still holding it let attempt = crate::unmount_guard::begin(&state, token, path)?; drop(guard); diff --git a/control/crates/layerfs-sandbox/tests/capacity_read.rs b/control/crates/layerfs-sandbox/tests/capacity_read.rs new file mode 100644 index 0000000000..b90709a375 --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/capacity_read.rs @@ -0,0 +1,355 @@ +//! The `backend.daemon.connection_failures` read model's bounds (#540, +//! #543): one query ends its control session instead of dropping the socket, +//! a daemon that trickles is fenced at the deadline, a read on the Sandbox's +//! pool takes no second connection, and a connection a fence or a session end +//! quarantined is never lent again or reported as an attempt. +//! +//! Nothing runs: a stand-in daemon answers the query on the connections it +//! accepts, and counts what it saw. +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request, + RetainedModel, SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; +use layerfs_home::home::paths; +use layerfs_home::secrets; +use layerfs_sandbox::daemon::capacity; +use layerfs_sandbox::daemon::channel::ControlPhase; +use layerfs_sandbox::daemon::operator; +use layerfs_sandbox::daemon::pool::{Controls, Dial}; +use layerfs_sandbox::host::{Call as HostCall, Host, Message}; +use layerfs_sandbox::record::SandboxRecord; +use layerfs_sandbox::{OperationCause, SdkError}; +use std::io::Write; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("lfs-capr-{tag}-{}-{sequence}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +/// A model of a pool with no vacant slot (and no retained failure). +fn model() -> SdkValue { + CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel::default(), + } + .to_sdk() +} + +fn dial(endpoint: SocketAddr) -> Dial { + Dial { + endpoint, + secret: CLIENT_PRIVATE, + peer: duplex::public_key(&DAEMON_PRIVATE).unwrap(), + instance: DAEMON_INSTANCE, + } +} + +/// What a stand-in daemon saw, and how it treats the query. +#[derive(Default)] +struct Seen { + connections: AtomicUsize, + queries: AtomicUsize, + ended: AtomicBool, + fenced: AtomicBool, +} + +/// How the stand-in answers the capacity query. +#[derive(Clone, Copy)] +enum Behavior { + /// With the model. + Model, + /// Never: it trickles one byte every `TRICKLE` until the far side closes + /// the socket (`Seen::fenced`). + Trickle, +} + +const TRICKLE: Duration = Duration::from_millis(400); + +/// A stand-in daemon at its own loopback port. +fn daemon(answer: Behavior) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let seen = Arc::new(Seen::default()); + let counted = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + counted.connections.fetch_add(1, Ordering::SeqCst); + let seen = counted.clone(); + std::thread::spawn(move || serve(stream, answer, &seen)); + } + }); + (endpoint, seen) +} + +/// One stand-in connection: its Hello, then each call's answer; a query ends +/// the session when the caller sends its end. +fn serve(stream: TcpStream, answer: Behavior, seen: &Seen) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + // kept for a trickling daemon: ciphertext is the channel's to write + let mut raw = stream.try_clone().ok(); + let Ok(mut channel) = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) else { + return; + }; + while let Ok(bytes) = channel.receive.receive() { + let Ok(call) = Call::decode(bytes) else { break }; + let id = call.id; + let reply = match call.request { + Request::Hello(_) => Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + Request::EndSession => { + seen.ended.store(true, Ordering::SeqCst); + Reply::SessionEnded + } + Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + seen.queries.fetch_add(1, Ordering::SeqCst); + if matches!(answer, Behavior::Trickle) { + if let Some(raw) = raw.as_mut() { + trickle(raw, seen); + } + break; + } + Reply::Sdk(model()) + } + _ => continue, + }; + if channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .is_err() + { + break; + } + } +} + +/// Keeps the far side waiting without ever completing a record, until it ends +/// the connection itself. +fn trickle(raw: &mut TcpStream, seen: &Seen) { + let byte = [0u8; 1]; + for _ in 0..20 { + std::thread::sleep(TRICKLE); + if raw.write_all(&byte).and_then(|()| raw.flush()).is_err() { + seen.fenced.store(true, Ordering::SeqCst); + return; + } + } +} + +/// Sandbox `s1`'s folder in `home`, recording the daemon keys its queries +/// read (`control.json`) and, with `forward`, the record that reaches its +/// daemon through that forward. +fn sandbox_folder(home: &Path, endpoint: SocketAddr, forward: bool) { + let dir = paths::sandbox_dir(home, "s1"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let keys = SdkValue::map([ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]); + secrets::save_daemon_keys(&dir, &keys).unwrap(); + if forward { + let record = SdkValue::map([ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("s-1")), + ( + "daemon", + SdkValue::map([( + "forward", + SdkValue::map([ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]), + )]), + ), + ]); + layerfs_home::home::records::write_record(&dir.join("record.json"), &record).unwrap(); + layerfs_home::home::records::write_record( + &paths::worker_record(home, "native"), + &SdkValue::map([("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + } +} + +fn record(name: &str) -> SandboxRecord { + SandboxRecord::from_record(name, SdkValue::map([])) +} + +#[test] +fn a_query_ends_its_control_session() { + let (endpoint, seen) = daemon(Behavior::Model); + let home = tempdir("session"); + sandbox_folder(&home, endpoint, false); + let got = capacity::read(&home, &record("s1"), endpoint).unwrap(); + assert_eq!(got.admission.pool_running, 8); + // the one conversation ended before the socket was dropped, which the + // daemon would otherwise log as an unexpected end + assert!(seen.ended.load(Ordering::SeqCst), "no EndSession was sent"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_trickling_daemon_is_fenced_at_the_deadline() { + let (endpoint, seen) = daemon(Behavior::Trickle); + let home = tempdir("trickle"); + sandbox_folder(&home, endpoint, false); + let at = Instant::now(); + let refused = capacity::read(&home, &record("s1"), endpoint).err(); + let elapsed = at.elapsed(); + assert!(refused.is_some(), "{refused:?}"); + assert!( + elapsed < capacity::WAIT + Duration::from_millis(1500), + "the read outlived its deadline: {elapsed:?}" + ); + // the socket was shut down, not merely abandoned: the daemon's own write + // failed + let until = Instant::now() + Duration::from_secs(3); + while !seen.fenced.load(Ordering::SeqCst) && Instant::now() < until { + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + seen.fenced.load(Ordering::SeqCst), + "the fence did not close the connection" + ); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_pooled_read_takes_no_second_connection() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + let one = capacity::lent(&controls).unwrap(); + let two = capacity::lent(&controls).unwrap(); + assert_eq!(CapacityModel::parse(&one).admission.pool_vacant, 0); + assert_eq!(CapacityModel::parse(&two).admission.pool_running, 8); + // both reads went over the one connection the pool holds, so a get takes + // a single control slot for the list and the capacity read together + assert_eq!(seen.connections.load(Ordering::SeqCst), 1); + assert_eq!(seen.queries.load(Ordering::SeqCst), 2); +} + +#[test] +fn the_worker_s_side_reads_the_capacity_model() { + let (endpoint, seen) = daemon(Behavior::Model); + let home = tempdir("worker"); + std::fs::create_dir_all(&home).unwrap(); + sandbox_folder(&home, endpoint, true); + // the request a machine's worker decodes (`sandbox.host`) and the host + // operation it runs: no Store is opened and no image is loaded + let host = Host { + home: home.clone(), + store: "shop".to_owned(), + project: "api".to_owned(), + sandbox: Some("s1".to_owned()), + store_record: None, + store_path: None, + deploy: None, + postgresql: None, + message: Message::Absent, + started: None, + }; + let answer = layerfs_sandbox::dispatch::call(host, HostCall::Capacity).unwrap(); + let model = CapacityModel::parse(&answer); + assert_eq!(model.admission.pool_vacant, 0, "{answer:?}"); + assert_eq!(model.admission.reserved_vacant, 1, "{answer:?}"); + assert_eq!(seen.queries.load(Ordering::SeqCst), 1); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_fenced_connection_is_not_lent_again() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + { + let lease = controls.lend().unwrap().0; + // the fence's end, outside any exchange: the channel is shut down + // while this `Control` never saw a call fail, which is the window + // between a returned reply and `drop(fence)` + lease.close_handle().unwrap().close().unwrap(); + assert!( + lease.is_failed(), + "a channel a fence closed was still reported healthy" + ); + } + // and the pool closed it rather than lending it again: the next read + // opens a connection and answers, instead of reporting the dead one + let answer = capacity::lent(&controls).unwrap(); + assert_eq!(CapacityModel::parse(&answer).admission.pool_vacant, 0); + assert_eq!(seen.connections.load(Ordering::SeqCst), 2); +} + +#[test] +fn a_quarantined_connection_attempts_nothing() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + let mut lease = controls.lend().unwrap().0; + lease.close_handle().unwrap().close().unwrap(); + // the call a pooled lease would make next: refused before sending, so it + // is not an attempt whose effect is unknown + let error = operator::call(&mut lease, CAPACITY_OPERATION, SdkValue::Null).unwrap_err(); + match error { + SdkError::Operation(failure) => match failure.cause { + OperationCause::Exchange(channel) => { + assert!( + !channel.attempted, + "a closed channel was reported attempted" + ); + assert_eq!(channel.phase, ControlPhase::Encode); + } + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + } + assert_eq!(seen.queries.load(Ordering::SeqCst), 0); +} diff --git a/control/crates/layerfs-sandbox/tests/host_frame.rs b/control/crates/layerfs-sandbox/tests/host_frame.rs index d43a315aa9..a940d00b22 100644 --- a/control/crates/layerfs-sandbox/tests/host_frame.rs +++ b/control/crates/layerfs-sandbox/tests/host_frame.rs @@ -74,6 +74,7 @@ fn a_frame_reads_back_to_its_request() { held: None, }, Call::ListWorkspaces { path: None }, + Call::Capacity, Call::Mount { branch: Some("dev".to_owned()), name: Some("w2".to_owned()), diff --git a/control/crates/layerfs-sandbox/tests/pool_admission.rs b/control/crates/layerfs-sandbox/tests/pool_admission.rs new file mode 100644 index 0000000000..cbd9429ffd --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/pool_admission.rs @@ -0,0 +1,82 @@ +//! A daemon's advisory capacity notice (#540) as a client reads it: a +//! daemon whose pool has no vacant slot answers the next connection with +//! the notice before any handshake, so the refusal is `Capacity` naming the +//! resource, the vacant slots and the retained failure count, not a bare +//! EOF. Nothing is sent by the client and nothing is retried, so the caller +//! may try again. A daemon that is still starting says so the same way. +use layerfs_bridge::duplex; +use layerfs_bridge::duplex::admission::{self, AdmissionNotice, AdmissionReason}; +use layerfs_sandbox::daemon::pool::Dial; +use std::io::Read; +use std::net::{SocketAddr, TcpListener}; +use std::time::Duration; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// A daemon that refuses every connection with `notice` before any +/// handshake, exactly as an accept loop does when no slot is vacant: it +/// writes the notice, shuts its write end and drains what the client sent. +fn refusing(notice: AdmissionNotice) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let _ = stream.set_write_timeout(Some(Duration::from_millis(100))); + let _ = stream.set_read_timeout(Some(Duration::from_millis(50))); + if admission::write(&mut stream, ¬ice).is_err() { + continue; + } + let mut sink = [0u8; 512]; + let _ = stream.read(&mut sink); + } + }); + endpoint +} + +fn dial(endpoint: SocketAddr) -> Dial { + Dial { + endpoint, + secret: CLIENT_PRIVATE, + peer: duplex::public_key(&DAEMON_PRIVATE).unwrap(), + instance: DAEMON_INSTANCE, + } +} + +#[test] +fn a_full_pool_refuses_the_next_client_with_its_counts() { + let endpoint = refusing(AdmissionNotice::new(AdmissionReason::NoPoolSlot, 3, 0, 1)); + let refusal = dial(endpoint) + .open_within(Duration::from_secs(2)) + .err() + .expect("a full pool admits nothing"); + assert_eq!(refusal.code, "Capacity", "{refusal:?}"); + let detail = &refusal.detail; + assert_eq!(detail.get_text("resource"), Some("daemon_control_slot")); + assert_eq!(detail.get_int("pool_vacant"), Some(0)); + assert_eq!(detail.get_int("reserved_vacant"), Some(1)); + assert_eq!(detail.get_int("retained"), Some(3)); + assert_eq!(detail.get_text("admission"), Some("no-pool-slot")); + // nothing was attempted, so a later open is safe + assert!(detail.get_bool("retryable")); + assert!( + refusal.message.contains("no vacant control slot"), + "{refusal:?}" + ); + assert!(refusal.message.contains("3 retained"), "{refusal:?}"); +} + +#[test] +fn a_still_starting_daemon_is_not_a_full_pool() { + let endpoint = refusing(AdmissionNotice::new(AdmissionReason::Starting, 0, 8, 1)); + let refusal = dial(endpoint) + .open_within(Duration::from_secs(2)) + .err() + .expect("a starting daemon admits nothing"); + // its own code and admission, not Capacity: a full pool it is not + assert_eq!(refusal.code, "Unavailable", "{refusal:?}"); + assert_eq!(refusal.detail.get_text("admission"), Some("starting")); + assert!(refusal.detail.get_bool("retryable")); +} diff --git a/core/docs/issues/540/control-slot-custody.png b/core/docs/issues/540/control-slot-custody.png new file mode 100644 index 0000000000..6a01c53eae Binary files /dev/null and b/core/docs/issues/540/control-slot-custody.png differ diff --git a/core/docs/issues/540/recovery-sequence.png b/core/docs/issues/540/recovery-sequence.png new file mode 100644 index 0000000000..3b8373f6c2 Binary files /dev/null and b/core/docs/issues/540/recovery-sequence.png differ diff --git a/docs/architecture/api/sdk/sandbox/README.md b/docs/architecture/api/sdk/sandbox/README.md index 759acfeacd..d1724454f6 100644 --- a/docs/architecture/api/sdk/sandbox/README.md +++ b/docs/architecture/api/sdk/sandbox/README.md @@ -36,7 +36,7 @@ unique across the home. | --- | --- | --- | --- | --- | | [`create`](create.md) | `project` (Project), `name` (Name), `machine?` (Name) | `image` (Text) | the Sandbox record with `state`; `forward_failed` when a direct daemon forward did not open | its machine: here or `sandbox.host` in the worker | | [`list`](list.md) | `project?` (Project) | `page_size`, `page_pos` (Int) | `{ items: [record], next_page_pos }` | caller's process, home records only | -| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, and `down_reason` when its machine is down | its machine or its daemon; starts nothing | +| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing | | [`close`](close.md) | `name` (Name) | `force` (Bool) | `{ kind: "Closed", sandbox, unknown, closed }`, or `{ closed, was_gone: true }`, or `{ closed, was, left? }` for a record with no id | its machine: here or `sandbox.host` in the worker | The record is `SandboxRecord::to_sdk` @@ -47,7 +47,11 @@ its stored fields except `v`, `runtime_id` and `daemon`, plus the derived ([`op.rs`](../../../../../control/crates/layerfs-home/src/op.rs)). Each Workspace in `get` carries `ws` (`:` or `:`), `name` and `path` -([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)). +([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)), and +its `daemon_capacity` is the daemon's `backend.daemon.connection_failures` +read model, read best effort and bounded to 2 s on the connection the +Workspace list already lent +([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)). ## Arguments and options diff --git a/docs/architecture/api/sdk/sandbox/get.md b/docs/architecture/api/sdk/sandbox/get.md index 0a91d70c2b..accb9c91d5 100644 --- a/docs/architecture/api/sdk/sandbox/get.md +++ b/docs/architecture/api/sdk/sandbox/get.md @@ -11,8 +11,8 @@ its daemon holds, without starting anything. | --- | --- | | Family | [sandbox](README.md) | | Library entry | [`sandbox/get.rs`](../../../../../api/crates/layerfs-api/src/sandbox/get.rs) | -| Implemented in | the library entry parses a typed `GetRequest` → `layerfs_sandbox::family::get` → `get::get`, whose typed `Got` it shapes ([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)) | -| Runs in | the caller's process; the Workspace list comes from the Sandbox daemon, dialed from here (forwarded or local) or from the machine's worker | +| Implemented in | the library entry parses a typed `GetRequest` → `layerfs_sandbox::family::get` → `get::get`, whose typed `Got` (its `daemon_capacity` included) it shapes ([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)) | +| Runs in | the caller's process; the Workspace list and the daemon's capacity read model come from the Sandbox daemon, dialed from here (forwarded or local) or from the machine's worker | ## Signature @@ -58,7 +58,15 @@ layerfs sandbox get [--page-size ] [--page-pos ] head_moved_since_mount, dirty: {inodes, entries}?, pending_merge, revision } ], next_page_pos: int | null, - down_reason?: text } + down_reason?: text, + daemon_capacity?: { admission: { pool_vacant, pool_running, + reserved_vacant, reserved_running }, + retained: { limit, ttl_ms, count, dropped, recorded, + failures: [ { slot, class, generation, owner, + received_bytes, request, reply_sent, + cause, fence_error, + diagnostic_error, age_ms, + expires_in_ms } ] } } } ``` | Field | Kind | Meaning | @@ -70,6 +78,7 @@ layerfs sandbox get [--page-size ] [--page-pos ] | `workspaces[].*` | | the daemon's read model per Workspace: id, namespace, Branch, base Commit and Layer, root, `activity`, dirty counts ([`operator/inspect.rs`](../../../../../engine/crates/layerfs-store-service/src/operator/inspect.rs) `workspace_value`) | | `next_page_pos` | int or null | next page of `workspaces` | | `down_reason` | text | the machine's worker or the forwarded daemon was not reached; `workspaces` is then `[]` | +| `daemon_capacity` | record | the daemon's `backend.daemon.connection_failures` read model (#540): control admission by slot class and the bounded retained connection failures that used to hold a slot ([`control_capacity.rs`](../../../../../engine/crates/layerfs-bridge/src/control_capacity.rs)). It is read once, best effort, and only for a Sandbox that is `running` and whose machine and forward were reached; nothing is started and no second control slot is taken, so the field is simply absent when the daemon refuses, does not answer within 2 s, or was not asked ([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)) | ## Where it runs @@ -86,10 +95,23 @@ layerfs sandbox get [--page-size ] [--page-pos ] v Unreached / Unavailable --> record + workspaces [] + down_reason NoSuchSandbox --> state "lost", workspaces [] + + running and not down, same route and the same pooled connection: + backend.daemon.connection_failures --> daemon_capacity (2 s, best effort) ``` `get` starts no machine and loads no image (#535 P4); reading never clears -an intent. See the [routing table](../../../control/routing/routing-table.md) and the +an intent. The capacity read is a second host operation on the connection +the Workspace list lent from the Sandbox's pool +([`daemon/pool.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs)), +bounded to 2 s for admission and for +the exchange and fenced at that deadline ([`daemon/fence.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs)), +so a daemon that trickles cannot hold the slot, and a connection the fence +ends is quarantined rather than lent again; a window the admission wait used +up refuses the read before it is sent. A forwarded daemon's forward +and its machine are checked before it is dialed (`direct::check`), so a +stopped machine is `down_reason` and never a 2 s wait on a reused port. See +the [routing table](../../../control/routing/routing-table.md) and the [daemon channel](../../../control/routing/daemon-channel.md). ## Errors diff --git a/docs/architecture/control/routing/daemon-channel.md b/docs/architecture/control/routing/daemon-channel.md index a788b0c3ad..67bd11168d 100644 --- a/docs/architecture/control/routing/daemon-channel.md +++ b/docs/architecture/control/routing/daemon-channel.md @@ -33,7 +33,7 @@ side of the channel is in [Engine: Daemon](../../engine/daemon/README.md). | | | lend(): idle one --> else open one more (if open < limit) | | --> else wait for a free one, <= 60 s --> Busy | - | drop(Lease): failed Control --> closed, open -= 1 | + | drop(Lease): quarantined Control --> closed, open -= 1 | | healthy Control --> back to idle | +--------------------------------------------------------------+ | @@ -43,7 +43,7 @@ side of the channel is in [Engine: Daemon](../../engine/daemon/README.md). | Part | What it is | Source | | --- | --- | --- | -| `Control` | one sequential authenticated connection; numbers its calls; once an exchange fails it is quarantined and refuses every later call before sending | [`daemon/channel.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/channel.rs) | +| `Control` | one sequential authenticated connection; numbers its calls; once an exchange fails, or an applied fence (`close_handle`, `daemon/fence.rs`) closes its channel, it is quarantined and refuses every later call before sending | [`daemon/channel.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/channel.rs) | | `Dial` | how to open another connection to the same daemon instance: endpoint, secret, peer key, instance | [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | | `Controls` | the Sandbox's pool: up to `CONNECTIONS` (6) per process; the daemon admits `DAEMON_CONNECTIONS` (8), leaving the rest to other processes | `daemon/pool.rs`, [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) | | `Lease` | one connection lent for one call's exchanges, returned on drop | `daemon/pool.rs` | @@ -82,7 +82,7 @@ opening a connection sends only Hello, so it attempts no operation. | `workspace.commit` | `Commit` | `values::commit`: Committed, Merged (with `merged`), or `Conflict` | [`daemon/workspace/commit.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/commit.rs), [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | `workspace.unmount` | `Unmount{token, discard}`, or `ForceUnmount` with `force` | `values::unmounted` (`Dirty` refusal) or `force_unmounted` | [`daemon/workspace/unmount.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/unmount.rs) | | `workspace.status` | `Status(token)`; the Branch head is read separately on the Store's route | `values::status` | [`daemon/workspace/status.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/status.rs) | -| `sandbox.get` Workspace list | SDK method `sandbox.list_workspaces`, the one daemon backend method sent | `sandbox/dispatch.rs` | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs) | +| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs); a window admission used up refuses the read, and a connection the fence ends is not lent again) | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | | `workspace.exec` | none on this channel: `bash -c` runs through the Sandbox's runtime | — | [`exec/run.rs`](../../../../control/crates/layerfs-sandbox/src/exec/run.rs) | `Locate` and `Cleanup` are read-only observations for a caller whose diff --git a/engine/crates/layerfs-bridge/src/control.rs b/engine/crates/layerfs-bridge/src/control.rs index 727da40470..cb8dd79c5c 100644 --- a/engine/crates/layerfs-bridge/src/control.rs +++ b/engine/crates/layerfs-bridge/src/control.rs @@ -164,6 +164,10 @@ fn boolean(value: u8) -> Result { } } +pub use crate::control_capacity::{ + AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, + OPERATION as CAPACITY_OPERATION, +}; pub use crate::control_forced::{ AbortDisposition, CommitKnowledge, DetachDisposition, ForceUnmounted, ForcedCleanup, ForcedFacts, ForcedOutcome, diff --git a/engine/crates/layerfs-bridge/src/control_capacity.rs b/engine/crates/layerfs-bridge/src/control_capacity.rs new file mode 100644 index 0000000000..61fe9819f3 --- /dev/null +++ b/engine/crates/layerfs-bridge/src/control_capacity.rs @@ -0,0 +1,214 @@ +//! The daemon's control-capacity read model (#540). +//! +//! One typed shape is the contract between the daemon that publishes it, the +//! operator read that forwards it, and the doctor that explains it: a +//! [`CapacityModel`] with one [`AdmissionModel`] (vacant and in-use control +//! slots by class) and one [`RetainedModel`] (the bounded store of connection +//! failures that used to retain a slot). It is a plain value object: parse a +//! daemon answer once, then read typed fields instead of string keys. +use crate::control::SdkValue; + +/// The daemon backend method that returns this read model. +pub const OPERATION: &str = "backend.daemon.connection_failures"; + +/// Vacant and in-use control slots by admission class. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct AdmissionModel { + /// Pool slots that no connection holds. + pub pool_vacant: i64, + /// Pool slots that an admitted connection holds. + pub pool_running: i64, + /// Reserved installer slots that no connection holds. + pub reserved_vacant: i64, + /// Reserved installer slots that an admitted connection holds. + pub reserved_running: i64, +} + +impl AdmissionModel { + /// The model in `value`, or all zero when the record does not carry one. + pub fn parse(value: &SdkValue) -> Self { + let admission = value.get("admission").unwrap_or(&SdkValue::Null); + Self { + pool_vacant: admission.get_int("pool_vacant").unwrap_or(0), + pool_running: admission.get_int("pool_running").unwrap_or(0), + reserved_vacant: admission.get_int("reserved_vacant").unwrap_or(0), + reserved_running: admission.get_int("reserved_running").unwrap_or(0), + } + } + + /// The model as the read model's `admission` record. + pub fn to_sdk(&self) -> SdkValue { + SdkValue::map([ + ("pool_vacant", SdkValue::Int(self.pool_vacant)), + ("pool_running", SdkValue::Int(self.pool_running)), + ("reserved_vacant", SdkValue::Int(self.reserved_vacant)), + ("reserved_running", SdkValue::Int(self.reserved_running)), + ]) + } + + /// Whether the pool has no vacant slot while it is in use at all. + pub fn exhausted(&self) -> bool { + self.pool_vacant == 0 && self.pool_running > 0 + } +} + +/// One connection failure that used to retain a control slot. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetainedFailureModel { + /// Slot the failed worker occupied; it is already free for a new connection. + pub slot: i64, + /// `pool` or `reserved`: the class of the slot that was released. + pub class: String, + /// Admission generation of the released worker. + pub generation: i64, + /// The worker that failed: daemon instance, pid and thread. + pub owner: String, + /// Bytes of a request that failed decoding, when one was received. + pub received_bytes: Option, + /// `control` or `install` when a request was decoded. + pub request: Option, + /// Whether the original reply was sent before the failure. + pub reply_sent: bool, + /// Bounded textual deciding cause. + pub cause: String, + /// Original socket fence failure, when one was observed. + pub fence_error: Option, + /// Distinct failed diagnostic transfer, when one happened. + pub diagnostic_error: Option, + /// Age at the observation that returned this record, in milliseconds. + pub age_ms: i64, + /// Remaining lifetime at the observation that returned this record, in + /// milliseconds. + pub expires_in_ms: i64, +} + +impl RetainedFailureModel { + /// The record in `value`, or `None` when it is not a failure record. + pub fn parse(value: &SdkValue) -> Option { + Some(Self { + slot: value.get_int("slot")?, + class: value.get_text("class")?.to_owned(), + generation: value.get_int("generation")?, + owner: value.get_text("owner")?.to_owned(), + received_bytes: value.get_int("received_bytes"), + request: value.get_text("request").map(str::to_owned), + reply_sent: value.get_bool("reply_sent"), + cause: value.get_text("cause")?.to_owned(), + fence_error: value.get_text("fence_error").map(str::to_owned), + diagnostic_error: value.get_text("diagnostic_error").map(str::to_owned), + age_ms: value.get_int("age_ms")?, + expires_in_ms: value.get_int("expires_in_ms")?, + }) + } + + /// The record as the read model's `retained.failures` item. + pub fn to_sdk(&self) -> SdkValue { + SdkValue::map([ + ("slot", SdkValue::Int(self.slot)), + ("class", SdkValue::text(&self.class)), + ("generation", SdkValue::Int(self.generation)), + ("owner", SdkValue::text(&self.owner)), + ( + "received_bytes", + self.received_bytes.map_or(SdkValue::Null, SdkValue::Int), + ), + ("request", SdkValue::opt_text(self.request.clone())), + ("reply_sent", SdkValue::Bool(self.reply_sent)), + ("cause", SdkValue::text(&self.cause)), + ("fence_error", SdkValue::opt_text(self.fence_error.clone())), + ( + "diagnostic_error", + SdkValue::opt_text(self.diagnostic_error.clone()), + ), + ("age_ms", SdkValue::Int(self.age_ms)), + ("expires_in_ms", SdkValue::Int(self.expires_in_ms)), + ]) + } +} + +/// The bounded store of connection failures that used to retain a slot. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct RetainedModel { + /// Most records the store keeps at once. + pub limit: i64, + /// How long a record stays operator-visible, in milliseconds. + pub ttl_ms: i64, + /// Visible records. + pub count: i64, + /// Records evicted by the limit or the TTL since startup. + pub dropped: i64, + /// Total records ever recorded since startup. + pub recorded: i64, + /// Visible records, oldest first. + pub failures: Vec, +} + +impl RetainedModel { + /// The model in `value`, or all zero when the record does not carry one. + pub fn parse(value: &SdkValue) -> Self { + let retained = value.get("retained").unwrap_or(&SdkValue::Null); + let failures = match retained.get("failures") { + Some(SdkValue::List(items)) => items + .iter() + .filter_map(RetainedFailureModel::parse) + .collect(), + _ => Vec::new(), + }; + Self { + limit: retained.get_int("limit").unwrap_or(0), + ttl_ms: retained.get_int("ttl_ms").unwrap_or(0), + count: retained.get_int("count").unwrap_or(0), + dropped: retained.get_int("dropped").unwrap_or(0), + recorded: retained.get_int("recorded").unwrap_or(0), + failures, + } + } + + /// The model as the read model's `retained` record. + pub fn to_sdk(&self) -> SdkValue { + SdkValue::map([ + ("limit", SdkValue::Int(self.limit)), + ("ttl_ms", SdkValue::Int(self.ttl_ms)), + ("count", SdkValue::Int(self.count)), + ("dropped", SdkValue::Int(self.dropped)), + ("recorded", SdkValue::Int(self.recorded)), + ( + "failures", + SdkValue::List( + self.failures + .iter() + .map(RetainedFailureModel::to_sdk) + .collect(), + ), + ), + ]) + } +} + +/// The daemon's control-capacity read model (#540). +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct CapacityModel { + /// Vacant and in-use control slots by class. + pub admission: AdmissionModel, + /// The bounded retained-failure store. + pub retained: RetainedModel, +} + +impl CapacityModel { + /// Parse one `backend.daemon.connection_failures` answer. A field the + /// record does not carry reads as zero, so a newer daemon stays legible. + pub fn parse(value: &SdkValue) -> Self { + Self { + admission: AdmissionModel::parse(value), + retained: RetainedModel::parse(value), + } + } + + /// The answer a daemon publishes for this model. + pub fn to_sdk(&self) -> SdkValue { + SdkValue::map([ + ("admission", self.admission.to_sdk()), + ("retained", self.retained.to_sdk()), + ]) + } +} diff --git a/engine/crates/layerfs-bridge/src/duplex/admission.rs b/engine/crates/layerfs-bridge/src/duplex/admission.rs new file mode 100644 index 0000000000..820ef6e88a --- /dev/null +++ b/engine/crates/layerfs-bridge/src/duplex/admission.rs @@ -0,0 +1,129 @@ +//! The daemon's advisory pre-handshake capacity notice (#540). +//! +//! When the accept loop finds no vacant control slot it would otherwise just +//! drop the socket, leaving the caller with a bare EOF. This fixed, bounded +//! frame tells the caller which resource was exhausted and how many failure +//! records are retained, so the product refusal can name the real resource. +//! It is written *before* the Noise handshake and is therefore unauthenticated: +//! it admits nothing, carries no secret, and every operation still crosses the +//! authenticated handshake and authorization. A forged notice can at most +//! cause a retry, exactly like a forged reset. +use std::io::Write; +use std::net::TcpStream; + +/// Magic that starts a capacity notice body. +pub const MAGIC: &[u8; 16] = b"LAYERFS-CAPACITY"; +/// The only notice version this build writes or reads. +pub const VERSION: u8 = 1; +/// Length of the body after the two-byte record header. +pub const BODY_LEN: usize = 24; + +/// Why no connection was admitted. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AdmissionReason { + /// Every pool slot was busy (or retained; the count is in `retained`). + NoPoolSlot = 1, + /// The daemon is still starting and admits no control yet. + Starting = 2, + /// The reserved slot is in use and only pool capacity was requested. + ReservedBusy = 3, +} + +impl AdmissionReason { + fn of(byte: u8) -> Option { + match byte { + 1 => Some(Self::NoPoolSlot), + 2 => Some(Self::Starting), + 3 => Some(Self::ReservedBusy), + _ => None, + } + } + + /// Stable text for a refusal detail. + pub fn as_str(self) -> &'static str { + match self { + Self::NoPoolSlot => "no-pool-slot", + Self::Starting => "starting", + Self::ReservedBusy => "reserved-busy", + } + } +} + +/// One decoded advisory notice. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AdmissionNotice { + /// Notice format version (always [`VERSION`] for a decoded notice). + pub version: u8, + /// Why admission ended. + pub reason: AdmissionReason, + /// Visible retained failure records, saturating at 255. + pub retained: u8, + /// Vacant pool slots, saturating at 255. + pub pool_vacant: u8, + /// Vacant reserved slots, saturating at 255. + pub reserved_vacant: u8, +} + +impl AdmissionNotice { + /// A notice with the current format version. + pub fn new( + reason: AdmissionReason, + retained: u8, + pool_vacant: u8, + reserved_vacant: u8, + ) -> Self { + Self { + version: VERSION, + reason, + retained, + pool_vacant, + reserved_vacant, + } + } +} + +/// The exact 24-byte body of `notice`. +pub fn encode(notice: &AdmissionNotice) -> [u8; BODY_LEN] { + let mut body = [0u8; BODY_LEN]; + body[..16].copy_from_slice(MAGIC); + body[16] = notice.version; + body[17] = notice.reason as u8; + body[18] = notice.retained; + body[19] = notice.pool_vacant; + body[20] = notice.reserved_vacant; + body +} + +/// The notice in `bytes`, or `None` when the record is anything else. A +/// random record that is not exactly a valid notice is never accepted. +pub fn classify(bytes: &[u8]) -> Option { + if bytes.len() != BODY_LEN || &bytes[..16] != MAGIC { + return None; + } + if bytes[16] != VERSION { + return None; + } + let reason = AdmissionReason::of(bytes[17])?; + if bytes[21..].iter().any(|byte| *byte != 0) { + return None; + } + Some(AdmissionNotice { + version: bytes[16], + reason, + retained: bytes[18], + pool_vacant: bytes[19], + reserved_vacant: bytes[20], + }) +} + +/// Writes `notice` as one length-prefixed record, then forces the write end +/// shut. The caller sets the socket timeouts. +pub fn write(stream: &mut TcpStream, notice: &AdmissionNotice) -> std::io::Result<()> { + let body = encode(notice); + let mut framed = [0u8; 2 + BODY_LEN]; + framed[..2].copy_from_slice(&(BODY_LEN as u16).to_be_bytes()); + framed[2..].copy_from_slice(&body); + stream.write_all(&framed)?; + stream.flush()?; + stream.shutdown(std::net::Shutdown::Write) +} diff --git a/engine/crates/layerfs-bridge/src/duplex/channel.rs b/engine/crates/layerfs-bridge/src/duplex/channel.rs index 0e56901692..76abb4c192 100644 --- a/engine/crates/layerfs-bridge/src/duplex/channel.rs +++ b/engine/crates/layerfs-bridge/src/duplex/channel.rs @@ -62,6 +62,13 @@ impl Connection { shared: self.receive.shared.clone(), }) } + /// Whether this channel is quarantined: a failed exchange, an ended + /// session or an application fence (`CloseHandle`) shut both directions + /// down, and no record may be sent on it again. All owners read the one + /// shared flag, so a fence's close is visible to whoever borrowed it. + pub fn is_quarantined(&self) -> bool { + self.receive.shared.ready().is_err() + } /// Clears explicit handshake I/O waits on the actual shared socket before long-lived operations. /// An original policy-change error quarantines/fences this owner once. pub fn clear_io_waits(&self) -> ChannelResult<()> { diff --git a/engine/crates/layerfs-bridge/src/duplex/error.rs b/engine/crates/layerfs-bridge/src/duplex/error.rs index 68c62ca309..713f5af76a 100644 --- a/engine/crates/layerfs-bridge/src/duplex/error.rs +++ b/engine/crates/layerfs-bridge/src/duplex/error.rs @@ -1,4 +1,5 @@ //! Transport failures retain original crypto/I/O and failed close evidence. +use super::admission::AdmissionNotice; use std::{fmt, io}; /// One duplex channel failure, without retries or inferred product disposition. @@ -14,6 +15,9 @@ pub enum ChannelError { RecordLimit, /// The physical Noise nonce space is exhausted. NonceExhausted, + /// The daemon refused admission before the handshake and sent its advisory + /// capacity notice (#540). Nothing was attempted; retrying is safe. + Admission(AdmissionNotice), /// Either direction previously failed or explicitly closed this connection. Quarantined, /// Connection quarantine also failed to interrupt socket I/O. @@ -42,6 +46,14 @@ impl fmt::Display for ChannelError { Self::Invalid(what) => write!(f, "invalid duplex channel: {what}"), Self::RecordLimit => f.write_str("duplex record processing window exceeded"), Self::NonceExhausted => f.write_str("duplex Noise nonce exhausted"), + Self::Admission(notice) => write!( + f, + "daemon admission refused ({}): retained {}, pool vacant {}, reserved vacant {}", + notice.reason.as_str(), + notice.retained, + notice.pool_vacant, + notice.reserved_vacant + ), Self::Quarantined => f.write_str("duplex channel quarantined"), Self::CloseFailed { original, close } => { write!(f, "{original}; shutdown also failed: {close}") diff --git a/engine/crates/layerfs-bridge/src/duplex/handshake.rs b/engine/crates/layerfs-bridge/src/duplex/handshake.rs index c532092c12..ea01daefbe 100644 --- a/engine/crates/layerfs-bridge/src/duplex/handshake.rs +++ b/engine/crates/layerfs-bridge/src/duplex/handshake.rs @@ -91,6 +91,11 @@ fn handshake( work.crypto_output_bytes += n as u64; io::write_record(&mut stream, &output[..n], work)?; io::read_record(&mut stream, 256, &mut input, work)?; + // A daemon that could not admit this connection writes a fixed + // advisory notice instead of a Noise message; never treat it as one. + if let Some(notice) = super::admission::classify(&input) { + return Err(ChannelError::Admission(notice)); + } work.crypto_attempts += 1; work.crypto_input_bytes += input.len() as u64; let read = state.read_message(&input, &mut payload)?; diff --git a/engine/crates/layerfs-bridge/src/duplex/mod.rs b/engine/crates/layerfs-bridge/src/duplex/mod.rs index e0bd8b6a4c..82e3d39e03 100644 --- a/engine/crates/layerfs-bridge/src/duplex/mod.rs +++ b/engine/crates/layerfs-bridge/src/duplex/mod.rs @@ -1,4 +1,5 @@ //! One-attempt authenticated duplex channels with bounded borrowed records. +pub mod admission; mod channel; mod ending; mod error; diff --git a/engine/crates/layerfs-bridge/src/lib.rs b/engine/crates/layerfs-bridge/src/lib.rs index e140bfc96b..b820257f67 100644 --- a/engine/crates/layerfs-bridge/src/lib.rs +++ b/engine/crates/layerfs-bridge/src/lib.rs @@ -11,6 +11,7 @@ pub mod provision; mod provision_wire; mod wire; +mod control_capacity; mod control_forced; mod control_history; mod control_merge; diff --git a/engine/crates/layerfs-bridge/tests/admission.rs b/engine/crates/layerfs-bridge/tests/admission.rs new file mode 100644 index 0000000000..e4b421da99 --- /dev/null +++ b/engine/crates/layerfs-bridge/tests/admission.rs @@ -0,0 +1,29 @@ +//! The advisory pre-handshake capacity notice (#540): one fixed 24-byte body +//! round-trips, and a record that is anything else is never a notice. +use layerfs_bridge::duplex::admission::{ + classify, encode, AdmissionNotice, AdmissionReason, BODY_LEN, MAGIC, +}; + +#[test] +fn encode_and_classify_round_trip() { + let notice = AdmissionNotice::new(AdmissionReason::NoPoolSlot, 8, 0, 1); + let body = encode(¬ice); + assert_eq!(body.len(), BODY_LEN); + assert_eq!(&body[..16], MAGIC); + assert_eq!(classify(&body), Some(notice)); +} + +#[test] +fn garbage_is_not_a_notice() { + assert_eq!(classify(&[0u8; BODY_LEN]), None); + assert_eq!(classify(&[0u8; 23]), None); + let mut body = encode(&AdmissionNotice::new(AdmissionReason::NoPoolSlot, 1, 1, 1)); + body[16] = 2; + assert_eq!(classify(&body), None); + let mut body = encode(&AdmissionNotice::new(AdmissionReason::NoPoolSlot, 1, 1, 1)); + body[17] = 0; + assert_eq!(classify(&body), None); + let mut body = encode(&AdmissionNotice::new(AdmissionReason::NoPoolSlot, 1, 1, 1)); + body[23] = 1; + assert_eq!(classify(&body), None); +} diff --git a/engine/crates/layerfs-bridge/tests/capacity_model.rs b/engine/crates/layerfs-bridge/tests/capacity_model.rs new file mode 100644 index 0000000000..c669707640 --- /dev/null +++ b/engine/crates/layerfs-bridge/tests/capacity_model.rs @@ -0,0 +1,74 @@ +//! The control-capacity read model (#540): the typed DTO round-trips the +//! daemon's answer, keeps a missing field legible, and drops a malformed +//! retained record instead of inventing one. +use layerfs_bridge::control::{AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel}; + +fn failure(slot: i64) -> RetainedFailureModel { + RetainedFailureModel { + slot, + class: "pool".into(), + generation: 4, + owner: "a1b2c3d4:812:layerfs-control-0".into(), + received_bytes: Some(24), + request: Some("control".into()), + reply_sent: false, + cause: "kernel request".into(), + fence_error: None, + diagnostic_error: Some("closed".into()), + age_ms: 12, + expires_in_ms: 3_600_000, + } +} + +#[test] +fn the_typed_model_round_trips_its_wire_record() { + let model = CapacityModel { + admission: AdmissionModel { + pool_vacant: 2, + pool_running: 6, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel { + limit: 64, + ttl_ms: 3_600_000, + count: 1, + dropped: 2, + recorded: 3, + failures: vec![failure(3)], + }, + }; + let wire = model.to_sdk(); + assert_eq!(CapacityModel::parse(&wire), model); + assert_eq!( + wire.get("retained").and_then(|v| v.get_int("count")), + Some(1) + ); +} + +#[test] +fn a_missing_or_malformed_record_reads_as_absent_not_invented() { + let value = layerfs_bridge::control::SdkValue::map([ + ("admission", layerfs_bridge::control::SdkValue::Null), + ( + "retained", + layerfs_bridge::control::SdkValue::map([ + ("limit", layerfs_bridge::control::SdkValue::Int(64)), + ( + "failures", + layerfs_bridge::control::SdkValue::List(vec![ + layerfs_bridge::control::SdkValue::map([( + "slot", + layerfs_bridge::control::SdkValue::Int(1), + )]), + ]), + ), + ]), + ), + ]); + let model = CapacityModel::parse(&value); + assert_eq!(model.admission, AdmissionModel::default()); + assert_eq!(model.retained.limit, 64); + assert_eq!(model.retained.count, 0); + assert!(model.retained.failures.is_empty()); +} diff --git a/engine/crates/layerfs-daemon/src/application/connection.rs b/engine/crates/layerfs-daemon/src/application/connection.rs index a77a58d9f9..57dc1e941a 100644 --- a/engine/crates/layerfs-daemon/src/application/connection.rs +++ b/engine/crates/layerfs-daemon/src/application/connection.rs @@ -1,40 +1,191 @@ //! Bounded real authenticated connection workers and retained original failures. -use super::{owner::Slot, Application, ApplicationError, ConnectionFailure}; +use super::{ + owner::{AdmissionClass, Finished, SlotClass, SlotTicket}, + retained::RetainedFailure, + Application, ApplicationError, ConnectionFailure, +}; use layerfs_bridge::{ - control::{Call, Reply, Request}, - duplex::{self as transport}, + control::{Call, DaemonPhase, Reply, Request}, + duplex::{ + self as transport, + admission::{self as admission_notice, AdmissionNotice, AdmissionReason}, + }, initial_record::InitialRecord, }; use std::{ - io::Write, + io::{Read, Write}, net::{TcpListener, TcpStream}, sync::Arc, thread, - time::Duration, + time::{Duration, Instant}, }; +/// Bounded conversion of an admission count to the notice's one byte. +fn saturating(count: usize) -> u8 { + u8::try_from(count).unwrap_or(u8::MAX) +} + +/// Writes one advisory pre-handshake capacity notice and drains what the +/// caller may still send, so it reads the notice rather than a reset. Every +/// socket operation is bounded; failure is ignored (the socket ends anyway). +fn write_capacity_notice(stream: &mut TcpStream, notice: &AdmissionNotice) { + let _ = stream.set_write_timeout(Some(Duration::from_millis(100))); + let _ = stream.set_read_timeout(Some(Duration::from_millis(50))); + if admission_notice::write(stream, notice).is_err() { + return; + } + let mut sink = [0u8; 512]; + let _ = stream.read(&mut sink); +} + +/// A synthetic failure for a worker that panicked: bounded, and recorded +/// through the same one-finish path as any other failure. +fn panic_failure(ticket: SlotTicket, payload: Box) -> ConnectionFailure { + let message = payload + .downcast_ref::<&str>() + .map(|text| (*text).to_owned()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "non-string panic payload".to_owned()); + let mut message = format!("connection worker panicked: {message}"); + if message.len() > 512 { + let mut end = 512; + while !message.is_char_boundary(end) { + end -= 1; + } + message.truncate(end); + message.push('…'); + } + ConnectionFailure { + slot: ticket.index, + received: None, + original: None, + cause: ApplicationError::Worker(message), + fence_error: None, + diagnostic_error: None, + } +} + +/// Owns one admission ticket. Exactly one `finish` happens: the explicit +/// call, or `Drop` during an unwind or a spawn failure. +struct SlotGuard { + application: Arc, + ticket: SlotTicket, + owner: String, + done: bool, +} + +impl SlotGuard { + fn finish(&mut self, result: Result<(), ConnectionFailure>) { + if self.done { + return; + } + self.done = true; + // Build the bounded record before taking the state lock. + let prebuilt = result.as_ref().err().map(|failure| { + RetainedFailure::describe( + failure, + self.ticket.class, + self.ticket.generation, + &self.owner, + ) + }); + // Poison recovery here only releases the slot and records; normal + // admission still reports `Poisoned` rather than trusting corrupt state. + let finished = { + let mut state = self + .application + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let finished = state + .slots + .finish(self.ticket, result, prebuilt, Instant::now()); + self.application.changed.notify_all(); + finished + }; + log_finished(finished); + } +} + +impl Drop for SlotGuard { + fn drop(&mut self) { + if !self.done { + self.finish(Err(ConnectionFailure { + slot: self.ticket.index, + received: None, + original: None, + cause: ApplicationError::Worker("dropped before completion".to_owned()), + fence_error: None, + diagnostic_error: None, + })); + } + } +} + +/// One bounded daemon-log line per terminal transition. +fn log_finished(finished: Finished) { + match finished { + Finished::Clean | Finished::Stale => {} + Finished::Released(failure) => { + let _ = writeln!(std::io::stderr().lock(), "{failure:?}"); + } + Finished::Retained(record) => { + let _ = writeln!(std::io::stderr().lock(), "{record}"); + } + } +} + impl Application { /// Accepts ordinary authenticated controls on one listener. The listener and /// each admitted worker own their exact sockets; no command execution occurs. - /// A failed connection keeps its bounded slot/custody until explicit transfer. + /// A failed connection releases its slot at once; a compact record of a + /// failure that used to retain custody stays in the bounded store. pub fn serve(self: &Arc, listener: TcpListener) -> Result<(), ApplicationError> { loop { - let (stream, _) = listener.accept().map_err(ApplicationError::Io)?; - let slot = { + let (mut stream, _) = listener.accept().map_err(ApplicationError::Io)?; + let ticket = { let mut state = self.state.lock().map_err(|_| ApplicationError::Poisoned)?; - let Some(slot) = state.slots.iter().position(|v| matches!(v, Slot::Vacant)) else { - // No authenticated operation has entered; admission ends this - // socket once, rather than allocating an unbounded worker. - drop(stream); - continue; - }; - state.slots[slot] = Slot::Running; - slot + let picked = state.slots.admit(AdmissionClass::ControlPool).or_else(|| { + // Only while the one-time installer may still run: an + // ordinary control never takes the reserved slot. + if matches!( + state.startup.phase, + DaemonPhase::InstallPending | DaemonPhase::Installing + ) { + state.slots.admit(AdmissionClass::Install) + } else { + None + } + }); + match picked { + Some(ticket) => ticket, + None => { + let counts = state.slots.counts(); + let retained = state.slots.visible(); + state.slots.reap(Instant::now()); + let reason = if state.startup.phase != DaemonPhase::ControlReady { + AdmissionReason::Starting + } else { + AdmissionReason::NoPoolSlot + }; + let notice = AdmissionNotice::new( + reason, + saturating(retained), + saturating(counts.pool_vacant), + saturating(counts.reserved_vacant), + ); + drop(state); + // No authenticated operation has entered; tell the + // caller why once, then end the socket (#540). + write_capacity_notice(&mut stream, ¬ice); + continue; + } + } }; let previous = self .workers .lock() - .map_err(|_| ApplicationError::Poisoned)?[slot] + .map_err(|_| ApplicationError::Poisoned)?[ticket.index] .take(); if let Some(worker) = previous { worker.join().map_err(|_| { @@ -44,60 +195,55 @@ impl Application { })?; } let application = self.clone(); + let owner = self.owner_label(ticket); + // The guard owns the ticket: if the worker body unwinds (or the + // thread never starts), it releases the slot and records exactly + // once. The normal path calls `finish` explicitly, so no path can + // leak a slot (#540). + let guard = SlotGuard { + application: self.clone(), + ticket, + owner, + done: false, + }; let worker = thread::Builder::new() - .name(format!("layerfs-control-{slot}")) + .name(format!("layerfs-control-{}", ticket.index)) .spawn(move || { - let mut result = application.connection(stream, slot); - if let Err(failure) = &mut result { - if failure.original.is_none() - && failure.received.is_none() - && matches!( - failure.cause, - ApplicationError::Channel(_) | ApplicationError::Io(_) - ) - { - // No decoded product attempt/custody exists at this boundary. - // Transfer this bounded original diagnostic to the daemon - // stderr owner before releasing capacity. Failed transfer - // keeps the original receipt and its distinct output error. - match writeln!(std::io::stderr().lock(), "{failure:?}") { - Ok(()) => result = Ok(()), - Err(error) => failure.diagnostic_error = Some(error), - } - } - } - { - // Poison recovery retains this exact result only; poisoned - // state never authorizes another product operation. - let mut state = application.state.lock().unwrap_or_else(|e| e.into_inner()); - state.slots[slot] = match result { - Ok(()) => Slot::Vacant, - Err(cause) => Slot::Retained(Box::new(cause)), - }; - application.changed.notify_all(); - } + let mut guard = guard; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + application.connection(stream, ticket) + })) + .unwrap_or_else(|payload| Err(panic_failure(ticket, payload))); + guard.finish(result); }); match worker { Ok(worker) => { self.workers .lock() - .map_err(|_| ApplicationError::Poisoned)?[slot] = Some(worker) + .map_err(|_| ApplicationError::Poisoned)?[ticket.index] = Some(worker) } - Err(cause) => { - let mut state = self.state.lock().map_err(|_| ApplicationError::Poisoned)?; - state.slots[slot] = Slot::Retained(Box::new(ConnectionFailure { - slot, - received: None, - original: None, - cause: ApplicationError::Io(cause), - fence_error: None, - diagnostic_error: None, - })); + Err(_cause) => { + // Spawning dropped the closure, so the guard's `Drop` + // already released the slot and recorded the failure. } } } } - fn connection(&self, stream: TcpStream, slot: usize) -> Result<(), ConnectionFailure> { + + /// A bounded owner label for one worker's retained record. + fn owner_label(&self, ticket: SlotTicket) -> String { + format!( + "{:02x}{:02x}{:02x}{:02x}:{}:layerfs-control-{}", + self.instance[0], + self.instance[1], + self.instance[2], + self.instance[3], + std::process::id(), + ticket.index + ) + } + fn connection(&self, stream: TcpStream, ticket: SlotTicket) -> Result<(), ConnectionFailure> { + let slot = ticket.index; let timeout = Some(Duration::from_millis(u64::from( self.setup.limits.handshake_ms, ))); @@ -202,7 +348,7 @@ impl Application { } let session_end = matches!(call.request, Request::EndSession) && outcome.is_ok(); - let reserved_refusal = slot == usize::from(self.setup.limits.connections) + let reserved_refusal = ticket.class == SlotClass::Reserved && matches!( &outcome, Err(crate::control::Failure::Rejected( @@ -372,6 +518,15 @@ impl Application { let service = state.startup.service.clone(); let phase = state.startup.phase; drop(state); + // Operator-visible retained failures live with the application, not + // with the Store control Service, so answer this one here. + if let Request::Sdk(call) = request { + if call.method == CONNECTION_FAILURES { + return self + .connection_failures() + .map(|value| Success::reply(Reply::Sdk(value))); + } + } match service { Some(service) => service.execute_control(request), None if phase == DaemonPhase::Retained => Err(Failure::Rejected( @@ -384,6 +539,54 @@ impl Application { )), } } + + /// The operator read model of control admission and retained failures. + fn connection_failures( + &self, + ) -> Result { + use crate::control::Failure; + use layerfs_bridge::control::{ + AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, + }; + let counts = self.connection_slots().map_err(|_| Failure::Poisoned)?; + let snapshot = self + .retained_connection_failures() + .map_err(|_| Failure::Poisoned)?; + let failures = snapshot + .failures + .iter() + .map(|failure| RetainedFailureModel { + slot: failure.slot as i64, + class: failure.class.to_owned(), + generation: failure.generation as i64, + owner: failure.owner.clone(), + received_bytes: failure.received_bytes.map(|bytes| bytes as i64), + request: failure.request.map(str::to_owned), + reply_sent: failure.reply_sent, + cause: failure.cause.clone(), + fence_error: failure.fence_error.clone(), + diagnostic_error: failure.diagnostic_error.clone(), + age_ms: failure.age.as_millis() as i64, + expires_in_ms: failure.expires_in.as_millis() as i64, + }); + Ok(CapacityModel { + admission: AdmissionModel { + pool_vacant: counts.pool_vacant as i64, + pool_running: counts.pool_running as i64, + reserved_vacant: counts.reserved_vacant as i64, + reserved_running: counts.reserved_running as i64, + }, + retained: RetainedModel { + limit: snapshot.limit as i64, + ttl_ms: snapshot.ttl.as_millis() as i64, + count: snapshot.failures.len() as i64, + dropped: snapshot.dropped as i64, + recorded: snapshot.recorded as i64, + failures: failures.collect(), + }, + } + .to_sdk()) + } } /// The Workspace of a Commit request, observed or not. @@ -402,3 +605,7 @@ fn reported_ready(outcome: &Result), /// Original install result retained after application publication synchronization failed. InstallPublication(Box>>), + /// A connection worker panicked or was dropped before it could finish. + Worker(String), /// Actual application state synchronization failed; ownership remains held. Poisoned, } @@ -62,6 +64,7 @@ impl fmt::Display for ApplicationError { Self::InstallPublication(_) => { f.write_str("original install result retained; application publication poisoned") } + Self::Worker(message) => write!(f, "connection worker: {message}"), Self::Poisoned => f.write_str("application state poisoned; original custody retained"), } } @@ -69,7 +72,8 @@ impl fmt::Display for ApplicationError { impl std::error::Error for ApplicationError {} /// Original failed connection admission/dispatch result and input custody. pub struct ConnectionFailure { - /// Exact admitted slot identity; never recycled while this custody remains. + /// Exact admitted slot identity at the failure; the slot is released once + /// this result is recorded, so a later connection may occupy it. pub slot: usize, /// Original malformed metadata bytes only; successful requests own decoded fields. pub received: Option>, diff --git a/engine/crates/layerfs-daemon/src/application/mod.rs b/engine/crates/layerfs-daemon/src/application/mod.rs index 76671b12a8..65427ebb16 100644 --- a/engine/crates/layerfs-daemon/src/application/mod.rs +++ b/engine/crates/layerfs-daemon/src/application/mod.rs @@ -7,9 +7,11 @@ mod diagnostics; mod failure; mod filesystem; mod owner; +mod retained; mod serve; pub use cli::main; pub use diagnostics::resources::{ResourceDiagnosticError, ResourceFailure, ResourceObservation}; pub(crate) use diagnostics::{commit_parts, namespace_values, storage_values}; pub use failure::{ApplicationError, ConnectionFailure}; pub use owner::Application; +pub use retained::{RetainedFailure, RetainedSnapshot}; diff --git a/engine/crates/layerfs-daemon/src/application/owner.rs b/engine/crates/layerfs-daemon/src/application/owner.rs index e0c539a818..a8181601e6 100644 --- a/engine/crates/layerfs-daemon/src/application/owner.rs +++ b/engine/crates/layerfs-daemon/src/application/owner.rs @@ -1,4 +1,5 @@ //! One startup lifecycle owner, Store transfer and bounded connection custody. +use super::retained::{RetainedFailure, RetainedFailures, RetainedSnapshot}; use super::{ApplicationError, ConnectionFailure}; use crate::{ bootstrap::{open_store_observed, OpenedStore}, @@ -19,12 +20,222 @@ use layerfs_storage::ReservationBlocks; use std::{ sync::{Arc, Condvar, Mutex}, thread::JoinHandle, + time::Instant, }; -pub(super) enum Slot { +/// Slots reserved for the installer (and, later, a presence lease); ordinary +/// controls never consume them. +pub(super) const RESERVED_SLOTS: usize = 1; + +/// The fixed class of one slot index. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SlotClass { + /// Ordinary authenticated control connections. + Pool, + /// Installer today, a presence lease later. + Reserved, +} + +impl SlotClass { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Pool => "pool", + Self::Reserved => "reserved", + } + } +} + +/// What a caller is admitted for; maps to the slot class it may use. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum AdmissionClass { + /// An ordinary control connection. + ControlPool, + /// The one-time Store installer. + Install, + /// A future presence lease. + #[allow(dead_code)] + Lease, +} + +impl AdmissionClass { + fn slot_class(self) -> SlotClass { + match self { + Self::ControlPool => SlotClass::Pool, + Self::Install | Self::Lease => SlotClass::Reserved, + } + } +} + +/// One admission slot: free, or running one worker generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SlotState { Vacant, - Running, - Retained(Box), + Running { generation: u64, class: SlotClass }, +} + +/// Proof that one admission owns a slot and generation. Every path ends in +/// exactly one `Slots::finish` with this ticket; a stale ticket changes +/// nothing, so a duplicate or late finish cannot free a newer worker. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct SlotTicket { + pub(super) index: usize, + pub(super) generation: u64, + pub(super) class: SlotClass, +} + +/// What finishing one connection did with its failure, if any. +pub(super) enum Finished { + /// The worker ended without a failure. + Clean, + /// The failure was written to the daemon log and needs no record. + Released(Box), + /// The failure was recorded in the bounded store; its slot is already free. + Retained(Box), + /// A stale or duplicate finish: nothing changed. + Stale, +} + +/// Admission counts by class. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SlotCounts { + pub pool_vacant: usize, + pub pool_running: usize, + pub reserved_vacant: usize, + pub reserved_running: usize, +} + +/// The fixed control-slot table and the bounded retained-failure store beside it. +/// +/// A retained failure never owns a slot: `finish` always leaves it `Vacant` +/// and only records the failure. Admission capacity therefore cannot leak +/// (#540). The layout is `connections` `Pool` entries followed by +/// [`RESERVED_SLOTS`] `Reserved` entries; a class only ever admits within its +/// own range. +pub(super) struct Slots { + slots: Vec, + pool: usize, + next_generation: u64, + retained: RetainedFailures, +} + +impl Slots { + pub(super) fn new(count: usize) -> Self { + let pool = count.saturating_sub(RESERVED_SLOTS); + let mut slots = Vec::with_capacity(pool + RESERVED_SLOTS); + slots.resize(pool, SlotState::Vacant); + slots.resize(pool + RESERVED_SLOTS, SlotState::Vacant); + Self { + slots, + pool, + next_generation: 0, + retained: RetainedFailures::default(), + } + } + + fn class_of(&self, index: usize) -> SlotClass { + if index < self.pool { + SlotClass::Pool + } else { + SlotClass::Reserved + } + } + + /// Claims one vacant slot of `class`, or `None` when that class is full. + /// A pool admission never takes a reserved slot and vice versa. + pub(super) fn admit(&mut self, class: AdmissionClass) -> Option { + let wanted = class.slot_class(); + let index = self.slots.iter().enumerate().find_map(|(index, value)| { + (self.class_of(index) == wanted && matches!(value, SlotState::Vacant)).then_some(index) + })?; + self.generation_ticket(index, wanted) + } + + fn generation_ticket(&mut self, index: usize, class: SlotClass) -> Option { + self.next_generation = self.next_generation.saturating_add(1); + let generation = self.next_generation; + *self.slots.get_mut(index)? = SlotState::Running { generation, class }; + Some(SlotTicket { + index, + generation, + class, + }) + } + + /// Ends one worker: release its slot and, for a retained failure, record + /// the already-built record. A ticket whose generation or class no longer + /// matches changes nothing. + pub(super) fn finish( + &mut self, + ticket: SlotTicket, + result: Result<(), ConnectionFailure>, + prebuilt: Option, + now: Instant, + ) -> Finished { + let running = matches!( + self.slots.get(ticket.index), + Some(SlotState::Running { generation, class }) + if *generation == ticket.generation && *class == ticket.class + ); + if !running { + return Finished::Stale; + } + self.slots[ticket.index] = SlotState::Vacant; + match result { + Ok(()) => Finished::Clean, + Err(failure) if releases_after_logging(&failure) => { + Finished::Released(Box::new(failure)) + } + Err(_) => { + let record = prebuilt.expect("a retained failure is described before the lock"); + let record = self.retained.record(record, now); + Finished::Retained(Box::new(record)) + } + } + } + + pub(super) fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { + self.retained.snapshot(now) + } + + /// Admission counts for each class. + pub(super) fn counts(&self) -> SlotCounts { + let mut counts = SlotCounts { + pool_vacant: 0, + pool_running: 0, + reserved_vacant: 0, + reserved_running: 0, + }; + for (index, value) in self.slots.iter().enumerate() { + match (self.class_of(index), value) { + (SlotClass::Pool, SlotState::Vacant) => counts.pool_vacant += 1, + (SlotClass::Pool, SlotState::Running { .. }) => counts.pool_running += 1, + (SlotClass::Reserved, SlotState::Vacant) => counts.reserved_vacant += 1, + (SlotClass::Reserved, SlotState::Running { .. }) => counts.reserved_running += 1, + } + } + counts + } + + /// Visible retained records, for the admission notice. + pub(super) fn visible(&mut self) -> usize { + self.retained.len() + } + + /// Drops expired records; the accept loop reaps when it refuses admission. + pub(super) fn reap(&mut self, now: Instant) { + self.retained.reap(now); + } +} + +/// A failure with no decoded product attempt or custody: the existing daemon +/// log line is the whole record, and the slot is released. +fn releases_after_logging(failure: &ConnectionFailure) -> bool { + failure.original.is_none() + && failure.received.is_none() + && matches!( + failure.cause, + ApplicationError::Channel(_) | ApplicationError::Io(_) + ) } pub(super) struct Startup { pub phase: DaemonPhase, @@ -35,7 +246,7 @@ pub(super) struct Startup { } pub(super) struct State { pub startup: Startup, - pub slots: Vec, + pub slots: Slots, } /// One real daemon application; Store/Overlay are initialized once and reused. pub struct Application { @@ -201,7 +412,7 @@ impl Application { existing, install_failure: None, }, - slots: (0..count).map(|_| Slot::Vacant).collect(), + slots: Slots::new(count), }), changed: Condvar::new(), workers: Mutex::new((0..count).map(|_| None).collect()), @@ -271,27 +482,16 @@ impl Application { } Ok(self.observation(&state)) } - /// Transfers original failed connection custody out only at an explicit operator request. - /// Taking a receipt does not resolve its unknowns, replay it or remove artifacts. - pub fn take_connection_failure( - &self, - slot: usize, - ) -> Result>, ApplicationError> { + /// The bounded retained-failure store, pruned at this observation. + pub fn retained_connection_failures(&self) -> Result { let mut state = self.state.lock().map_err(|_| ApplicationError::Poisoned)?; - let current = state.slots.get_mut(slot).ok_or_else(|| { - ApplicationError::Io(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "connection slot", - )) - })?; - if !matches!(current, Slot::Retained(_)) { - return Ok(None); - } - let Slot::Retained(failure) = std::mem::replace(current, Slot::Vacant) else { - unreachable!() - }; - self.changed.notify_all(); - Ok(Some(failure)) + Ok(state.slots.snapshot(Instant::now())) + } + + /// Vacant and in-use control slots, by class. + pub fn connection_slots(&self) -> Result { + let state = self.state.lock().map_err(|_| ApplicationError::Poisoned)?; + Ok(state.slots.counts()) } } diff --git a/engine/crates/layerfs-daemon/src/application/retained.rs b/engine/crates/layerfs-daemon/src/application/retained.rs new file mode 100644 index 0000000000..f7697d4030 --- /dev/null +++ b/engine/crates/layerfs-daemon/src/application/retained.rs @@ -0,0 +1,281 @@ +//! Bounded operator-visible records of connection failures that used to retain a slot. +//! +//! A failed connection worker used to keep its admission slot `Retained` until +//! an explicit transfer that no product path ever made. That cost one of the +//! daemon's fixed control slots per failure until every new connection was +//! refused (#540). The slot is now released at once; what survives here is a +//! compact, bounded, owned and timed record with its exact deciding cause. +use super::{ApplicationError, ConnectionFailure}; +use std::{ + collections::VecDeque, + fmt, + time::{Duration, Instant}, +}; + +/// Most retained records kept at once. The oldest is dropped first. +pub(super) const LIMIT: usize = 64; +/// How long a retained record stays operator-visible. +pub(super) const TTL: Duration = Duration::from_secs(60 * 60); +/// Most bytes a retained textual cause keeps. +const CAUSE_BYTES: usize = 512; + +/// One connection failure that used to retain its slot, as an operator sees it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetainedFailure { + /// Slot the failed worker occupied; it is already free for a new connection. + pub slot: usize, + /// `pool` or `reserved`: the class of the slot that was released. + pub class: &'static str, + /// Admission generation of the released worker; a stale finish never frees + /// a newer one. + pub generation: u64, + /// The worker that failed: daemon instance, pid and thread. + pub owner: String, + /// Bytes of a request that failed decoding, when one was received. + pub received_bytes: Option, + /// `control` or `install` when a request was decoded, else `None`. + pub request: Option<&'static str>, + /// Whether the original reply was sent before the failure. + pub reply_sent: bool, + /// Bounded textual deciding cause. + pub cause: String, + /// Original socket fence failure, when one was observed. + pub fence_error: Option, + /// Distinct failed diagnostic transfer, when one happened. + pub diagnostic_error: Option, + /// Age at the observation that returned this record. + pub age: Duration, + /// Remaining lifetime at the observation that returned this record. + pub expires_in: Duration, + /// Visible records at the moment this one was recorded. + pub retained: usize, + /// Configured record limit at the moment this one was recorded. + pub limit: usize, +} + +impl RetainedFailure { + /// The bounded record for `failure`, built outside the state lock. + pub(super) fn describe( + failure: &ConnectionFailure, + class: super::owner::SlotClass, + generation: u64, + owner: &str, + ) -> Self { + Self { + slot: failure.slot, + class: class.as_str(), + generation, + owner: owner.to_owned(), + received_bytes: failure.received.as_ref().map(Vec::len), + request: request_kind(failure), + reply_sent: reply_sent(failure), + cause: bounded(&failure.cause.to_string()), + fence_error: failure + .fence_error + .as_ref() + .map(|error| bounded(&error.to_string())), + diagnostic_error: failure + .diagnostic_error + .as_ref() + .map(|error| bounded(&error.to_string())), + age: Duration::ZERO, + expires_in: TTL, + retained: 0, + limit: LIMIT, + } + } +} + +impl fmt::Display for RetainedFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ConnectionFailure") + .field("slot", &self.slot) + .field("class", &self.class) + .field("generation", &self.generation) + .field("received_bytes", &self.received_bytes) + .field("original_kind", &self.request) + .field("reply", &if self.reply_sent { "sent" } else { "not_sent" }) + .field("cause", &self.cause) + .field("fence_error", &self.fence_error) + .field("diagnostic_error", &self.diagnostic_error) + .field("retained", &format!("{}/{}", self.retained, self.limit)) + .field("owner", &self.owner) + .field("expires_in_ms", &self.expires_in.as_millis()) + .finish() + } +} + +/// A bounded observation of the retained-failure store. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RetainedSnapshot { + /// Records, oldest first. + pub failures: Vec, + /// Records evicted by the limit or the TTL since startup. + pub dropped: u64, + /// Total records ever recorded since startup. + pub recorded: u64, + /// Configured record limit. + pub limit: usize, + /// Configured record TTL. + pub ttl: Duration, +} + +struct Entry { + record: RetainedFailure, + at: Instant, + deadline: Instant, +} + +/// A fixed-capacity, TTL-pruned ring of retained connection failures. Its +/// capacity is reserved when it is built, so recording under the state lock +/// never allocates. +pub(super) struct RetainedFailures { + entries: VecDeque, + dropped: u64, + recorded: u64, + limit: usize, + ttl: Duration, +} + +impl Default for RetainedFailures { + fn default() -> Self { + Self::new(LIMIT, TTL) + } +} + +impl RetainedFailures { + pub(super) fn new(limit: usize, ttl: Duration) -> Self { + let limit = limit.max(1); + Self { + entries: VecDeque::with_capacity(limit), + dropped: 0, + recorded: 0, + limit, + ttl, + } + } + + /// Records one already-described failure, prunes expired records and + /// evicts the oldest beyond the limit. Returns the exact record that now + /// is visible, with its store position filled in. + pub(super) fn record(&mut self, mut record: RetainedFailure, now: Instant) -> RetainedFailure { + self.prune(now); + while self.entries.len() >= self.limit { + self.entries.pop_front(); + self.dropped = self.dropped.saturating_add(1); + } + record.retained = self.entries.len() + 1; + record.limit = self.limit; + record.expires_in = self.ttl; + self.entries.push_back(Entry { + record: record.clone(), + at: now, + deadline: now + self.ttl, + }); + self.recorded = self.recorded.saturating_add(1); + record + } + + /// Drops records whose deadline passed; a stale record is never returned. + pub(super) fn prune(&mut self, now: Instant) { + while let Some(front) = self.entries.front() { + if now < front.deadline { + break; + } + self.entries.pop_front(); + self.dropped = self.dropped.saturating_add(1); + } + } + + /// The lazy reaper: the accept loop calls it when it refuses admission. + pub(super) fn reap(&mut self, now: Instant) { + self.prune(now); + } + + /// A copy of the visible records, oldest first, with their ages. + pub(super) fn snapshot(&mut self, now: Instant) -> RetainedSnapshot { + self.prune(now); + RetainedSnapshot { + failures: self + .entries + .iter() + .map(|entry| { + let mut record = entry.record.clone(); + record.age = now.saturating_duration_since(entry.at); + record.expires_in = self.ttl.saturating_sub(record.age); + record + }) + .collect(), + dropped: self.dropped, + recorded: self.recorded, + limit: self.limit, + ttl: self.ttl, + } + } + + pub(super) fn len(&self) -> usize { + self.entries.len() + } +} + +/// Whether a request was decoded, from the direct field or the cause that +/// owns the original receipt. +fn request_kind(failure: &ConnectionFailure) -> Option<&'static str> { + if let Some(original) = &failure.original { + return Some(match original { + layerfs_bridge::initial_record::InitialRecord::Control(_) => "control", + layerfs_bridge::initial_record::InitialRecord::Install(_) => "install", + }); + } + match &failure.cause { + ApplicationError::Control(serve) => serve + .original + .as_ref() + .map(|served| kind(&served.call.request)), + ApplicationError::RetainedControl(served) => Some(kind(&served.call.request)), + ApplicationError::SessionEnd { served, .. } => Some(kind(&served.call.request)), + _ => None, + } +} + +fn kind(request: &layerfs_bridge::control::Request) -> &'static str { + use layerfs_bridge::control::Request; + match request { + Request::Mount { .. } => "mount", + Request::MountNamed { .. } => "mount", + Request::Commit(_) => "commit", + Request::Status(_) => "status", + Request::Unmount(_) => "unmount", + Request::ForceUnmount { .. } => "force-unmount", + Request::Attach(_) => "attach", + Request::Locate(_) => "locate", + Request::Cleanup(_) => "cleanup", + Request::Fork(_) => "fork", + Request::Branches { .. } => "branches", + Request::History(_) => "history", + Request::Sdk(_) => "sdk", + Request::Hello(_) => "hello", + Request::EndSession => "end-session", + Request::Observed { .. } => "observed", + } +} + +/// Whether the original reply reached the caller. A send/fence failure did +/// not; a retained receipt or a session-end observation follows a sent reply. +fn reply_sent(failure: &ConnectionFailure) -> bool { + matches!( + failure.cause, + ApplicationError::RetainedControl(_) | ApplicationError::SessionEnd { .. } + ) +} + +fn bounded(text: &str) -> String { + if text.len() <= CAUSE_BYTES { + return text.to_owned(); + } + let mut end = CAUSE_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &text[..end]) +}