diff --git a/api/crates/layerfs-api/src/sandbox/get.rs b/api/crates/layerfs-api/src/sandbox/get.rs index f1e05b8467..2839ca84b8 100644 --- a/api/crates/layerfs-api/src/sandbox/get.rs +++ b/api/crates/layerfs-api/src/sandbox/get.rs @@ -5,7 +5,7 @@ use crate::calls::Context; use layerfs_bridge::control::SdkValue; use layerfs_home::home::with_field; use layerfs_home::outcome::{Outcome, View}; -use layerfs_sandbox::family::{get, GetRequest, Page}; +use layerfs_sandbox::family::{get, GetRequest, Got, Page}; /// Its table entry. pub(crate) const CALL: Call = call( @@ -17,9 +17,11 @@ pub(crate) const CALL: Call = call( run, ); -/// Runs `sandbox.get`: its record with its derived `state`, one page of -/// its `workspaces` (each with its `path`, `name` and `ws`), why its -/// machine did not answer (`down_reason`) and `next_page_pos`. +/// Runs `sandbox.get`: its record with its derived `state`, one page of its +/// `workspaces` (each with its `path`, `name` and `ws`), why its machine did +/// not answer (`down_reason`), the daemon's `daemon_capacity` read model when +/// the Sandbox answered it (#540), and `next_page_pos`. The library reads it +/// best effort and bounded, so this layer only shows what it got. fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { let request = GetRequest { name: args.get_text("name").unwrap_or("").to_owned(), @@ -29,18 +31,28 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { }, }; let (home, events) = context.parts(); - let got = get(home, events, &request)?; - let workspaces = got.workspaces.into_iter().map(|listed| { + let Got { + record, + state, + workspaces, + down_reason, + next, + daemon_capacity, + } = get(home, events, &request)?; + let workspaces = workspaces.into_iter().map(|listed| { let item = with_field(listed.item, "path", SdkValue::Text(listed.path)); let item = with_field(item, "name", SdkValue::opt_text(listed.name)); with_field(item, "ws", SdkValue::Text(listed.ws)) }); - let shown = got.record.to_sdk(View::Full); - let shown = with_field(shown, "state", SdkValue::text(got.state)); + let shown = record.to_sdk(View::Full); + let shown = with_field(shown, "state", SdkValue::text(state)); let mut shown = with_field(shown, "workspaces", SdkValue::List(workspaces.collect())); - if let Some(why) = got.down_reason { + if let Some(why) = down_reason { shown = with_field(shown, "down_reason", SdkValue::Text(why)); } - let next = super::list::next_page_pos(got.next); + if let Some(model) = daemon_capacity { + shown = with_field(shown, "daemon_capacity", model.to_sdk()); + } + let next = super::list::next_page_pos(next); Ok(with_field(shown, "next_page_pos", next)) } diff --git a/api/crates/layerfs-api/tests/sandbox_capacity.rs b/api/crates/layerfs-api/tests/sandbox_capacity.rs new file mode 100644 index 0000000000..feb60971a2 --- /dev/null +++ b/api/crates/layerfs-api/tests/sandbox_capacity.rs @@ -0,0 +1,365 @@ +//! `sandbox.get`'s `daemon_capacity` (#540, #543): the daemon's +//! `backend.daemon.connection_failures` read model, shown best-effort and +//! bounded on the connection the Workspace list lent from this process's +//! pool. Nothing runs: a stand-in daemon answers the Sandbox's Workspace list +//! and its capacity model on the same connection, a second one answers the +//! list and then says nothing, and a Sandbox whose machine or forward is +//! already down is not dialed at all. +#![cfg(unix)] +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, DaemonPhase, DaemonStatus, Reply, Request, RetainedModel, + SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; +use std::net::{SocketAddr, TcpListener}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("lfs-scap-{tag}-{}-{sequence}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +/// What the stand-in daemon answers `sandbox.list_workspaces` with: no +/// Workspaces, like a Sandbox nothing has mounted in. +fn no_workspaces() -> SdkValue { + SdkValue::map([ + ("kind", t("workspaces")), + ("version", SdkValue::Int(1)), + ("workspaces", SdkValue::List(vec![])), + ]) +} + +/// A stand-in daemon at its own loopback port, and how many connections it +/// served: it answers this Sandbox's Workspace list and its capacity model +/// with `model` on the connection it was given. With `stall` it answers the +/// Workspace list, reads the capacity query and never answers it, which is +/// what a daemon stuck behind its own slots looks like. +fn daemon(model: SdkValue, stall: bool) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let model = Arc::new(model); + let served = Arc::new(AtomicUsize::new(0)); + let counted = served.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + counted.fetch_add(1, Ordering::SeqCst); + let model = model.clone(); + // one worker per connection: the pool keeps one and reuses it + std::thread::spawn(move || serve(stream, &model, stall)); + } + }); + (endpoint, served) +} + +/// One stand-in connection: its Hello, then each call's answer until the +/// caller closes it. +fn serve(stream: std::net::TcpStream, model: &SdkValue, stall: bool) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + let Ok(mut channel) = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) else { + return; + }; + while let Ok(bytes) = channel.receive.receive() { + let Ok(call) = Call::decode(bytes) else { break }; + let id = call.id; + let reply = match call.request { + Request::Hello(_) => Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + Request::Sdk(sdk) if sdk.method == "sandbox.list_workspaces" => { + Reply::Sdk(no_workspaces()) + } + Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + if stall { + // read but never answered: the read model's own fence, not + // a closed socket, ends its wait + std::thread::sleep(Duration::from_secs(6)); + break; + } + Reply::Sdk(model.clone()) + } + _ => continue, + }; + if channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .is_err() + { + break; + } + } +} + +/// A record from `fields`, in order. +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +/// The home's machine, Store `shop` and project `api`; the machine's `worker` +/// record only while it runs (its absence is a stopped machine, #535 D11). +fn home_records(home: &Path, worker: bool) { + write_record( + &paths::machine_record(home, "native"), + &map(&[("name", t("native")), ("platform", t("native"))]), + ) + .unwrap(); + if worker { + // the forward's machine answers: a forward with no recorded process is + // not checked against one + write_record( + &paths::worker_record(home, "native"), + &map(&[("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + } + write_record( + &paths::store_record(home, "shop"), + &map(&[ + ("name", t("shop")), + ("kind", t("sqlite")), + ("binding", SdkValue::Bytes(vec![7; 16])), + ("cursor", SdkValue::Bytes(vec![8; 32])), + ("profile", t("disposable")), + ("machine", t("native")), + ]), + ) + .unwrap(); + write_record( + &paths::project_record(home, "shop", "api"), + &map(&[("name", t("api"))]), + ) + .unwrap(); +} + +/// Sandbox `s1` on `native`, recorded with its `id` (`None`: gone, its create +/// never recorded one) whose daemon this host reaches through the forward to +/// `endpoint`, carried by forward process `pid` (`None`: a port nothing +/// carries). +fn sandbox_record(home: &Path, endpoint: SocketAddr, id: Option<&str>, pid: Option) { + let mut forward = vec![ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]; + if let Some(pid) = pid { + forward.push(("pid", SdkValue::Int(pid))); + } + let mut fields = vec![ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ]; + if let Some(id) = id { + fields.push(("id", t(id))); + } + fields.push(( + "daemon", + map(&[("name", t("lfs-x-s1")), ("forward", map(&forward))]), + )); + write_record(&paths::sandbox_record(home, "s1"), &map(&fields)).unwrap(); + write_record( + &paths::sandbox_control(home, "s1"), + &map(&[ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]), + ) + .unwrap(); +} + +/// A model with no vacant pool slot and one retained failure. +fn busy() -> SdkValue { + let model = layerfs_bridge::control::CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel { + limit: 64, + ttl_ms: 3_600_000, + count: 1, + dropped: 0, + recorded: 1, + failures: vec![layerfs_bridge::control::RetainedFailureModel { + slot: 0, + class: "pool".into(), + generation: 6, + owner: "daemon instance, pid 4, thread layerfs-control-0".into(), + received_bytes: None, + request: Some("unmount".into()), + reply_sent: false, + cause: "control: the kernel control did not answer".into(), + fence_error: None, + diagnostic_error: None, + age_ms: 12, + expires_in_ms: 3_599_988, + }], + }, + }; + model.to_sdk() +} + +#[test] +fn a_running_sandbox_s_get_shows_its_daemon_capacity() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("shown"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, Some("s-1"), None); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + let capacity = got + .get("daemon_capacity") + .unwrap_or_else(|| panic!("{got:?}")); + let admission = capacity.get("admission").unwrap(); + assert_eq!(admission.get_int("pool_vacant"), Some(0), "{capacity:?}"); + assert_eq!(admission.get_int("pool_running"), Some(8), "{capacity:?}"); + assert_eq!( + admission.get_int("reserved_vacant"), + Some(1), + "{capacity:?}" + ); + let retained = capacity.get("retained").unwrap(); + assert_eq!(retained.get_int("count"), Some(1), "{capacity:?}"); + assert_eq!(retained.get_int("limit"), Some(64), "{capacity:?}"); + let Some(SdkValue::List(failures)) = retained.get("failures") else { + panic!("{capacity:?}") + }; + assert_eq!(failures.len(), 1, "{capacity:?}"); + assert_eq!( + failures[0].get_text("cause"), + Some("control: the kernel control did not answer") + ); + assert_eq!(failures[0].get_int("generation"), Some(6)); + assert_eq!(failures[0].get_text("request"), Some("unmount")); + // the Workspace list and the capacity read share the one pooled + // connection (#543): a get takes no second control slot of its own + assert_eq!(served.load(Ordering::SeqCst), 1, "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_daemon_that_does_not_answer_leaves_daemon_capacity_out() { + let (endpoint, _served) = daemon(busy(), true); + let home = tempdir("stalled"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, Some("s-1"), None); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + // the read model's own bound ends the wait; the get still answers + assert!( + elapsed < Duration::from_secs(4), + "sandbox.get stalled for {elapsed:?}" + ); + assert!( + elapsed >= Duration::from_secs(1), + "the bounded query did not wait: {elapsed:?}" + ); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_stopped_machine_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("stopped"); + home::open(&home).unwrap(); + // the machine records no worker: nothing starts it (#535 D11) + home_records(&home, false); + sandbox_record(&home, endpoint, Some("s-1"), None); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + assert!(got.get("down_reason").is_some(), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + // a down machine is answered from its records: no port is dialed and + // nothing waits for a daemon that is not there + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn an_ended_forward_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("forward-ended"); + home::open(&home).unwrap(); + home_records(&home, true); + // a recorded forward process that is not alive: its port may be reused by + // another process, so it is never dialed + sandbox_record(&home, endpoint, Some("s-1"), Some(3_999_999)); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + assert!(got.get("down_reason").is_some(), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_sandbox_with_no_id_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("gone"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, None, None); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(got.get_text("state"), Some("gone"), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-control/src/doctor/capacity.rs b/control/crates/layerfs-control/src/doctor/capacity.rs index 6f35cf23c1..f69dda51a1 100644 --- a/control/crates/layerfs-control/src/doctor/capacity.rs +++ b/control/crates/layerfs-control/src/doctor/capacity.rs @@ -5,29 +5,15 @@ //! naming its endpoint and the operation, never an error, because the //! doctor changes nothing and a stopped daemon is a normal state. //! Two more warnings come from the model itself: retained connection -//! failures, and a pool with no vacant control slot. +//! failures, and a pool with no vacant control slot. The one bounded query +//! itself is `layerfs_sandbox::daemon::capacity`, shared with the +//! `daemon_capacity` field `sandbox.get` shows. use super::report::Findings; use super::scan::Scanned; -use layerfs_bridge::control::{CapacityModel, SdkRefusal, SdkValue}; -use layerfs_home::home::paths; -use layerfs_home::outcome::{failed, refused}; -use layerfs_home::secrets; -use layerfs_sandbox::daemon::operator; -use layerfs_sandbox::direct; -use layerfs_sandbox::record::SandboxRecord; +use layerfs_bridge::control::{CapacityModel, SdkRefusal}; +use layerfs_sandbox::daemon::capacity::{endpoint, read, OPERATION, WAIT}; use std::net::SocketAddr; use std::path::Path; -use std::sync::mpsc; -use std::time::Duration; - -/// The daemon's operator read model of control admission and retained -/// connection failures. -const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; - -/// How long one daemon query may take: connecting, its Hello and its one -/// exchange. Short on purpose: the doctor must not stall behind a daemon -/// that is unreachable, as an ordinary 60 s admission wait would. -const WAIT: Duration = Duration::from_secs(2); /// Queries the daemon of every running Sandbox whose endpoint is known. pub(super) fn check(home: &Path, scanned: &Scanned, findings: &mut Findings) { @@ -43,73 +29,13 @@ pub(super) fn check(home: &Path, scanned: &Scanned, findings: &mut Findings) { // listens, and the doctor starts nothing to find out continue; }; - match probe_bounded(home, sandbox, endpoint) { - Ok(value) => capacity(findings, name, endpoint, &CapacityModel::parse(&value)), + match read(home, sandbox, endpoint) { + Ok(model) => capacity(findings, name, endpoint, &model), Err(refusal) => unanswered(findings, name, endpoint, &refusal), } } } -/// The address of a running Sandbox's daemon when its record names one to -/// this host: a forwarded machine's forward, else the endpoint the start -/// recorded in `control.json`. Nothing is observed or started. -fn endpoint(home: &Path, sandbox: &SandboxRecord) -> Option { - if let Some(forward) = sandbox.daemon().and_then(direct::forward_of) { - return Some(forward); - } - let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name())) - .ok() - .flatten()?; - keys.get_text("endpoint")?.parse().ok() -} - -/// [`probe`] under a hard wall: a daemon that accepts a connection and then -/// says nothing must not hold the doctor past [`WAIT`]. -fn probe_bounded( - home: &Path, - sandbox: &SandboxRecord, - endpoint: SocketAddr, -) -> Result { - let home = home.to_owned(); - let sandbox = sandbox.clone(); - let (sent, received) = mpsc::channel(); - std::thread::spawn(move || { - let _ = sent.send(probe(&home, &sandbox, endpoint)); - }); - match received.recv_timeout(WAIT) { - Ok(result) => result, - Err(_) => Err(refused( - "Unavailable", - format!("the daemon did not answer within {}s", WAIT.as_secs()), - )), - } -} - -/// One daemon query: dial its recorded endpoint with its keys for at most -/// [`WAIT`], then read the capacity model once. -fn probe( - home: &Path, - sandbox: &SandboxRecord, - endpoint: SocketAddr, -) -> Result { - let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name()))? - .ok_or_else(|| refused("NoSuchSandbox", "the Sandbox has no daemon keys"))?; - let dial = layerfs_sandbox::daemon::pool::Dial { - endpoint, - secret: secrets::key(&keys, "secret")?, - peer: secrets::key(&keys, "peer")?, - instance: secrets::key(&keys, "instance")?, - }; - let mut control = dial.open_within(WAIT)?; - operator::call(&mut control, OPERATION, SdkValue::Null).map_err(|error| match error { - layerfs_sandbox::SdkError::Refused(refusal) => refusal, - layerfs_sandbox::SdkError::Operation(failure) => { - layerfs_sandbox::daemon::values::refusal(&failure).unwrap_or_else(|| failed(failure)) - } - layerfs_sandbox::SdkError::Mount(failure) => failed(failure), - }) -} - /// A daemon that did not answer: a warning naming its endpoint and the /// operation. An exhausted pool is reported from the refusal's admission /// notice, which names its resource and vacant slots; the daemon never diff --git a/control/crates/layerfs-sandbox/src/daemon/capacity.rs b/control/crates/layerfs-sandbox/src/daemon/capacity.rs new file mode 100644 index 0000000000..31b41b1556 --- /dev/null +++ b/control/crates/layerfs-sandbox/src/daemon/capacity.rs @@ -0,0 +1,141 @@ +//! A running Sandbox daemon's control-capacity read model (#540): its +//! `backend.daemon.connection_failures` answer, read once, best-effort and +//! bounded. The doctor explains it and `sandbox.get` shows it as +//! `daemon_capacity`; both ask through here so one place decides where the +//! daemon is, how long an answer may take, and what an unanswered query is. +//! +//! Both reads end at [`WAIT`], and their connection is fenced at that +//! deadline, so a daemon that trickles its handshake or its one reply keeps +//! neither a thread, a connection nor a control slot past it. +use crate::daemon::fence::Fence; +use crate::daemon::pool::{Controls, Dial}; +use crate::daemon::{operator, trace}; +use crate::direct; +use crate::internal::Control; +use crate::record::SandboxRecord; +use layerfs_bridge::control::{CapacityModel, Request, SdkRefusal, SdkValue}; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused}; +use layerfs_home::secrets; +use std::net::SocketAddr; +use std::path::Path; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// The daemon's operator read model of control admission and retained +/// connection failures. +pub const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; + +/// How long one daemon query may take: connecting, its Hello and its one +/// exchange. Short on purpose: a reader must not stall behind a daemon that +/// is unreachable, as an ordinary 60 s admission wait would. +pub const WAIT: Duration = Duration::from_secs(2); + +/// The model [`read`] gives: the daemon's answer, parsed once. +pub fn read( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let home = home.to_owned(); + let sandbox = sandbox.clone(); + bounded(WAIT, move || query(&home, &sandbox, endpoint)) + .map(|answer| CapacityModel::parse(&answer)) +} + +/// The daemon's answer to [`OPERATION`] on a connection the Sandbox's pool +/// lends (`sandbox.get`, #543): at most [`WAIT`] for admission and for the +/// exchange, which is fenced at the same deadline, so this read takes no +/// second connection and no second slot of its own. A window the admission +/// wait used up refuses the read before it is sent, and a connection the +/// fence ends stays quarantined: the pool closes it instead of lending it +/// again. Nothing is sent again. +pub fn lent(controls: &Controls) -> Result { + let deadline = Instant::now() + WAIT; + let mut control = trace::control_within(controls, WAIT)?; + if deadline.saturating_duration_since(Instant::now()).is_zero() { + // the admission wait used the whole window. The connection is healthy + // and returns to the pool; arming a fence for a deadline that already + // passed would shut it down for nothing, so nothing is sent. + return Err(refused( + "Unavailable", + format!( + "no control connection came free within {}s, so the capacity read was not sent", + WAIT.as_secs() + ), + )); + } + let fence = Fence::channel((control.close_handle().map_err(failed))?, deadline); + let answer = exchange(&mut control); + drop(fence); + answer +} + +/// The address of a running Sandbox's daemon when its record names one to +/// this host: a forwarded machine's forward, else the endpoint the start +/// recorded in `control.json`. Nothing is observed or started. +pub fn endpoint(home: &Path, sandbox: &SandboxRecord) -> Option { + if let Some(forward) = sandbox.daemon().and_then(direct::forward_of) { + return Some(forward); + } + let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name())) + .ok() + .flatten()?; + keys.get_text("endpoint")?.parse().ok() +} + +/// [`query`] under a hard wall: a daemon that accepts a connection and then +/// says nothing must not hold the caller past [`WAIT`], even where the fence +/// itself cannot be armed. +fn bounded( + within: Duration, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result { + let (sent, received) = mpsc::channel(); + std::thread::spawn(move || { + let _ = sent.send(work()); + }); + match received.recv_timeout(within) { + Ok(result) => result, + Err(_) => Err(refused( + "Unavailable", + format!("the daemon did not answer within {}s", within.as_secs()), + )), + } +} + +/// One daemon query on a connection dialed for it: its recorded endpoint with +/// its keys, for at most [`WAIT`], fenced at the same deadline, then read the +/// capacity model once. +fn query( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name()))? + .ok_or_else(|| refused("NoSuchSandbox", "the Sandbox has no daemon keys"))?; + let dial = Dial { + endpoint, + secret: secrets::key(&keys, "secret")?, + peer: secrets::key(&keys, "peer")?, + instance: secrets::key(&keys, "instance")?, + }; + let (mut control, fence) = dial.open_within(WAIT)?; + let answer = exchange(&mut control); + // this conversation ends here: a connection dropped instead leaves the + // daemon reading a closed socket, which it logs as an unexpected end + let _ = control.call(Request::EndSession); + drop(fence); + answer +} + +/// The one exchange: the daemon's answer, or the refusal it gave, kept exact. +fn exchange(control: &mut Control) -> Result { + operator::call(control, OPERATION, SdkValue::Null).map_err(|error| match error { + crate::SdkError::Refused(refusal) => refusal, + crate::SdkError::Operation(failure) => { + crate::daemon::values::refusal(&failure).unwrap_or_else(|| failed(failure)) + } + crate::SdkError::Mount(failure) => failed(failure), + }) +} diff --git a/control/crates/layerfs-sandbox/src/daemon/channel.rs b/control/crates/layerfs-sandbox/src/daemon/channel.rs index 9853146c60..49c161ef81 100644 --- a/control/crates/layerfs-sandbox/src/daemon/channel.rs +++ b/control/crates/layerfs-sandbox/src/daemon/channel.rs @@ -1,7 +1,7 @@ //! One attempted authenticated control exchange, retaining the original unknown. use layerfs_bridge::{ control::{Answer, Call, ControlError, Reply, Request, SdkValue}, - duplex::{ChannelError, ChannelWork, Connection}, + duplex::{ChannelError, ChannelWork, CloseHandle, Connection}, }; use std::fmt; /// One sequential control connection. Concurrent callers may use separate channels. @@ -74,10 +74,13 @@ impl Control { pub fn take_telemetry(&mut self) -> Vec { std::mem::take(&mut self.telemetry) } - /// True once an exchange on this connection failed: it is quarantined, - /// and every later call on it is refused before sending. + /// True once this connection cannot carry another call: an exchange + /// failed on it, its session ended, or an application fence + /// ([`Control::close_handle`]) shut its channel down. The pool closes it + /// instead of lending it again, and a later call is refused before + /// sending, so a quarantined connection is never reported as an attempt. pub fn is_failed(&self) -> bool { - self.failed + self.failed || self.connection.is_quarantined() } /// Sends once and awaits the original reply once. Typed remote refusals are /// replies; failed delivery retains an unknown, never replays the command. @@ -118,7 +121,10 @@ impl Control { let mut attempted = false; let mut received = None; let result = (|| { - if self.failed { + if self.is_failed() { + // a fence or an ended session may have quarantined the channel + // after this owner's last call: nothing is sent on it + self.failed = true; return Err(ControlCause::Channel(ChannelError::Quarantined)); } let record = call.encode().map_err(ControlCause::Protocol)?; @@ -219,7 +225,7 @@ impl Control { manifest: &layerfs_bridge::provision::StoreManifest, ) -> Result> { - if self.failed { + if self.is_failed() { return Err(Box::new(layerfs_project::store::InstallFailure { source: store.path.clone(), manifest: manifest.clone(), @@ -243,6 +249,12 @@ impl Control { pub fn clear_io_waits(&self) -> Result<(), ChannelError> { self.connection.clear_io_waits() } + /// A clone of this channel's socket for an application-owned fence + /// (`daemon::fence`): it carries no send or replay capability, and + /// shutting it down quarantines the channel. + pub fn close_handle(&self) -> Result { + self.connection.close_handle() + } /// Actual cumulative duplex send/receive work for this connection. pub fn channel_work(&self) -> (ChannelWork, ChannelWork) { (self.connection.send.work(), self.connection.receive.work()) diff --git a/control/crates/layerfs-sandbox/src/daemon/fence.rs b/control/crates/layerfs-sandbox/src/daemon/fence.rs new file mode 100644 index 0000000000..fdbbf4345b --- /dev/null +++ b/control/crates/layerfs-sandbox/src/daemon/fence.rs @@ -0,0 +1,82 @@ +//! One deadline a control connection cannot outlive (#540, #543). +//! +//! A bounded read that only sets socket timeouts bounds each read and write +//! system call, not the exchange: a daemon that trickles one byte just +//! inside the timeout can hold the reader, its connection and its pool slot +//! far past the time the caller promised. A fence instead holds a clone of +//! the connection's socket and shuts both directions down at one absolute +//! deadline, which ends the blocked call immediately. The doctor's bounded +//! observation (`doctor/sandbox_channel.rs` outside this crate) uses the same +//! pattern. +use layerfs_bridge::duplex::CloseHandle; +use std::net::{Shutdown, TcpStream}; +use std::sync::mpsc; +use std::time::Instant; + +/// What this fence ends at its deadline: a socket cloned from a connection +/// being opened, or the application-owned close handle of an established +/// channel. +enum End { + Socket(TcpStream), + Channel(CloseHandle), +} + +impl End { + fn shut(&self) { + match self { + Self::Socket(socket) => { + let _ = socket.shutdown(Shutdown::Both); + } + // the handle quarantines the channel itself, and the pool reads + // that flag, so it closes the connection instead of lending it + // again + Self::Channel(handle) => { + let _ = handle.close(); + } + } + } +} + +/// A socket fence, armed at one absolute deadline; its `Drop` cancels the +/// deadline and joins the thread that waits for it. +pub struct Fence { + stop: mpsc::Sender<()>, + thread: Option>, +} + +impl Fence { + /// Arms `socket` (a clone of the connection's) at `deadline`. + pub fn on(socket: TcpStream, deadline: Instant) -> Self { + Self::armed(End::Socket(socket), deadline) + } + + /// Arms established channel `handle` at `deadline`. + pub fn channel(handle: CloseHandle, deadline: Instant) -> Self { + Self::armed(End::Channel(handle), deadline) + } + + fn armed(end: End, deadline: Instant) -> Self { + let (stop, wait) = mpsc::channel(); + let thread = std::thread::spawn(move || { + if matches!( + wait.recv_timeout(deadline.saturating_duration_since(Instant::now())), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + end.shut(); + } + }); + Self { + stop, + thread: Some(thread), + } + } +} + +impl Drop for Fence { + fn drop(&mut self) { + let _ = self.stop.send(()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/control/crates/layerfs-sandbox/src/daemon/mod.rs b/control/crates/layerfs-sandbox/src/daemon/mod.rs index 93718cd2d4..a9a3eb2000 100644 --- a/control/crates/layerfs-sandbox/src/daemon/mod.rs +++ b/control/crates/layerfs-sandbox/src/daemon/mod.rs @@ -3,10 +3,14 @@ #[doc(hidden)] pub mod call; #[doc(hidden)] +pub mod capacity; +#[doc(hidden)] pub mod channel; #[doc(hidden)] pub mod failure; #[doc(hidden)] +pub mod fence; +#[doc(hidden)] pub mod internal; #[doc(hidden)] pub mod operator; diff --git a/control/crates/layerfs-sandbox/src/daemon/pool.rs b/control/crates/layerfs-sandbox/src/daemon/pool.rs index b5612d6e15..e7eab3a0be 100644 --- a/control/crates/layerfs-sandbox/src/daemon/pool.rs +++ b/control/crates/layerfs-sandbox/src/daemon/pool.rs @@ -5,12 +5,14 @@ //! every call wait for the one before it, a 1 ms read behind a whole Commit. A //! Sandbox keeps up to [`CONNECTIONS`] connections and opens one only when //! every open one is busy. Each exchange runs whole on one connection, sent -//! once. A connection whose exchange failed is quarantined by its `Control` -//! and is closed rather than reused; the failed call is never sent again. +//! once. A connection whose exchange failed, or whose channel an application +//! fence (`daemon::fence`) shut down, is quarantined by its `Control` and is +//! closed rather than reused; the failed call is never sent again. //! When the daemon admits no more connections, the Sandbox keeps the ones it //! has and callers take turns on them, each waiting at most [`LEND_WAIT`] //! before it is refused `Busy`. Opening a connection is not an attempt //! of any operation: nothing is sent before the connection is ready. +use crate::daemon::fence::Fence; use crate::internal::Control; use layerfs_bridge::control::{DaemonPhase, HelloRequest, Reply, Request, SdkRefusal, SdkValue}; use layerfs_bridge::duplex::{ @@ -196,12 +198,14 @@ impl Dial { self.open_admitted(ADMISSION_WAIT) } - /// [`Dial::open`] within `within` (#540 doctor): connecting, the - /// handshake and its Hello must finish inside that time, and the - /// connection keeps those I/O waits for the one bounded call made on - /// it. Nothing is retried and no admission wait applies: a daemon that - /// refuses or does not answer is reported as it came. - pub fn open_within(&self, within: Duration) -> Result { + /// [`Dial::open_within`], with the fence that ends this connection at + /// the same deadline: connecting, the handshake and its Hello must finish + /// inside that time, and a daemon that trickles any of them is shut down + /// at `deadline` rather than kept waiting. The connection keeps those I/O + /// waits for the one bounded call made on it. Nothing is retried and no + /// admission wait applies: a daemon that refuses or does not answer is + /// reported as it came. + pub fn open_within(&self, within: Duration) -> Result<(Control, Fence), SdkRefusal> { let deadline = Instant::now() + within; let remaining = || { deadline @@ -214,6 +218,9 @@ impl Dial { stream .set_write_timeout(Some(remaining())) .map_err(failed)?; + // armed before the handshake: a trickling far side cannot outlive the + // deadline this open promised, and neither can the one call made on it + let fence = Fence::on(stream.try_clone().map_err(failed)?, deadline); let connection = match initiate_observed(stream, &self.secret, self.peer).result { Ok(connection) => connection, Err(ChannelError::Admission(notice)) => return Err(notice_refusal(notice)), @@ -221,14 +228,16 @@ impl Dial { }; // the waits stay: the caller makes exactly one call and drops it let mut control = Control::new(connection); - match control + let hello = control .call(Request::Hello(HelloRequest { expected_instance: Some(self.instance), wait_for_store: false, })) - .map_err(failed)? - { - Reply::Hello(status) if status.phase == DaemonPhase::ControlReady => Ok(control), + .map_err(failed)?; + match hello { + Reply::Hello(status) if status.phase == DaemonPhase::ControlReady => { + Ok((control, fence)) + } _ => Err(refused("StaleSandbox", "saved daemon is not ControlReady")), } } @@ -403,7 +412,8 @@ impl Drop for Lease<'_> { .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if control.is_failed() { - // Quarantined: closed, and a later call may open another. + // Quarantined: its own failure or its channel's fence ended it, + // and a later call may open another. pool.open -= 1; drop(control); } else { diff --git a/control/crates/layerfs-sandbox/src/direct.rs b/control/crates/layerfs-sandbox/src/direct.rs index 280f7ef3de..f6188b3306 100644 --- a/control/crates/layerfs-sandbox/src/direct.rs +++ b/control/crates/layerfs-sandbox/src/direct.rs @@ -285,22 +285,26 @@ pub fn host(home: &Path, record: &SandboxRecord, target: &Target, message: Messa /// `sandbox.get`'s Workspace list of forwarded Sandbox `record`: a daemon /// that cannot be reached is [`Unreached`], which the get reports as down. pub fn list(home: &Path, record: &SandboxRecord) -> std::result::Result { - match list_workspaces(home, record) { + match served(home, record, Call::ListWorkspaces { path: None }) { Err(refusal) if refusal.code == "Unavailable" => Err(Unreached(refusal)), other => Ok(other), } } -/// `sandbox.list_workspaces` on forwarded Sandbox `record` in this process, -/// under the Store's lease (the Sandbox half of `workspace::direct::call`). -fn list_workspaces(home: &Path, record: &SandboxRecord) -> Outcome { +/// The daemon's control-capacity read model of forwarded Sandbox `record` +/// (#540), on the connection the Workspace list lent from this process's +/// pool for it (#543). A refused or unanswered read is the caller's to omit. +pub fn capacity(home: &Path, record: &SandboxRecord) -> Outcome { + served(home, record, Call::Capacity) +} + +/// Host `call` on forwarded Sandbox `record` in this process, under the +/// Store's lease (the Sandbox half of `workspace::direct::call`). +fn served(home: &Path, record: &SandboxRecord, call: Call) -> Outcome { let home = home.to_path_buf(); let target = resolve::target(&home, record.project().unwrap_or(""))?; let _lease = lease::hold(&home, &target.store)?; - dispatch::call( - host(&home, record, &target, Message::Absent), - Call::ListWorkspaces { path: None }, - ) + dispatch::call(host(&home, record, &target, Message::Absent), call) } /// The Workspaces forwarded Sandbox `record` holds, which its close sends diff --git a/control/crates/layerfs-sandbox/src/dispatch.rs b/control/crates/layerfs-sandbox/src/dispatch.rs index 09f3a68ccf..f610b3bf54 100644 --- a/control/crates/layerfs-sandbox/src/dispatch.rs +++ b/control/crates/layerfs-sandbox/src/dispatch.rs @@ -19,6 +19,9 @@ pub fn call(host: Host, call: Call) -> Outcome { let directory = crate::files::sandbox_dir(&host)?; listed(daemon_backend(&host, &call)?, &directory) } + // the operator view of control admission and retained failures + // (#540): a bounded read model, never a product operation + Call::Capacity => capacity(&host), Call::Create { platform } => crate::lifecycle::create(&host, platform.as_deref()), Call::Close { force, held } => { crate::lifecycle::close(&host, force, held.as_deref().unwrap_or_default()) @@ -103,6 +106,15 @@ fn daemon_backend(host: &Host, call: &Call) -> Outcome { }) } +/// The daemon's `backend.daemon.connection_failures` read model (#540): the +/// connection the Workspace list lent from the Sandbox's pool, under one +/// bound for admission and the exchange (#543). Nothing starts: the caller +/// asks only a Sandbox this process still reaches. +fn capacity(host: &Host) -> Outcome { + let state = registry::sandbox_live(host)?; + crate::daemon::capacity::lent(&state.controls) +} + /// Refuses an invalid Commit `message` before any effect. pub fn checked(message: &Message) -> Result<(), layerfs_bridge::control::SdkRefusal> { match message { diff --git a/control/crates/layerfs-sandbox/src/get.rs b/control/crates/layerfs-sandbox/src/get.rs index 3dbb7b4445..cb8485bd77 100644 --- a/control/crates/layerfs-sandbox/src/get.rs +++ b/control/crates/layerfs-sandbox/src/get.rs @@ -4,7 +4,7 @@ use super::family::{page, sandbox_of, Page}; use super::record::SandboxRecord; use crate::host::Call; use crate::route::reach; -use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::control::{CapacityModel, SdkRefusal, SdkValue}; use layerfs_home::events::Events; use layerfs_machine::worker::Unreached; use std::path::Path; @@ -44,6 +44,9 @@ pub struct Got { pub down_reason: Option, /// The next page's position; `None` after the last. pub next: Option, + /// Its daemon's control-capacity read model (#540), when a running + /// Sandbox answered the bounded read (#543). + pub daemon_capacity: Option, } /// Its record with one page of its Workspaces. Reading never clears an @@ -80,15 +83,47 @@ pub fn get(home: &Path, events: &mut Events<'_>, request: &GetRequest) -> Result Err(Unreached(why)) => (Vec::new(), Some(why.message), false), }; let (workspaces, next) = page(all, request.page); + let state = record.state(gone); + // best effort, bounded, and only for a Sandbox its daemon still serves: + // this read never fails a get (#540, #543) + let down = down_reason.is_some(); + let daemon_capacity = capacity(&home, events, &record, gone, state, down); Ok(Got { - state: record.state(gone), + state, record, workspaces, down_reason, next, + daemon_capacity, }) } +/// The daemon's `backend.daemon.connection_failures` read model of a Sandbox +/// this process still reaches (#540): read once, best effort, and omitted +/// when the Sandbox is gone, is not running, or its machine or forward is +/// already down — a get dials nothing for one of those (#543). +fn capacity( + home: &Path, + events: &mut Events<'_>, + record: &SandboxRecord, + gone: bool, + state: &str, + down: bool, +) -> Option { + if gone || down || state != "running" { + return None; + } + record.id()?; + // the forwarded daemon is dialed from here, its machine's from its worker: + // either route reads it with the connection the Workspace list lent + let answer = if direct::forwarded(record) { + direct::capacity(home, record) + } else { + reach(home, events, record, Call::Capacity, crate::dispatch::call).ok()? + }; + Some(CapacityModel::parse(&answer.ok()?)) +} + /// Workspace `item` of Sandbox `sandbox`, as its daemon answered it. fn listed_workspace(sandbox: &str, item: &SdkValue) -> ListedWorkspace { let path = (item.get_text("path")) diff --git a/control/crates/layerfs-sandbox/src/host/call.rs b/control/crates/layerfs-sandbox/src/host/call.rs index f3263bcb67..92ced5f1d5 100644 --- a/control/crates/layerfs-sandbox/src/host/call.rs +++ b/control/crates/layerfs-sandbox/src/host/call.rs @@ -19,6 +19,9 @@ pub enum Op { SandboxClose, /// `sandbox.list_workspaces`: its daemon's Workspace list. ListWorkspaces, + /// `backend.daemon.connection_failures`: its daemon's control-capacity + /// read model (#540), a bounded operator read. + Capacity, /// `workspace.mount`. Mount, /// `workspace.exec`. @@ -35,10 +38,11 @@ pub enum Op { /// Every host operation and the name a request carries (`op`). A /// machine's worker runs exactly these. -pub const OPS: [(Op, &str); 9] = [ +pub const OPS: [(Op, &str); 10] = [ (Op::SandboxCreate, "sandbox.create"), (Op::SandboxClose, "sandbox.close"), (Op::ListWorkspaces, "sandbox.list_workspaces"), + (Op::Capacity, layerfs_bridge::control::CAPACITY_OPERATION), (Op::Mount, "workspace.mount"), (Op::Exec, "workspace.exec"), (Op::ExecCancel, "workspace.exec_cancel"), @@ -100,6 +104,9 @@ pub enum Call { }, /// `sandbox.list_workspaces`, with the Workspace `path` it may name. ListWorkspaces { path: Option }, + /// `backend.daemon.connection_failures`: its daemon's control-capacity + /// read model (#540). + Capacity, /// `workspace.mount(branch, { name })`. Mount { branch: Option, @@ -149,6 +156,7 @@ impl Call { Self::Create { .. } => Op::SandboxCreate, Self::Close { .. } => Op::SandboxClose, Self::ListWorkspaces { .. } => Op::ListWorkspaces, + Self::Capacity => Op::Capacity, Self::Mount { .. } => Op::Mount, Self::Exec { .. } => Op::Exec, Self::ExecCancel { .. } => Op::ExecCancel, diff --git a/control/crates/layerfs-sandbox/src/host/frame.rs b/control/crates/layerfs-sandbox/src/host/frame.rs index afce72800c..6848360ddc 100644 --- a/control/crates/layerfs-sandbox/src/host/frame.rs +++ b/control/crates/layerfs-sandbox/src/host/frame.rs @@ -81,6 +81,8 @@ fn fields(call: &Call, sent: &Message, sandbox: Option<&str>) -> Vec<(&'static s Call::ListWorkspaces { path } | Call::Status { path } => { fields.extend(path.as_deref().map(|p| ("path", text(p)))) } + // a bounded operator read: it sends no field of its own + Call::Capacity => {} Call::Mount { branch, name } => { fields.push(("branch", opt(branch))); fields.extend(name.as_deref().map(|n| ("name", text(n)))); @@ -248,6 +250,7 @@ fn call(op: Op, more: &SdkValue) -> Call { }, }, Op::ListWorkspaces => Call::ListWorkspaces { path: text("path") }, + Op::Capacity => Call::Capacity, Op::Mount => Call::Mount { branch: text("branch"), name: text("name"), diff --git a/control/crates/layerfs-sandbox/tests/capacity_read.rs b/control/crates/layerfs-sandbox/tests/capacity_read.rs new file mode 100644 index 0000000000..b90709a375 --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/capacity_read.rs @@ -0,0 +1,355 @@ +//! The `backend.daemon.connection_failures` read model's bounds (#540, +//! #543): one query ends its control session instead of dropping the socket, +//! a daemon that trickles is fenced at the deadline, a read on the Sandbox's +//! pool takes no second connection, and a connection a fence or a session end +//! quarantined is never lent again or reported as an attempt. +//! +//! Nothing runs: a stand-in daemon answers the query on the connections it +//! accepts, and counts what it saw. +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request, + RetainedModel, SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; +use layerfs_home::home::paths; +use layerfs_home::secrets; +use layerfs_sandbox::daemon::capacity; +use layerfs_sandbox::daemon::channel::ControlPhase; +use layerfs_sandbox::daemon::operator; +use layerfs_sandbox::daemon::pool::{Controls, Dial}; +use layerfs_sandbox::host::{Call as HostCall, Host, Message}; +use layerfs_sandbox::record::SandboxRecord; +use layerfs_sandbox::{OperationCause, SdkError}; +use std::io::Write; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("lfs-capr-{tag}-{}-{sequence}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +/// A model of a pool with no vacant slot (and no retained failure). +fn model() -> SdkValue { + CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel::default(), + } + .to_sdk() +} + +fn dial(endpoint: SocketAddr) -> Dial { + Dial { + endpoint, + secret: CLIENT_PRIVATE, + peer: duplex::public_key(&DAEMON_PRIVATE).unwrap(), + instance: DAEMON_INSTANCE, + } +} + +/// What a stand-in daemon saw, and how it treats the query. +#[derive(Default)] +struct Seen { + connections: AtomicUsize, + queries: AtomicUsize, + ended: AtomicBool, + fenced: AtomicBool, +} + +/// How the stand-in answers the capacity query. +#[derive(Clone, Copy)] +enum Behavior { + /// With the model. + Model, + /// Never: it trickles one byte every `TRICKLE` until the far side closes + /// the socket (`Seen::fenced`). + Trickle, +} + +const TRICKLE: Duration = Duration::from_millis(400); + +/// A stand-in daemon at its own loopback port. +fn daemon(answer: Behavior) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let seen = Arc::new(Seen::default()); + let counted = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + counted.connections.fetch_add(1, Ordering::SeqCst); + let seen = counted.clone(); + std::thread::spawn(move || serve(stream, answer, &seen)); + } + }); + (endpoint, seen) +} + +/// One stand-in connection: its Hello, then each call's answer; a query ends +/// the session when the caller sends its end. +fn serve(stream: TcpStream, answer: Behavior, seen: &Seen) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + // kept for a trickling daemon: ciphertext is the channel's to write + let mut raw = stream.try_clone().ok(); + let Ok(mut channel) = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) else { + return; + }; + while let Ok(bytes) = channel.receive.receive() { + let Ok(call) = Call::decode(bytes) else { break }; + let id = call.id; + let reply = match call.request { + Request::Hello(_) => Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + Request::EndSession => { + seen.ended.store(true, Ordering::SeqCst); + Reply::SessionEnded + } + Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + seen.queries.fetch_add(1, Ordering::SeqCst); + if matches!(answer, Behavior::Trickle) { + if let Some(raw) = raw.as_mut() { + trickle(raw, seen); + } + break; + } + Reply::Sdk(model()) + } + _ => continue, + }; + if channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .is_err() + { + break; + } + } +} + +/// Keeps the far side waiting without ever completing a record, until it ends +/// the connection itself. +fn trickle(raw: &mut TcpStream, seen: &Seen) { + let byte = [0u8; 1]; + for _ in 0..20 { + std::thread::sleep(TRICKLE); + if raw.write_all(&byte).and_then(|()| raw.flush()).is_err() { + seen.fenced.store(true, Ordering::SeqCst); + return; + } + } +} + +/// Sandbox `s1`'s folder in `home`, recording the daemon keys its queries +/// read (`control.json`) and, with `forward`, the record that reaches its +/// daemon through that forward. +fn sandbox_folder(home: &Path, endpoint: SocketAddr, forward: bool) { + let dir = paths::sandbox_dir(home, "s1"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let keys = SdkValue::map([ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]); + secrets::save_daemon_keys(&dir, &keys).unwrap(); + if forward { + let record = SdkValue::map([ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("s-1")), + ( + "daemon", + SdkValue::map([( + "forward", + SdkValue::map([ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]), + )]), + ), + ]); + layerfs_home::home::records::write_record(&dir.join("record.json"), &record).unwrap(); + layerfs_home::home::records::write_record( + &paths::worker_record(home, "native"), + &SdkValue::map([("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + } +} + +fn record(name: &str) -> SandboxRecord { + SandboxRecord::from_record(name, SdkValue::map([])) +} + +#[test] +fn a_query_ends_its_control_session() { + let (endpoint, seen) = daemon(Behavior::Model); + let home = tempdir("session"); + sandbox_folder(&home, endpoint, false); + let got = capacity::read(&home, &record("s1"), endpoint).unwrap(); + assert_eq!(got.admission.pool_running, 8); + // the one conversation ended before the socket was dropped, which the + // daemon would otherwise log as an unexpected end + assert!(seen.ended.load(Ordering::SeqCst), "no EndSession was sent"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_trickling_daemon_is_fenced_at_the_deadline() { + let (endpoint, seen) = daemon(Behavior::Trickle); + let home = tempdir("trickle"); + sandbox_folder(&home, endpoint, false); + let at = Instant::now(); + let refused = capacity::read(&home, &record("s1"), endpoint).err(); + let elapsed = at.elapsed(); + assert!(refused.is_some(), "{refused:?}"); + assert!( + elapsed < capacity::WAIT + Duration::from_millis(1500), + "the read outlived its deadline: {elapsed:?}" + ); + // the socket was shut down, not merely abandoned: the daemon's own write + // failed + let until = Instant::now() + Duration::from_secs(3); + while !seen.fenced.load(Ordering::SeqCst) && Instant::now() < until { + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + seen.fenced.load(Ordering::SeqCst), + "the fence did not close the connection" + ); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_pooled_read_takes_no_second_connection() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + let one = capacity::lent(&controls).unwrap(); + let two = capacity::lent(&controls).unwrap(); + assert_eq!(CapacityModel::parse(&one).admission.pool_vacant, 0); + assert_eq!(CapacityModel::parse(&two).admission.pool_running, 8); + // both reads went over the one connection the pool holds, so a get takes + // a single control slot for the list and the capacity read together + assert_eq!(seen.connections.load(Ordering::SeqCst), 1); + assert_eq!(seen.queries.load(Ordering::SeqCst), 2); +} + +#[test] +fn the_worker_s_side_reads_the_capacity_model() { + let (endpoint, seen) = daemon(Behavior::Model); + let home = tempdir("worker"); + std::fs::create_dir_all(&home).unwrap(); + sandbox_folder(&home, endpoint, true); + // the request a machine's worker decodes (`sandbox.host`) and the host + // operation it runs: no Store is opened and no image is loaded + let host = Host { + home: home.clone(), + store: "shop".to_owned(), + project: "api".to_owned(), + sandbox: Some("s1".to_owned()), + store_record: None, + store_path: None, + deploy: None, + postgresql: None, + message: Message::Absent, + started: None, + }; + let answer = layerfs_sandbox::dispatch::call(host, HostCall::Capacity).unwrap(); + let model = CapacityModel::parse(&answer); + assert_eq!(model.admission.pool_vacant, 0, "{answer:?}"); + assert_eq!(model.admission.reserved_vacant, 1, "{answer:?}"); + assert_eq!(seen.queries.load(Ordering::SeqCst), 1); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_fenced_connection_is_not_lent_again() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + { + let lease = controls.lend().unwrap().0; + // the fence's end, outside any exchange: the channel is shut down + // while this `Control` never saw a call fail, which is the window + // between a returned reply and `drop(fence)` + lease.close_handle().unwrap().close().unwrap(); + assert!( + lease.is_failed(), + "a channel a fence closed was still reported healthy" + ); + } + // and the pool closed it rather than lending it again: the next read + // opens a connection and answers, instead of reporting the dead one + let answer = capacity::lent(&controls).unwrap(); + assert_eq!(CapacityModel::parse(&answer).admission.pool_vacant, 0); + assert_eq!(seen.connections.load(Ordering::SeqCst), 2); +} + +#[test] +fn a_quarantined_connection_attempts_nothing() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + let mut lease = controls.lend().unwrap().0; + lease.close_handle().unwrap().close().unwrap(); + // the call a pooled lease would make next: refused before sending, so it + // is not an attempt whose effect is unknown + let error = operator::call(&mut lease, CAPACITY_OPERATION, SdkValue::Null).unwrap_err(); + match error { + SdkError::Operation(failure) => match failure.cause { + OperationCause::Exchange(channel) => { + assert!( + !channel.attempted, + "a closed channel was reported attempted" + ); + assert_eq!(channel.phase, ControlPhase::Encode); + } + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + } + assert_eq!(seen.queries.load(Ordering::SeqCst), 0); +} diff --git a/control/crates/layerfs-sandbox/tests/host_frame.rs b/control/crates/layerfs-sandbox/tests/host_frame.rs index d43a315aa9..a940d00b22 100644 --- a/control/crates/layerfs-sandbox/tests/host_frame.rs +++ b/control/crates/layerfs-sandbox/tests/host_frame.rs @@ -74,6 +74,7 @@ fn a_frame_reads_back_to_its_request() { held: None, }, Call::ListWorkspaces { path: None }, + Call::Capacity, Call::Mount { branch: Some("dev".to_owned()), name: Some("w2".to_owned()), diff --git a/control/crates/layerfs-sandbox/tests/pool_admission.rs b/control/crates/layerfs-sandbox/tests/pool_admission.rs new file mode 100644 index 0000000000..cbd9429ffd --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/pool_admission.rs @@ -0,0 +1,82 @@ +//! A daemon's advisory capacity notice (#540) as a client reads it: a +//! daemon whose pool has no vacant slot answers the next connection with +//! the notice before any handshake, so the refusal is `Capacity` naming the +//! resource, the vacant slots and the retained failure count, not a bare +//! EOF. Nothing is sent by the client and nothing is retried, so the caller +//! may try again. A daemon that is still starting says so the same way. +use layerfs_bridge::duplex; +use layerfs_bridge::duplex::admission::{self, AdmissionNotice, AdmissionReason}; +use layerfs_sandbox::daemon::pool::Dial; +use std::io::Read; +use std::net::{SocketAddr, TcpListener}; +use std::time::Duration; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// A daemon that refuses every connection with `notice` before any +/// handshake, exactly as an accept loop does when no slot is vacant: it +/// writes the notice, shuts its write end and drains what the client sent. +fn refusing(notice: AdmissionNotice) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let _ = stream.set_write_timeout(Some(Duration::from_millis(100))); + let _ = stream.set_read_timeout(Some(Duration::from_millis(50))); + if admission::write(&mut stream, ¬ice).is_err() { + continue; + } + let mut sink = [0u8; 512]; + let _ = stream.read(&mut sink); + } + }); + endpoint +} + +fn dial(endpoint: SocketAddr) -> Dial { + Dial { + endpoint, + secret: CLIENT_PRIVATE, + peer: duplex::public_key(&DAEMON_PRIVATE).unwrap(), + instance: DAEMON_INSTANCE, + } +} + +#[test] +fn a_full_pool_refuses_the_next_client_with_its_counts() { + let endpoint = refusing(AdmissionNotice::new(AdmissionReason::NoPoolSlot, 3, 0, 1)); + let refusal = dial(endpoint) + .open_within(Duration::from_secs(2)) + .err() + .expect("a full pool admits nothing"); + assert_eq!(refusal.code, "Capacity", "{refusal:?}"); + let detail = &refusal.detail; + assert_eq!(detail.get_text("resource"), Some("daemon_control_slot")); + assert_eq!(detail.get_int("pool_vacant"), Some(0)); + assert_eq!(detail.get_int("reserved_vacant"), Some(1)); + assert_eq!(detail.get_int("retained"), Some(3)); + assert_eq!(detail.get_text("admission"), Some("no-pool-slot")); + // nothing was attempted, so a later open is safe + assert!(detail.get_bool("retryable")); + assert!( + refusal.message.contains("no vacant control slot"), + "{refusal:?}" + ); + assert!(refusal.message.contains("3 retained"), "{refusal:?}"); +} + +#[test] +fn a_still_starting_daemon_is_not_a_full_pool() { + let endpoint = refusing(AdmissionNotice::new(AdmissionReason::Starting, 0, 8, 1)); + let refusal = dial(endpoint) + .open_within(Duration::from_secs(2)) + .err() + .expect("a starting daemon admits nothing"); + // its own code and admission, not Capacity: a full pool it is not + assert_eq!(refusal.code, "Unavailable", "{refusal:?}"); + assert_eq!(refusal.detail.get_text("admission"), Some("starting")); + assert!(refusal.detail.get_bool("retryable")); +} diff --git a/docs/architecture/api/sdk/sandbox/README.md b/docs/architecture/api/sdk/sandbox/README.md index 759acfeacd..d1724454f6 100644 --- a/docs/architecture/api/sdk/sandbox/README.md +++ b/docs/architecture/api/sdk/sandbox/README.md @@ -36,7 +36,7 @@ unique across the home. | --- | --- | --- | --- | --- | | [`create`](create.md) | `project` (Project), `name` (Name), `machine?` (Name) | `image` (Text) | the Sandbox record with `state`; `forward_failed` when a direct daemon forward did not open | its machine: here or `sandbox.host` in the worker | | [`list`](list.md) | `project?` (Project) | `page_size`, `page_pos` (Int) | `{ items: [record], next_page_pos }` | caller's process, home records only | -| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, and `down_reason` when its machine is down | its machine or its daemon; starts nothing | +| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing | | [`close`](close.md) | `name` (Name) | `force` (Bool) | `{ kind: "Closed", sandbox, unknown, closed }`, or `{ closed, was_gone: true }`, or `{ closed, was, left? }` for a record with no id | its machine: here or `sandbox.host` in the worker | The record is `SandboxRecord::to_sdk` @@ -47,7 +47,11 @@ its stored fields except `v`, `runtime_id` and `daemon`, plus the derived ([`op.rs`](../../../../../control/crates/layerfs-home/src/op.rs)). Each Workspace in `get` carries `ws` (`:` or `:`), `name` and `path` -([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)). +([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)), and +its `daemon_capacity` is the daemon's `backend.daemon.connection_failures` +read model, read best effort and bounded to 2 s on the connection the +Workspace list already lent +([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)). ## Arguments and options diff --git a/docs/architecture/api/sdk/sandbox/get.md b/docs/architecture/api/sdk/sandbox/get.md index 0a91d70c2b..accb9c91d5 100644 --- a/docs/architecture/api/sdk/sandbox/get.md +++ b/docs/architecture/api/sdk/sandbox/get.md @@ -11,8 +11,8 @@ its daemon holds, without starting anything. | --- | --- | | Family | [sandbox](README.md) | | Library entry | [`sandbox/get.rs`](../../../../../api/crates/layerfs-api/src/sandbox/get.rs) | -| Implemented in | the library entry parses a typed `GetRequest` → `layerfs_sandbox::family::get` → `get::get`, whose typed `Got` it shapes ([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)) | -| Runs in | the caller's process; the Workspace list comes from the Sandbox daemon, dialed from here (forwarded or local) or from the machine's worker | +| Implemented in | the library entry parses a typed `GetRequest` → `layerfs_sandbox::family::get` → `get::get`, whose typed `Got` (its `daemon_capacity` included) it shapes ([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)) | +| Runs in | the caller's process; the Workspace list and the daemon's capacity read model come from the Sandbox daemon, dialed from here (forwarded or local) or from the machine's worker | ## Signature @@ -58,7 +58,15 @@ layerfs sandbox get [--page-size ] [--page-pos ] head_moved_since_mount, dirty: {inodes, entries}?, pending_merge, revision } ], next_page_pos: int | null, - down_reason?: text } + down_reason?: text, + daemon_capacity?: { admission: { pool_vacant, pool_running, + reserved_vacant, reserved_running }, + retained: { limit, ttl_ms, count, dropped, recorded, + failures: [ { slot, class, generation, owner, + received_bytes, request, reply_sent, + cause, fence_error, + diagnostic_error, age_ms, + expires_in_ms } ] } } } ``` | Field | Kind | Meaning | @@ -70,6 +78,7 @@ layerfs sandbox get [--page-size ] [--page-pos ] | `workspaces[].*` | | the daemon's read model per Workspace: id, namespace, Branch, base Commit and Layer, root, `activity`, dirty counts ([`operator/inspect.rs`](../../../../../engine/crates/layerfs-store-service/src/operator/inspect.rs) `workspace_value`) | | `next_page_pos` | int or null | next page of `workspaces` | | `down_reason` | text | the machine's worker or the forwarded daemon was not reached; `workspaces` is then `[]` | +| `daemon_capacity` | record | the daemon's `backend.daemon.connection_failures` read model (#540): control admission by slot class and the bounded retained connection failures that used to hold a slot ([`control_capacity.rs`](../../../../../engine/crates/layerfs-bridge/src/control_capacity.rs)). It is read once, best effort, and only for a Sandbox that is `running` and whose machine and forward were reached; nothing is started and no second control slot is taken, so the field is simply absent when the daemon refuses, does not answer within 2 s, or was not asked ([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)) | ## Where it runs @@ -86,10 +95,23 @@ layerfs sandbox get [--page-size ] [--page-pos ] v Unreached / Unavailable --> record + workspaces [] + down_reason NoSuchSandbox --> state "lost", workspaces [] + + running and not down, same route and the same pooled connection: + backend.daemon.connection_failures --> daemon_capacity (2 s, best effort) ``` `get` starts no machine and loads no image (#535 P4); reading never clears -an intent. See the [routing table](../../../control/routing/routing-table.md) and the +an intent. The capacity read is a second host operation on the connection +the Workspace list lent from the Sandbox's pool +([`daemon/pool.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs)), +bounded to 2 s for admission and for +the exchange and fenced at that deadline ([`daemon/fence.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs)), +so a daemon that trickles cannot hold the slot, and a connection the fence +ends is quarantined rather than lent again; a window the admission wait used +up refuses the read before it is sent. A forwarded daemon's forward +and its machine are checked before it is dialed (`direct::check`), so a +stopped machine is `down_reason` and never a 2 s wait on a reused port. See +the [routing table](../../../control/routing/routing-table.md) and the [daemon channel](../../../control/routing/daemon-channel.md). ## Errors diff --git a/docs/architecture/control/routing/daemon-channel.md b/docs/architecture/control/routing/daemon-channel.md index a788b0c3ad..67bd11168d 100644 --- a/docs/architecture/control/routing/daemon-channel.md +++ b/docs/architecture/control/routing/daemon-channel.md @@ -33,7 +33,7 @@ side of the channel is in [Engine: Daemon](../../engine/daemon/README.md). | | | lend(): idle one --> else open one more (if open < limit) | | --> else wait for a free one, <= 60 s --> Busy | - | drop(Lease): failed Control --> closed, open -= 1 | + | drop(Lease): quarantined Control --> closed, open -= 1 | | healthy Control --> back to idle | +--------------------------------------------------------------+ | @@ -43,7 +43,7 @@ side of the channel is in [Engine: Daemon](../../engine/daemon/README.md). | Part | What it is | Source | | --- | --- | --- | -| `Control` | one sequential authenticated connection; numbers its calls; once an exchange fails it is quarantined and refuses every later call before sending | [`daemon/channel.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/channel.rs) | +| `Control` | one sequential authenticated connection; numbers its calls; once an exchange fails, or an applied fence (`close_handle`, `daemon/fence.rs`) closes its channel, it is quarantined and refuses every later call before sending | [`daemon/channel.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/channel.rs) | | `Dial` | how to open another connection to the same daemon instance: endpoint, secret, peer key, instance | [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | | `Controls` | the Sandbox's pool: up to `CONNECTIONS` (6) per process; the daemon admits `DAEMON_CONNECTIONS` (8), leaving the rest to other processes | `daemon/pool.rs`, [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) | | `Lease` | one connection lent for one call's exchanges, returned on drop | `daemon/pool.rs` | @@ -82,7 +82,7 @@ opening a connection sends only Hello, so it attempts no operation. | `workspace.commit` | `Commit` | `values::commit`: Committed, Merged (with `merged`), or `Conflict` | [`daemon/workspace/commit.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/commit.rs), [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | `workspace.unmount` | `Unmount{token, discard}`, or `ForceUnmount` with `force` | `values::unmounted` (`Dirty` refusal) or `force_unmounted` | [`daemon/workspace/unmount.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/unmount.rs) | | `workspace.status` | `Status(token)`; the Branch head is read separately on the Store's route | `values::status` | [`daemon/workspace/status.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/status.rs) | -| `sandbox.get` Workspace list | SDK method `sandbox.list_workspaces`, the one daemon backend method sent | `sandbox/dispatch.rs` | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs) | +| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs); a window admission used up refuses the read, and a connection the fence ends is not lent again) | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | | `workspace.exec` | none on this channel: `bash -c` runs through the Sandbox's runtime | — | [`exec/run.rs`](../../../../control/crates/layerfs-sandbox/src/exec/run.rs) | `Locate` and `Cleanup` are read-only observations for a caller whose diff --git a/engine/crates/layerfs-bridge/src/duplex/channel.rs b/engine/crates/layerfs-bridge/src/duplex/channel.rs index 0e56901692..76abb4c192 100644 --- a/engine/crates/layerfs-bridge/src/duplex/channel.rs +++ b/engine/crates/layerfs-bridge/src/duplex/channel.rs @@ -62,6 +62,13 @@ impl Connection { shared: self.receive.shared.clone(), }) } + /// Whether this channel is quarantined: a failed exchange, an ended + /// session or an application fence (`CloseHandle`) shut both directions + /// down, and no record may be sent on it again. All owners read the one + /// shared flag, so a fence's close is visible to whoever borrowed it. + pub fn is_quarantined(&self) -> bool { + self.receive.shared.ready().is_err() + } /// Clears explicit handshake I/O waits on the actual shared socket before long-lived operations. /// An original policy-change error quarantines/fences this owner once. pub fn clear_io_waits(&self) -> ChannelResult<()> {