From 5002922ea1a4c4204617f77d1913bd433f5781d6 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:41:40 +0800 Subject: [PATCH 1/6] feat(sandbox): expose the daemon's control capacity in sandbox.get (#540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sandbox.get` now carries `daemon_capacity`: the daemon's `backend.daemon.connection_failures` read model, the one `layerfs doctor` explains and its fix text already pointed at. The field is best-effort and bounded to 2 s: the query runs in its own thread behind a hard wall, so a daemon that accepts and stalls cannot hold the read, and a daemon that does not answer (stopped, gone, at no known endpoint, or holding no vacant control slot) leaves the field out instead of failing `sandbox.get`. The field is only added for a running Sandbox, and nothing is started to find out. The one query itself moved out of the doctor into `layerfs_sandbox::daemon::capacity` (`endpoint`, `read`, `OPERATION`, `WAIT`), so the doctor and the read model share one implementation: where the daemon is, how long an answer may take, and what an unanswered query is are decided once. Tests: layerfs-api --test sandbox_capacity 2 passed — a stand-in daemon's model reaches `sandbox.get` with its admission counts and retained failure record, and a daemon that reads the query and never answers leaves the field out after the bounded wait while the get still answers. layerfs-control --test doctor unchanged and passing (6 passed). Clippy -D warnings clean on layerfs-api, layerfs-control, layerfs-bridge and layerfs-sandbox; no test failures in those four or layerfs-daemon. Production LOC: 156991 -> 157011 (delta +20); api 4966 -> 4975 (+9), control +11. Counted with tools/production_loc.py over core/crates/*/src, engine/crates/*/src, control/crates/*/src and api/crates/*/src (tests, docs, manifests and generated artifacts excluded). --- api/crates/layerfs-api/src/sandbox/get.rs | 16 +- .../layerfs-api/tests/sandbox_capacity.rs | 290 ++++++++++++++++++ .../layerfs-control/src/doctor/capacity.rs | 88 +----- .../layerfs-sandbox/src/daemon/capacity.rs | 90 ++++++ .../crates/layerfs-sandbox/src/daemon/mod.rs | 2 + 5 files changed, 404 insertions(+), 82 deletions(-) create mode 100644 api/crates/layerfs-api/tests/sandbox_capacity.rs create mode 100644 control/crates/layerfs-sandbox/src/daemon/capacity.rs diff --git a/api/crates/layerfs-api/src/sandbox/get.rs b/api/crates/layerfs-api/src/sandbox/get.rs index f1e05b8467..6d46def847 100644 --- a/api/crates/layerfs-api/src/sandbox/get.rs +++ b/api/crates/layerfs-api/src/sandbox/get.rs @@ -5,6 +5,7 @@ use crate::calls::Context; use layerfs_bridge::control::SdkValue; use layerfs_home::home::with_field; use layerfs_home::outcome::{Outcome, View}; +use layerfs_sandbox::daemon::capacity; use layerfs_sandbox::family::{get, GetRequest, Page}; /// Its table entry. @@ -19,7 +20,9 @@ pub(crate) const CALL: Call = call( /// Runs `sandbox.get`: its record with its derived `state`, one page of /// its `workspaces` (each with its `path`, `name` and `ws`), why its -/// machine did not answer (`down_reason`) and `next_page_pos`. +/// machine did not answer (`down_reason`), the daemon's +/// `daemon_capacity` read model when it answers within +/// [`capacity::WAIT`] (#540), and `next_page_pos`. fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { let request = GetRequest { name: args.get_text("name").unwrap_or("").to_owned(), @@ -30,6 +33,14 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { }; let (home, events) = context.parts(); let got = get(home, events, &request)?; + // best-effort and bounded: a daemon that does not answer (stopped, gone, + // at no known endpoint, or holding no vacant control slot) leaves the + // field out and never fails the read + let daemon_capacity = match got.state { + "running" => capacity::endpoint(home, &got.record) + .and_then(|endpoint| capacity::read(home, &got.record, endpoint).ok()), + _ => None, + }; let workspaces = got.workspaces.into_iter().map(|listed| { let item = with_field(listed.item, "path", SdkValue::Text(listed.path)); let item = with_field(item, "name", SdkValue::opt_text(listed.name)); @@ -41,6 +52,9 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { if let Some(why) = got.down_reason { shown = with_field(shown, "down_reason", SdkValue::Text(why)); } + if let Some(model) = daemon_capacity { + shown = with_field(shown, "daemon_capacity", model.to_sdk()); + } let next = super::list::next_page_pos(got.next); Ok(with_field(shown, "next_page_pos", next)) } diff --git a/api/crates/layerfs-api/tests/sandbox_capacity.rs b/api/crates/layerfs-api/tests/sandbox_capacity.rs new file mode 100644 index 0000000000..b9b813adaf --- /dev/null +++ b/api/crates/layerfs-api/tests/sandbox_capacity.rs @@ -0,0 +1,290 @@ +//! `sandbox.get`'s `daemon_capacity` (#540): the daemon's +//! `backend.daemon.connection_failures` read model, shown best-effort and +//! bounded the same way the doctor reads it. Nothing runs: a stand-in +//! daemon answers the Sandbox's Workspace list and its capacity model, a +//! second one answers the list and then says nothing. +#![cfg(unix)] +use layerfs_api::home::{self, paths, write_record}; +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, DaemonPhase, DaemonStatus, Reply, Request, RetainedModel, + SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; +use std::net::{SocketAddr, TcpListener}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +fn tempdir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("lfs-scap-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +/// What the stand-in daemon answers `sandbox.list_workspaces` with: no +/// Workspaces, like a Sandbox nothing has mounted in. +fn no_workspaces() -> SdkValue { + SdkValue::map([ + ("kind", t("workspaces")), + ("version", SdkValue::Int(1)), + ("workspaces", SdkValue::List(vec![])), + ]) +} + +/// A stand-in daemon at its own loopback port: it answers this Sandbox's +/// Workspace list, then its capacity model with `model`. With `stall` it +/// answers the Workspace list, reads the capacity query and never answers +/// it, which is what a daemon stuck behind its own slots looks like. +fn daemon(model: SdkValue, stall: bool) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let model = std::sync::Arc::new(model); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + let model = model.clone(); + // one worker per connection: a used list connection stays open + // beside the capacity query's own + std::thread::spawn(move || serve(stream, &model, stall)); + } + }); + endpoint +} + +/// One stand-in connection: its Hello, then each call's answer until the +/// caller closes it. +fn serve(stream: std::net::TcpStream, model: &SdkValue, stall: bool) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + let Ok(mut channel) = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) else { + return; + }; + while let Ok(bytes) = channel.receive.receive() { + let Ok(call) = Call::decode(bytes) else { break }; + let id = call.id; + let reply = match call.request { + Request::Hello(_) => Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + Request::Sdk(sdk) if sdk.method == "sandbox.list_workspaces" => { + Reply::Sdk(no_workspaces()) + } + Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + if stall { + // read but never answered: the caller's own bound, not a + // closed socket, ends its wait + std::thread::sleep(Duration::from_secs(6)); + break; + } + Reply::Sdk(model.clone()) + } + _ => continue, + }; + if channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .is_err() + { + break; + } + } +} + +/// A record from `fields`, in order. +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +/// A home with Store `shop` (project `api`) and Sandbox `s1` on `native`, +/// recorded as a started Sandbox whose daemon this host reaches through the +/// forward `daemon` names; its keys are `endpoint`'s. +fn forwarded_sandbox(home: &Path, endpoint: SocketAddr) { + write_record( + &paths::machine_record(home, "native"), + &map(&[("name", t("native")), ("platform", t("native"))]), + ) + .unwrap(); + // the forward's machine answers: a forward with no recorded process is + // not checked against one + write_record( + &paths::worker_record(home, "native"), + &map(&[("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + write_record( + &paths::store_record(home, "shop"), + &map(&[ + ("name", t("shop")), + ("kind", t("sqlite")), + ("binding", SdkValue::Bytes(vec![7; 16])), + ("cursor", SdkValue::Bytes(vec![8; 32])), + ("profile", t("disposable")), + ("machine", t("native")), + ]), + ) + .unwrap(); + write_record( + &paths::project_record(home, "shop", "api"), + &map(&[("name", t("api"))]), + ) + .unwrap(); + write_record( + &paths::sandbox_record(home, "s1"), + &map(&[ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("s-1")), + ( + "daemon", + map(&[ + ("name", t("lfs-x-s1")), + ( + "forward", + map(&[ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]), + ), + ]), + ), + ]), + ) + .unwrap(); + write_record( + &paths::sandbox_control(home, "s1"), + &map(&[ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]), + ) + .unwrap(); +} + +/// A model with no vacant pool slot and one retained failure. +fn busy() -> SdkValue { + let model = layerfs_bridge::control::CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel { + limit: 64, + ttl_ms: 3_600_000, + count: 1, + dropped: 0, + recorded: 1, + failures: vec![layerfs_bridge::control::RetainedFailureModel { + slot: 0, + class: "pool".into(), + generation: 6, + owner: "daemon instance, pid 4, thread layerfs-control-0".into(), + received_bytes: None, + request: Some("unmount".into()), + reply_sent: false, + cause: "control: the kernel control did not answer".into(), + fence_error: None, + diagnostic_error: None, + age_ms: 12, + expires_in_ms: 3_599_988, + }], + }, + }; + model.to_sdk() +} + +#[test] +fn a_running_sandbox_s_get_shows_its_daemon_capacity() { + let endpoint = daemon(busy(), false); + let home = tempdir("shown"); + home::open(&home).unwrap(); + forwarded_sandbox(&home, endpoint); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + let capacity = got + .get("daemon_capacity") + .unwrap_or_else(|| panic!("{got:?}")); + let admission = capacity.get("admission").unwrap(); + assert_eq!(admission.get_int("pool_vacant"), Some(0), "{capacity:?}"); + assert_eq!(admission.get_int("pool_running"), Some(8), "{capacity:?}"); + assert_eq!( + admission.get_int("reserved_vacant"), + Some(1), + "{capacity:?}" + ); + let retained = capacity.get("retained").unwrap(); + assert_eq!(retained.get_int("count"), Some(1), "{capacity:?}"); + assert_eq!(retained.get_int("limit"), Some(64), "{capacity:?}"); + let Some(SdkValue::List(failures)) = retained.get("failures") else { + panic!("{capacity:?}") + }; + assert_eq!(failures.len(), 1, "{capacity:?}"); + assert_eq!( + failures[0].get_text("cause"), + Some("control: the kernel control did not answer") + ); + assert_eq!(failures[0].get_int("generation"), Some(6)); + assert_eq!(failures[0].get_text("request"), Some("unmount")); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_daemon_that_does_not_answer_leaves_daemon_capacity_out() { + let endpoint = daemon(busy(), true); + let home = tempdir("stalled"); + home::open(&home).unwrap(); + forwarded_sandbox(&home, endpoint); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + // the read model's own bound ends the wait; the get still answers + assert!( + elapsed < Duration::from_secs(4), + "sandbox.get stalled for {elapsed:?}" + ); + assert!( + elapsed >= Duration::from_secs(1), + "the bounded query did not wait: {elapsed:?}" + ); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-control/src/doctor/capacity.rs b/control/crates/layerfs-control/src/doctor/capacity.rs index 6f35cf23c1..f69dda51a1 100644 --- a/control/crates/layerfs-control/src/doctor/capacity.rs +++ b/control/crates/layerfs-control/src/doctor/capacity.rs @@ -5,29 +5,15 @@ //! naming its endpoint and the operation, never an error, because the //! doctor changes nothing and a stopped daemon is a normal state. //! Two more warnings come from the model itself: retained connection -//! failures, and a pool with no vacant control slot. +//! failures, and a pool with no vacant control slot. The one bounded query +//! itself is `layerfs_sandbox::daemon::capacity`, shared with the +//! `daemon_capacity` field `sandbox.get` shows. use super::report::Findings; use super::scan::Scanned; -use layerfs_bridge::control::{CapacityModel, SdkRefusal, SdkValue}; -use layerfs_home::home::paths; -use layerfs_home::outcome::{failed, refused}; -use layerfs_home::secrets; -use layerfs_sandbox::daemon::operator; -use layerfs_sandbox::direct; -use layerfs_sandbox::record::SandboxRecord; +use layerfs_bridge::control::{CapacityModel, SdkRefusal}; +use layerfs_sandbox::daemon::capacity::{endpoint, read, OPERATION, WAIT}; use std::net::SocketAddr; use std::path::Path; -use std::sync::mpsc; -use std::time::Duration; - -/// The daemon's operator read model of control admission and retained -/// connection failures. -const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; - -/// How long one daemon query may take: connecting, its Hello and its one -/// exchange. Short on purpose: the doctor must not stall behind a daemon -/// that is unreachable, as an ordinary 60 s admission wait would. -const WAIT: Duration = Duration::from_secs(2); /// Queries the daemon of every running Sandbox whose endpoint is known. pub(super) fn check(home: &Path, scanned: &Scanned, findings: &mut Findings) { @@ -43,73 +29,13 @@ pub(super) fn check(home: &Path, scanned: &Scanned, findings: &mut Findings) { // listens, and the doctor starts nothing to find out continue; }; - match probe_bounded(home, sandbox, endpoint) { - Ok(value) => capacity(findings, name, endpoint, &CapacityModel::parse(&value)), + match read(home, sandbox, endpoint) { + Ok(model) => capacity(findings, name, endpoint, &model), Err(refusal) => unanswered(findings, name, endpoint, &refusal), } } } -/// The address of a running Sandbox's daemon when its record names one to -/// this host: a forwarded machine's forward, else the endpoint the start -/// recorded in `control.json`. Nothing is observed or started. -fn endpoint(home: &Path, sandbox: &SandboxRecord) -> Option { - if let Some(forward) = sandbox.daemon().and_then(direct::forward_of) { - return Some(forward); - } - let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name())) - .ok() - .flatten()?; - keys.get_text("endpoint")?.parse().ok() -} - -/// [`probe`] under a hard wall: a daemon that accepts a connection and then -/// says nothing must not hold the doctor past [`WAIT`]. -fn probe_bounded( - home: &Path, - sandbox: &SandboxRecord, - endpoint: SocketAddr, -) -> Result { - let home = home.to_owned(); - let sandbox = sandbox.clone(); - let (sent, received) = mpsc::channel(); - std::thread::spawn(move || { - let _ = sent.send(probe(&home, &sandbox, endpoint)); - }); - match received.recv_timeout(WAIT) { - Ok(result) => result, - Err(_) => Err(refused( - "Unavailable", - format!("the daemon did not answer within {}s", WAIT.as_secs()), - )), - } -} - -/// One daemon query: dial its recorded endpoint with its keys for at most -/// [`WAIT`], then read the capacity model once. -fn probe( - home: &Path, - sandbox: &SandboxRecord, - endpoint: SocketAddr, -) -> Result { - let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name()))? - .ok_or_else(|| refused("NoSuchSandbox", "the Sandbox has no daemon keys"))?; - let dial = layerfs_sandbox::daemon::pool::Dial { - endpoint, - secret: secrets::key(&keys, "secret")?, - peer: secrets::key(&keys, "peer")?, - instance: secrets::key(&keys, "instance")?, - }; - let mut control = dial.open_within(WAIT)?; - operator::call(&mut control, OPERATION, SdkValue::Null).map_err(|error| match error { - layerfs_sandbox::SdkError::Refused(refusal) => refusal, - layerfs_sandbox::SdkError::Operation(failure) => { - layerfs_sandbox::daemon::values::refusal(&failure).unwrap_or_else(|| failed(failure)) - } - layerfs_sandbox::SdkError::Mount(failure) => failed(failure), - }) -} - /// A daemon that did not answer: a warning naming its endpoint and the /// operation. An exhausted pool is reported from the refusal's admission /// notice, which names its resource and vacant slots; the daemon never diff --git a/control/crates/layerfs-sandbox/src/daemon/capacity.rs b/control/crates/layerfs-sandbox/src/daemon/capacity.rs new file mode 100644 index 0000000000..12fa70907b --- /dev/null +++ b/control/crates/layerfs-sandbox/src/daemon/capacity.rs @@ -0,0 +1,90 @@ +//! A running Sandbox daemon's control-capacity read model (#540): its +//! `backend.daemon.connection_failures` answer, read once, best-effort and +//! bounded. The doctor explains it and `sandbox.get` shows it as +//! `daemon_capacity`; both ask through here so one place decides where the +//! daemon is, how long an answer may take, and what an unanswered query is. +use crate::daemon::{operator, pool::Dial}; +use crate::direct; +use crate::record::SandboxRecord; +use layerfs_bridge::control::{CapacityModel, SdkRefusal}; +use layerfs_home::home::paths; +use layerfs_home::outcome::{failed, refused}; +use layerfs_home::secrets; +use std::net::SocketAddr; +use std::path::Path; +use std::sync::mpsc; +use std::time::Duration; + +/// The daemon's operator read model of control admission and retained +/// connection failures. +pub const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; + +/// How long one daemon query may take: connecting, its Hello and its one +/// exchange. Short on purpose: a reader must not stall behind a daemon that +/// is unreachable, as an ordinary 60 s admission wait would. +pub const WAIT: Duration = Duration::from_secs(2); + +/// The address of a running Sandbox's daemon when its record names one to +/// this host: a forwarded machine's forward, else the endpoint the start +/// recorded in `control.json`. Nothing is observed or started. +pub fn endpoint(home: &Path, sandbox: &SandboxRecord) -> Option { + if let Some(forward) = sandbox.daemon().and_then(direct::forward_of) { + return Some(forward); + } + let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name())) + .ok() + .flatten()?; + keys.get_text("endpoint")?.parse().ok() +} + +/// [`query`] under a hard wall: a daemon that accepts a connection and then +/// says nothing must not hold the caller past [`WAIT`]. +pub fn read( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let home = home.to_owned(); + let sandbox = sandbox.clone(); + let (sent, received) = mpsc::channel(); + std::thread::spawn(move || { + let _ = sent.send(query(&home, &sandbox, endpoint)); + }); + match received.recv_timeout(WAIT) { + Ok(result) => result.map(|value| CapacityModel::parse(&value)), + Err(_) => Err(refused( + "Unavailable", + format!("the daemon did not answer within {}s", WAIT.as_secs()), + )), + } +} + +/// One daemon query: dial its recorded endpoint with its keys for at most +/// [`WAIT`], then read the capacity model once. +fn query( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name()))? + .ok_or_else(|| refused("NoSuchSandbox", "the Sandbox has no daemon keys"))?; + let dial = Dial { + endpoint, + secret: secrets::key(&keys, "secret")?, + peer: secrets::key(&keys, "peer")?, + instance: secrets::key(&keys, "instance")?, + }; + let mut control = dial.open_within(WAIT)?; + operator::call( + &mut control, + OPERATION, + layerfs_bridge::control::SdkValue::Null, + ) + .map_err(|error| match error { + crate::SdkError::Refused(refusal) => refusal, + crate::SdkError::Operation(failure) => { + crate::daemon::values::refusal(&failure).unwrap_or_else(|| failed(failure)) + } + crate::SdkError::Mount(failure) => failed(failure), + }) +} diff --git a/control/crates/layerfs-sandbox/src/daemon/mod.rs b/control/crates/layerfs-sandbox/src/daemon/mod.rs index 93718cd2d4..137668a150 100644 --- a/control/crates/layerfs-sandbox/src/daemon/mod.rs +++ b/control/crates/layerfs-sandbox/src/daemon/mod.rs @@ -3,6 +3,8 @@ #[doc(hidden)] pub mod call; #[doc(hidden)] +pub mod capacity; +#[doc(hidden)] pub mod channel; #[doc(hidden)] pub mod failure; From 8978c2eb41838c3c52b462b79696da232df3e0f5 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:51:42 +0800 Subject: [PATCH 2/6] test(sandbox): cover the capacity notice a full pool's next client reads (#540) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A daemon whose pool has no vacant slot answers the next connection with its advisory notice before any handshake. This pins what a client makes of it: `Dial::open_within` refuses `Capacity` on resource `daemon_control_slot` with the notice's vacant-slot and retained counts, retryable because nothing was attempted; a daemon that is still starting is `Unavailable` with admission `starting` instead, so a full pool and a starting one stay distinguishable. Measured against a real macOS daemon before this test existed (LP543-PATCH §4): with eight admitted clients holding every pool slot, the ninth raw connection read `notice version=1 reason=NoPoolSlot retained=0 pool_vacant=0 reserved_vacant=1`. Tests: layerfs-sandbox --test pool_admission 2 passed. Clippy -D warnings clean. Production LOC: 157011 -> 157011 (delta 0); a test-only commit. --- .../layerfs-sandbox/tests/pool_admission.rs | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 control/crates/layerfs-sandbox/tests/pool_admission.rs diff --git a/control/crates/layerfs-sandbox/tests/pool_admission.rs b/control/crates/layerfs-sandbox/tests/pool_admission.rs new file mode 100644 index 0000000000..cbd9429ffd --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/pool_admission.rs @@ -0,0 +1,82 @@ +//! A daemon's advisory capacity notice (#540) as a client reads it: a +//! daemon whose pool has no vacant slot answers the next connection with +//! the notice before any handshake, so the refusal is `Capacity` naming the +//! resource, the vacant slots and the retained failure count, not a bare +//! EOF. Nothing is sent by the client and nothing is retried, so the caller +//! may try again. A daemon that is still starting says so the same way. +use layerfs_bridge::duplex; +use layerfs_bridge::duplex::admission::{self, AdmissionNotice, AdmissionReason}; +use layerfs_sandbox::daemon::pool::Dial; +use std::io::Read; +use std::net::{SocketAddr, TcpListener}; +use std::time::Duration; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// A daemon that refuses every connection with `notice` before any +/// handshake, exactly as an accept loop does when no slot is vacant: it +/// writes the notice, shuts its write end and drains what the client sent. +fn refusing(notice: AdmissionNotice) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let _ = stream.set_write_timeout(Some(Duration::from_millis(100))); + let _ = stream.set_read_timeout(Some(Duration::from_millis(50))); + if admission::write(&mut stream, ¬ice).is_err() { + continue; + } + let mut sink = [0u8; 512]; + let _ = stream.read(&mut sink); + } + }); + endpoint +} + +fn dial(endpoint: SocketAddr) -> Dial { + Dial { + endpoint, + secret: CLIENT_PRIVATE, + peer: duplex::public_key(&DAEMON_PRIVATE).unwrap(), + instance: DAEMON_INSTANCE, + } +} + +#[test] +fn a_full_pool_refuses_the_next_client_with_its_counts() { + let endpoint = refusing(AdmissionNotice::new(AdmissionReason::NoPoolSlot, 3, 0, 1)); + let refusal = dial(endpoint) + .open_within(Duration::from_secs(2)) + .err() + .expect("a full pool admits nothing"); + assert_eq!(refusal.code, "Capacity", "{refusal:?}"); + let detail = &refusal.detail; + assert_eq!(detail.get_text("resource"), Some("daemon_control_slot")); + assert_eq!(detail.get_int("pool_vacant"), Some(0)); + assert_eq!(detail.get_int("reserved_vacant"), Some(1)); + assert_eq!(detail.get_int("retained"), Some(3)); + assert_eq!(detail.get_text("admission"), Some("no-pool-slot")); + // nothing was attempted, so a later open is safe + assert!(detail.get_bool("retryable")); + assert!( + refusal.message.contains("no vacant control slot"), + "{refusal:?}" + ); + assert!(refusal.message.contains("3 retained"), "{refusal:?}"); +} + +#[test] +fn a_still_starting_daemon_is_not_a_full_pool() { + let endpoint = refusing(AdmissionNotice::new(AdmissionReason::Starting, 0, 8, 1)); + let refusal = dial(endpoint) + .open_within(Duration::from_secs(2)) + .err() + .expect("a starting daemon admits nothing"); + // its own code and admission, not Capacity: a full pool it is not + assert_eq!(refusal.code, "Unavailable", "{refusal:?}"); + assert_eq!(refusal.detail.get_text("admission"), Some("starting")); + assert!(refusal.detail.get_bool("retryable")); +} From 88c88fbecdb89cd02d696851797aa04edbded01d Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:03:22 +0800 Subject: [PATCH 3/6] fix(sandbox): restore the daemon capacity read in sandbox.get (#543) `sandbox.get` shows `daemon_capacity`, the daemon's `backend.daemon.connection_failures` read model, but only through the half of it that survived the `layerfs_v2_0_0` merge (82165e666): the API layer parsed an endpoint and dialed it itself. That half reaches neither a `docker` nor an `apple-container` Sandbox, whose daemon is behind a machine's worker; it dials a forwarded Sandbox's old port after its machine stopped or its forward exited because it skips `direct::check`; and it opened a second control connection per read, so one `sandbox.get` took two of the daemon's slots and, with seven slots held by other clients, reported no capacity at all - the field the doctor's own fix text tells an operator to read. This restores the operation author's design from 7cfe4460e, 52f9c1666 and 31375c942 (all present before that merge, lost when it resolved #540's conflicts): the read is a host operation again (`Call::Capacity`, `backend.daemon.connection_failures`), routed like the Workspace list it follows - `direct::capacity` for a forwarded Sandbox, its machine's worker for the rest - so `native`, `lima`, `docker` and `apple-container` all answer it, and `api/crates/layerfs-api/src/sandbox/get.rs` is a thin shell over the typed `Got` again. It is asked only for a `running` Sandbox whose machine and forward were reached, on the connection the Workspace list lent from the Sandbox's pool (one connection, one slot), within 2 s for admission and for the exchange, and fenced at that deadline (`daemon/fence.rs`, the pattern the doctor's bounded observation uses): a daemon that trickles its handshake or its reply is shut down rather than left holding a thread, a connection or a slot. Doctor behavior and wording are unchanged. Its own query now ends its control session instead of dropping the socket, which the daemon logged as an unexpected end. Docs: `docs/architecture/api/sdk/sandbox/get.md`, `docs/architecture/api/sdk/sandbox/README.md`, `docs/architecture/control/routing/daemon-channel.md`. Production LOC: 157011 -> 157130 (delta +119) Scope and method: first-party production Rust/SQL/JavaScript in core, engine, control and api (tests, fixtures, tools and docs excluded), counted with `tools/production_loc.py` at the parent commit 8978c2eb4 and the committed tree. control 38406 -> 38524 (+118), api 4975 -> 4976 (+1); core 39749 and engine 73881 unchanged. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- api/crates/layerfs-api/src/sandbox/get.rs | 40 ++++----- .../layerfs-sandbox/src/daemon/capacity.rs | 88 +++++++++++++------ .../layerfs-sandbox/src/daemon/channel.rs | 8 +- .../layerfs-sandbox/src/daemon/fence.rs | 81 +++++++++++++++++ .../crates/layerfs-sandbox/src/daemon/mod.rs | 2 + .../crates/layerfs-sandbox/src/daemon/pool.rs | 28 +++--- control/crates/layerfs-sandbox/src/direct.rs | 20 +++-- .../crates/layerfs-sandbox/src/dispatch.rs | 12 +++ control/crates/layerfs-sandbox/src/get.rs | 39 +++++++- .../crates/layerfs-sandbox/src/host/call.rs | 10 ++- .../crates/layerfs-sandbox/src/host/frame.rs | 3 + docs/architecture/api/sdk/sandbox/README.md | 8 +- docs/architecture/api/sdk/sandbox/get.md | 28 +++++- .../control/routing/daemon-channel.md | 2 +- 14 files changed, 294 insertions(+), 75 deletions(-) create mode 100644 control/crates/layerfs-sandbox/src/daemon/fence.rs diff --git a/api/crates/layerfs-api/src/sandbox/get.rs b/api/crates/layerfs-api/src/sandbox/get.rs index 6d46def847..2839ca84b8 100644 --- a/api/crates/layerfs-api/src/sandbox/get.rs +++ b/api/crates/layerfs-api/src/sandbox/get.rs @@ -5,8 +5,7 @@ use crate::calls::Context; use layerfs_bridge::control::SdkValue; use layerfs_home::home::with_field; use layerfs_home::outcome::{Outcome, View}; -use layerfs_sandbox::daemon::capacity; -use layerfs_sandbox::family::{get, GetRequest, Page}; +use layerfs_sandbox::family::{get, GetRequest, Got, Page}; /// Its table entry. pub(crate) const CALL: Call = call( @@ -18,11 +17,11 @@ pub(crate) const CALL: Call = call( run, ); -/// Runs `sandbox.get`: its record with its derived `state`, one page of -/// its `workspaces` (each with its `path`, `name` and `ws`), why its -/// machine did not answer (`down_reason`), the daemon's -/// `daemon_capacity` read model when it answers within -/// [`capacity::WAIT`] (#540), and `next_page_pos`. +/// Runs `sandbox.get`: its record with its derived `state`, one page of its +/// `workspaces` (each with its `path`, `name` and `ws`), why its machine did +/// not answer (`down_reason`), the daemon's `daemon_capacity` read model when +/// the Sandbox answered it (#540), and `next_page_pos`. The library reads it +/// best effort and bounded, so this layer only shows what it got. fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { let request = GetRequest { name: args.get_text("name").unwrap_or("").to_owned(), @@ -32,29 +31,28 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome { }, }; let (home, events) = context.parts(); - let got = get(home, events, &request)?; - // best-effort and bounded: a daemon that does not answer (stopped, gone, - // at no known endpoint, or holding no vacant control slot) leaves the - // field out and never fails the read - let daemon_capacity = match got.state { - "running" => capacity::endpoint(home, &got.record) - .and_then(|endpoint| capacity::read(home, &got.record, endpoint).ok()), - _ => None, - }; - let workspaces = got.workspaces.into_iter().map(|listed| { + let Got { + record, + state, + workspaces, + down_reason, + next, + daemon_capacity, + } = get(home, events, &request)?; + let workspaces = workspaces.into_iter().map(|listed| { let item = with_field(listed.item, "path", SdkValue::Text(listed.path)); let item = with_field(item, "name", SdkValue::opt_text(listed.name)); with_field(item, "ws", SdkValue::Text(listed.ws)) }); - let shown = got.record.to_sdk(View::Full); - let shown = with_field(shown, "state", SdkValue::text(got.state)); + let shown = record.to_sdk(View::Full); + let shown = with_field(shown, "state", SdkValue::text(state)); let mut shown = with_field(shown, "workspaces", SdkValue::List(workspaces.collect())); - if let Some(why) = got.down_reason { + if let Some(why) = down_reason { shown = with_field(shown, "down_reason", SdkValue::Text(why)); } if let Some(model) = daemon_capacity { shown = with_field(shown, "daemon_capacity", model.to_sdk()); } - let next = super::list::next_page_pos(got.next); + let next = super::list::next_page_pos(next); Ok(with_field(shown, "next_page_pos", next)) } diff --git a/control/crates/layerfs-sandbox/src/daemon/capacity.rs b/control/crates/layerfs-sandbox/src/daemon/capacity.rs index 12fa70907b..4bf1a3f1a1 100644 --- a/control/crates/layerfs-sandbox/src/daemon/capacity.rs +++ b/control/crates/layerfs-sandbox/src/daemon/capacity.rs @@ -3,17 +3,24 @@ //! bounded. The doctor explains it and `sandbox.get` shows it as //! `daemon_capacity`; both ask through here so one place decides where the //! daemon is, how long an answer may take, and what an unanswered query is. -use crate::daemon::{operator, pool::Dial}; +//! +//! Both reads end at [`WAIT`], and their connection is fenced at that +//! deadline, so a daemon that trickles its handshake or its one reply keeps +//! neither a thread, a connection nor a control slot past it. +use crate::daemon::fence::Fence; +use crate::daemon::pool::{Controls, Dial}; +use crate::daemon::{operator, trace}; use crate::direct; +use crate::internal::Control; use crate::record::SandboxRecord; -use layerfs_bridge::control::{CapacityModel, SdkRefusal}; +use layerfs_bridge::control::{CapacityModel, Request, SdkRefusal, SdkValue}; use layerfs_home::home::paths; use layerfs_home::outcome::{failed, refused}; use layerfs_home::secrets; use std::net::SocketAddr; use std::path::Path; use std::sync::mpsc; -use std::time::Duration; +use std::time::{Duration, Instant}; /// The daemon's operator read model of control admission and retained /// connection failures. @@ -24,6 +31,33 @@ pub const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION; /// is unreachable, as an ordinary 60 s admission wait would. pub const WAIT: Duration = Duration::from_secs(2); +/// The model [`read`] gives: the daemon's answer, parsed once. +pub fn read( + home: &Path, + sandbox: &SandboxRecord, + endpoint: SocketAddr, +) -> Result { + let home = home.to_owned(); + let sandbox = sandbox.clone(); + bounded(WAIT, move || query(&home, &sandbox, endpoint)) + .map(|answer| CapacityModel::parse(&answer)) +} + +/// The daemon's answer to [`OPERATION`] on a connection the Sandbox's pool +/// lends (`sandbox.get`, #543): at most [`WAIT`] for admission and for the +/// exchange, which is fenced at the same deadline, so this read takes no +/// second connection and no second slot of its own. Nothing is sent again, +/// and the connection goes back to the pool (or is closed, when the fence +/// quarantined it). +pub fn lent(controls: &Controls) -> Result { + let deadline = Instant::now() + WAIT; + let mut control = trace::control_within(controls, WAIT)?; + let fence = Fence::channel((control.close_handle().map_err(failed))?, deadline); + let answer = exchange(&mut control); + drop(fence); + answer +} + /// The address of a running Sandbox's daemon when its record names one to /// this host: a forwarded machine's forward, else the endpoint the start /// recorded in `control.json`. Nothing is observed or started. @@ -38,34 +72,33 @@ pub fn endpoint(home: &Path, sandbox: &SandboxRecord) -> Option { } /// [`query`] under a hard wall: a daemon that accepts a connection and then -/// says nothing must not hold the caller past [`WAIT`]. -pub fn read( - home: &Path, - sandbox: &SandboxRecord, - endpoint: SocketAddr, -) -> Result { - let home = home.to_owned(); - let sandbox = sandbox.clone(); +/// says nothing must not hold the caller past [`WAIT`], even where the fence +/// itself cannot be armed. +fn bounded( + within: Duration, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result { let (sent, received) = mpsc::channel(); std::thread::spawn(move || { - let _ = sent.send(query(&home, &sandbox, endpoint)); + let _ = sent.send(work()); }); - match received.recv_timeout(WAIT) { - Ok(result) => result.map(|value| CapacityModel::parse(&value)), + match received.recv_timeout(within) { + Ok(result) => result, Err(_) => Err(refused( "Unavailable", - format!("the daemon did not answer within {}s", WAIT.as_secs()), + format!("the daemon did not answer within {}s", within.as_secs()), )), } } -/// One daemon query: dial its recorded endpoint with its keys for at most -/// [`WAIT`], then read the capacity model once. +/// One daemon query on a connection dialed for it: its recorded endpoint with +/// its keys, for at most [`WAIT`], fenced at the same deadline, then read the +/// capacity model once. fn query( home: &Path, sandbox: &SandboxRecord, endpoint: SocketAddr, -) -> Result { +) -> Result { let keys = secrets::daemon_keys(&paths::sandbox_dir(home, sandbox.name()))? .ok_or_else(|| refused("NoSuchSandbox", "the Sandbox has no daemon keys"))?; let dial = Dial { @@ -74,13 +107,18 @@ fn query( peer: secrets::key(&keys, "peer")?, instance: secrets::key(&keys, "instance")?, }; - let mut control = dial.open_within(WAIT)?; - operator::call( - &mut control, - OPERATION, - layerfs_bridge::control::SdkValue::Null, - ) - .map_err(|error| match error { + let (mut control, fence) = dial.open_within(WAIT)?; + let answer = exchange(&mut control); + // this conversation ends here: a connection dropped instead leaves the + // daemon reading a closed socket, which it logs as an unexpected end + let _ = control.call(Request::EndSession); + drop(fence); + answer +} + +/// The one exchange: the daemon's answer, or the refusal it gave, kept exact. +fn exchange(control: &mut Control) -> Result { + operator::call(control, OPERATION, SdkValue::Null).map_err(|error| match error { crate::SdkError::Refused(refusal) => refusal, crate::SdkError::Operation(failure) => { crate::daemon::values::refusal(&failure).unwrap_or_else(|| failed(failure)) diff --git a/control/crates/layerfs-sandbox/src/daemon/channel.rs b/control/crates/layerfs-sandbox/src/daemon/channel.rs index 9853146c60..eabe0636e3 100644 --- a/control/crates/layerfs-sandbox/src/daemon/channel.rs +++ b/control/crates/layerfs-sandbox/src/daemon/channel.rs @@ -1,7 +1,7 @@ //! One attempted authenticated control exchange, retaining the original unknown. use layerfs_bridge::{ control::{Answer, Call, ControlError, Reply, Request, SdkValue}, - duplex::{ChannelError, ChannelWork, Connection}, + duplex::{ChannelError, ChannelWork, CloseHandle, Connection}, }; use std::fmt; /// One sequential control connection. Concurrent callers may use separate channels. @@ -243,6 +243,12 @@ impl Control { pub fn clear_io_waits(&self) -> Result<(), ChannelError> { self.connection.clear_io_waits() } + /// A clone of this channel's socket for an application-owned fence + /// (`daemon::fence`): it carries no send or replay capability, and + /// shutting it down quarantines the channel. + pub fn close_handle(&self) -> Result { + self.connection.close_handle() + } /// Actual cumulative duplex send/receive work for this connection. pub fn channel_work(&self) -> (ChannelWork, ChannelWork) { (self.connection.send.work(), self.connection.receive.work()) diff --git a/control/crates/layerfs-sandbox/src/daemon/fence.rs b/control/crates/layerfs-sandbox/src/daemon/fence.rs new file mode 100644 index 0000000000..bfb2b9e735 --- /dev/null +++ b/control/crates/layerfs-sandbox/src/daemon/fence.rs @@ -0,0 +1,81 @@ +//! One deadline a control connection cannot outlive (#540, #543). +//! +//! A bounded read that only sets socket timeouts bounds each read and write +//! system call, not the exchange: a daemon that trickles one byte just +//! inside the timeout can hold the reader, its connection and its pool slot +//! far past the time the caller promised. A fence instead holds a clone of +//! the connection's socket and shuts both directions down at one absolute +//! deadline, which ends the blocked call immediately. The doctor's bounded +//! observation (`doctor/sandbox_channel.rs` outside this crate) uses the same +//! pattern. +use layerfs_bridge::duplex::CloseHandle; +use std::net::{Shutdown, TcpStream}; +use std::sync::mpsc; +use std::time::Instant; + +/// What this fence ends at its deadline: a socket cloned from a connection +/// being opened, or the application-owned close handle of an established +/// channel. +enum End { + Socket(TcpStream), + Channel(CloseHandle), +} + +impl End { + fn shut(&self) { + match self { + Self::Socket(socket) => { + let _ = socket.shutdown(Shutdown::Both); + } + // the handle quarantines the channel too, so the pool closes the + // connection instead of lending it again + Self::Channel(handle) => { + let _ = handle.close(); + } + } + } +} + +/// A socket fence, armed at one absolute deadline; its `Drop` cancels the +/// deadline and joins the thread that waits for it. +pub struct Fence { + stop: mpsc::Sender<()>, + thread: Option>, +} + +impl Fence { + /// Arms `socket` (a clone of the connection's) at `deadline`. + pub fn on(socket: TcpStream, deadline: Instant) -> Self { + Self::armed(End::Socket(socket), deadline) + } + + /// Arms established channel `handle` at `deadline`. + pub fn channel(handle: CloseHandle, deadline: Instant) -> Self { + Self::armed(End::Channel(handle), deadline) + } + + fn armed(end: End, deadline: Instant) -> Self { + let (stop, wait) = mpsc::channel(); + let thread = std::thread::spawn(move || { + if matches!( + wait.recv_timeout(deadline.saturating_duration_since(Instant::now())), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + end.shut(); + } + }); + Self { + stop, + thread: Some(thread), + } + } +} + +impl Drop for Fence { + fn drop(&mut self) { + let _ = self.stop.send(()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/control/crates/layerfs-sandbox/src/daemon/mod.rs b/control/crates/layerfs-sandbox/src/daemon/mod.rs index 137668a150..a9a3eb2000 100644 --- a/control/crates/layerfs-sandbox/src/daemon/mod.rs +++ b/control/crates/layerfs-sandbox/src/daemon/mod.rs @@ -9,6 +9,8 @@ pub mod channel; #[doc(hidden)] pub mod failure; #[doc(hidden)] +pub mod fence; +#[doc(hidden)] pub mod internal; #[doc(hidden)] pub mod operator; diff --git a/control/crates/layerfs-sandbox/src/daemon/pool.rs b/control/crates/layerfs-sandbox/src/daemon/pool.rs index b5612d6e15..4588a3aff8 100644 --- a/control/crates/layerfs-sandbox/src/daemon/pool.rs +++ b/control/crates/layerfs-sandbox/src/daemon/pool.rs @@ -11,6 +11,7 @@ //! has and callers take turns on them, each waiting at most [`LEND_WAIT`] //! before it is refused `Busy`. Opening a connection is not an attempt //! of any operation: nothing is sent before the connection is ready. +use crate::daemon::fence::Fence; use crate::internal::Control; use layerfs_bridge::control::{DaemonPhase, HelloRequest, Reply, Request, SdkRefusal, SdkValue}; use layerfs_bridge::duplex::{ @@ -196,12 +197,14 @@ impl Dial { self.open_admitted(ADMISSION_WAIT) } - /// [`Dial::open`] within `within` (#540 doctor): connecting, the - /// handshake and its Hello must finish inside that time, and the - /// connection keeps those I/O waits for the one bounded call made on - /// it. Nothing is retried and no admission wait applies: a daemon that - /// refuses or does not answer is reported as it came. - pub fn open_within(&self, within: Duration) -> Result { + /// [`Dial::open_within`], with the fence that ends this connection at + /// the same deadline: connecting, the handshake and its Hello must finish + /// inside that time, and a daemon that trickles any of them is shut down + /// at `deadline` rather than kept waiting. The connection keeps those I/O + /// waits for the one bounded call made on it. Nothing is retried and no + /// admission wait applies: a daemon that refuses or does not answer is + /// reported as it came. + pub fn open_within(&self, within: Duration) -> Result<(Control, Fence), SdkRefusal> { let deadline = Instant::now() + within; let remaining = || { deadline @@ -214,6 +217,9 @@ impl Dial { stream .set_write_timeout(Some(remaining())) .map_err(failed)?; + // armed before the handshake: a trickling far side cannot outlive the + // deadline this open promised, and neither can the one call made on it + let fence = Fence::on(stream.try_clone().map_err(failed)?, deadline); let connection = match initiate_observed(stream, &self.secret, self.peer).result { Ok(connection) => connection, Err(ChannelError::Admission(notice)) => return Err(notice_refusal(notice)), @@ -221,14 +227,16 @@ impl Dial { }; // the waits stay: the caller makes exactly one call and drops it let mut control = Control::new(connection); - match control + let hello = control .call(Request::Hello(HelloRequest { expected_instance: Some(self.instance), wait_for_store: false, })) - .map_err(failed)? - { - Reply::Hello(status) if status.phase == DaemonPhase::ControlReady => Ok(control), + .map_err(failed)?; + match hello { + Reply::Hello(status) if status.phase == DaemonPhase::ControlReady => { + Ok((control, fence)) + } _ => Err(refused("StaleSandbox", "saved daemon is not ControlReady")), } } diff --git a/control/crates/layerfs-sandbox/src/direct.rs b/control/crates/layerfs-sandbox/src/direct.rs index 280f7ef3de..f6188b3306 100644 --- a/control/crates/layerfs-sandbox/src/direct.rs +++ b/control/crates/layerfs-sandbox/src/direct.rs @@ -285,22 +285,26 @@ pub fn host(home: &Path, record: &SandboxRecord, target: &Target, message: Messa /// `sandbox.get`'s Workspace list of forwarded Sandbox `record`: a daemon /// that cannot be reached is [`Unreached`], which the get reports as down. pub fn list(home: &Path, record: &SandboxRecord) -> std::result::Result { - match list_workspaces(home, record) { + match served(home, record, Call::ListWorkspaces { path: None }) { Err(refusal) if refusal.code == "Unavailable" => Err(Unreached(refusal)), other => Ok(other), } } -/// `sandbox.list_workspaces` on forwarded Sandbox `record` in this process, -/// under the Store's lease (the Sandbox half of `workspace::direct::call`). -fn list_workspaces(home: &Path, record: &SandboxRecord) -> Outcome { +/// The daemon's control-capacity read model of forwarded Sandbox `record` +/// (#540), on the connection the Workspace list lent from this process's +/// pool for it (#543). A refused or unanswered read is the caller's to omit. +pub fn capacity(home: &Path, record: &SandboxRecord) -> Outcome { + served(home, record, Call::Capacity) +} + +/// Host `call` on forwarded Sandbox `record` in this process, under the +/// Store's lease (the Sandbox half of `workspace::direct::call`). +fn served(home: &Path, record: &SandboxRecord, call: Call) -> Outcome { let home = home.to_path_buf(); let target = resolve::target(&home, record.project().unwrap_or(""))?; let _lease = lease::hold(&home, &target.store)?; - dispatch::call( - host(&home, record, &target, Message::Absent), - Call::ListWorkspaces { path: None }, - ) + dispatch::call(host(&home, record, &target, Message::Absent), call) } /// The Workspaces forwarded Sandbox `record` holds, which its close sends diff --git a/control/crates/layerfs-sandbox/src/dispatch.rs b/control/crates/layerfs-sandbox/src/dispatch.rs index 09f3a68ccf..f610b3bf54 100644 --- a/control/crates/layerfs-sandbox/src/dispatch.rs +++ b/control/crates/layerfs-sandbox/src/dispatch.rs @@ -19,6 +19,9 @@ pub fn call(host: Host, call: Call) -> Outcome { let directory = crate::files::sandbox_dir(&host)?; listed(daemon_backend(&host, &call)?, &directory) } + // the operator view of control admission and retained failures + // (#540): a bounded read model, never a product operation + Call::Capacity => capacity(&host), Call::Create { platform } => crate::lifecycle::create(&host, platform.as_deref()), Call::Close { force, held } => { crate::lifecycle::close(&host, force, held.as_deref().unwrap_or_default()) @@ -103,6 +106,15 @@ fn daemon_backend(host: &Host, call: &Call) -> Outcome { }) } +/// The daemon's `backend.daemon.connection_failures` read model (#540): the +/// connection the Workspace list lent from the Sandbox's pool, under one +/// bound for admission and the exchange (#543). Nothing starts: the caller +/// asks only a Sandbox this process still reaches. +fn capacity(host: &Host) -> Outcome { + let state = registry::sandbox_live(host)?; + crate::daemon::capacity::lent(&state.controls) +} + /// Refuses an invalid Commit `message` before any effect. pub fn checked(message: &Message) -> Result<(), layerfs_bridge::control::SdkRefusal> { match message { diff --git a/control/crates/layerfs-sandbox/src/get.rs b/control/crates/layerfs-sandbox/src/get.rs index 3dbb7b4445..cb8485bd77 100644 --- a/control/crates/layerfs-sandbox/src/get.rs +++ b/control/crates/layerfs-sandbox/src/get.rs @@ -4,7 +4,7 @@ use super::family::{page, sandbox_of, Page}; use super::record::SandboxRecord; use crate::host::Call; use crate::route::reach; -use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_bridge::control::{CapacityModel, SdkRefusal, SdkValue}; use layerfs_home::events::Events; use layerfs_machine::worker::Unreached; use std::path::Path; @@ -44,6 +44,9 @@ pub struct Got { pub down_reason: Option, /// The next page's position; `None` after the last. pub next: Option, + /// Its daemon's control-capacity read model (#540), when a running + /// Sandbox answered the bounded read (#543). + pub daemon_capacity: Option, } /// Its record with one page of its Workspaces. Reading never clears an @@ -80,15 +83,47 @@ pub fn get(home: &Path, events: &mut Events<'_>, request: &GetRequest) -> Result Err(Unreached(why)) => (Vec::new(), Some(why.message), false), }; let (workspaces, next) = page(all, request.page); + let state = record.state(gone); + // best effort, bounded, and only for a Sandbox its daemon still serves: + // this read never fails a get (#540, #543) + let down = down_reason.is_some(); + let daemon_capacity = capacity(&home, events, &record, gone, state, down); Ok(Got { - state: record.state(gone), + state, record, workspaces, down_reason, next, + daemon_capacity, }) } +/// The daemon's `backend.daemon.connection_failures` read model of a Sandbox +/// this process still reaches (#540): read once, best effort, and omitted +/// when the Sandbox is gone, is not running, or its machine or forward is +/// already down — a get dials nothing for one of those (#543). +fn capacity( + home: &Path, + events: &mut Events<'_>, + record: &SandboxRecord, + gone: bool, + state: &str, + down: bool, +) -> Option { + if gone || down || state != "running" { + return None; + } + record.id()?; + // the forwarded daemon is dialed from here, its machine's from its worker: + // either route reads it with the connection the Workspace list lent + let answer = if direct::forwarded(record) { + direct::capacity(home, record) + } else { + reach(home, events, record, Call::Capacity, crate::dispatch::call).ok()? + }; + Some(CapacityModel::parse(&answer.ok()?)) +} + /// Workspace `item` of Sandbox `sandbox`, as its daemon answered it. fn listed_workspace(sandbox: &str, item: &SdkValue) -> ListedWorkspace { let path = (item.get_text("path")) diff --git a/control/crates/layerfs-sandbox/src/host/call.rs b/control/crates/layerfs-sandbox/src/host/call.rs index f3263bcb67..92ced5f1d5 100644 --- a/control/crates/layerfs-sandbox/src/host/call.rs +++ b/control/crates/layerfs-sandbox/src/host/call.rs @@ -19,6 +19,9 @@ pub enum Op { SandboxClose, /// `sandbox.list_workspaces`: its daemon's Workspace list. ListWorkspaces, + /// `backend.daemon.connection_failures`: its daemon's control-capacity + /// read model (#540), a bounded operator read. + Capacity, /// `workspace.mount`. Mount, /// `workspace.exec`. @@ -35,10 +38,11 @@ pub enum Op { /// Every host operation and the name a request carries (`op`). A /// machine's worker runs exactly these. -pub const OPS: [(Op, &str); 9] = [ +pub const OPS: [(Op, &str); 10] = [ (Op::SandboxCreate, "sandbox.create"), (Op::SandboxClose, "sandbox.close"), (Op::ListWorkspaces, "sandbox.list_workspaces"), + (Op::Capacity, layerfs_bridge::control::CAPACITY_OPERATION), (Op::Mount, "workspace.mount"), (Op::Exec, "workspace.exec"), (Op::ExecCancel, "workspace.exec_cancel"), @@ -100,6 +104,9 @@ pub enum Call { }, /// `sandbox.list_workspaces`, with the Workspace `path` it may name. ListWorkspaces { path: Option }, + /// `backend.daemon.connection_failures`: its daemon's control-capacity + /// read model (#540). + Capacity, /// `workspace.mount(branch, { name })`. Mount { branch: Option, @@ -149,6 +156,7 @@ impl Call { Self::Create { .. } => Op::SandboxCreate, Self::Close { .. } => Op::SandboxClose, Self::ListWorkspaces { .. } => Op::ListWorkspaces, + Self::Capacity => Op::Capacity, Self::Mount { .. } => Op::Mount, Self::Exec { .. } => Op::Exec, Self::ExecCancel { .. } => Op::ExecCancel, diff --git a/control/crates/layerfs-sandbox/src/host/frame.rs b/control/crates/layerfs-sandbox/src/host/frame.rs index afce72800c..6848360ddc 100644 --- a/control/crates/layerfs-sandbox/src/host/frame.rs +++ b/control/crates/layerfs-sandbox/src/host/frame.rs @@ -81,6 +81,8 @@ fn fields(call: &Call, sent: &Message, sandbox: Option<&str>) -> Vec<(&'static s Call::ListWorkspaces { path } | Call::Status { path } => { fields.extend(path.as_deref().map(|p| ("path", text(p)))) } + // a bounded operator read: it sends no field of its own + Call::Capacity => {} Call::Mount { branch, name } => { fields.push(("branch", opt(branch))); fields.extend(name.as_deref().map(|n| ("name", text(n)))); @@ -248,6 +250,7 @@ fn call(op: Op, more: &SdkValue) -> Call { }, }, Op::ListWorkspaces => Call::ListWorkspaces { path: text("path") }, + Op::Capacity => Call::Capacity, Op::Mount => Call::Mount { branch: text("branch"), name: text("name"), diff --git a/docs/architecture/api/sdk/sandbox/README.md b/docs/architecture/api/sdk/sandbox/README.md index 759acfeacd..d1724454f6 100644 --- a/docs/architecture/api/sdk/sandbox/README.md +++ b/docs/architecture/api/sdk/sandbox/README.md @@ -36,7 +36,7 @@ unique across the home. | --- | --- | --- | --- | --- | | [`create`](create.md) | `project` (Project), `name` (Name), `machine?` (Name) | `image` (Text) | the Sandbox record with `state`; `forward_failed` when a direct daemon forward did not open | its machine: here or `sandbox.host` in the worker | | [`list`](list.md) | `project?` (Project) | `page_size`, `page_pos` (Int) | `{ items: [record], next_page_pos }` | caller's process, home records only | -| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, and `down_reason` when its machine is down | its machine or its daemon; starts nothing | +| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing | | [`close`](close.md) | `name` (Name) | `force` (Bool) | `{ kind: "Closed", sandbox, unknown, closed }`, or `{ closed, was_gone: true }`, or `{ closed, was, left? }` for a record with no id | its machine: here or `sandbox.host` in the worker | The record is `SandboxRecord::to_sdk` @@ -47,7 +47,11 @@ its stored fields except `v`, `runtime_id` and `daemon`, plus the derived ([`op.rs`](../../../../../control/crates/layerfs-home/src/op.rs)). Each Workspace in `get` carries `ws` (`:` or `:`), `name` and `path` -([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)). +([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)), and +its `daemon_capacity` is the daemon's `backend.daemon.connection_failures` +read model, read best effort and bounded to 2 s on the connection the +Workspace list already lent +([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)). ## Arguments and options diff --git a/docs/architecture/api/sdk/sandbox/get.md b/docs/architecture/api/sdk/sandbox/get.md index 0a91d70c2b..684cff486d 100644 --- a/docs/architecture/api/sdk/sandbox/get.md +++ b/docs/architecture/api/sdk/sandbox/get.md @@ -11,8 +11,8 @@ its daemon holds, without starting anything. | --- | --- | | Family | [sandbox](README.md) | | Library entry | [`sandbox/get.rs`](../../../../../api/crates/layerfs-api/src/sandbox/get.rs) | -| Implemented in | the library entry parses a typed `GetRequest` → `layerfs_sandbox::family::get` → `get::get`, whose typed `Got` it shapes ([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)) | -| Runs in | the caller's process; the Workspace list comes from the Sandbox daemon, dialed from here (forwarded or local) or from the machine's worker | +| Implemented in | the library entry parses a typed `GetRequest` → `layerfs_sandbox::family::get` → `get::get`, whose typed `Got` (its `daemon_capacity` included) it shapes ([`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs)) | +| Runs in | the caller's process; the Workspace list and the daemon's capacity read model come from the Sandbox daemon, dialed from here (forwarded or local) or from the machine's worker | ## Signature @@ -58,7 +58,15 @@ layerfs sandbox get [--page-size ] [--page-pos ] head_moved_since_mount, dirty: {inodes, entries}?, pending_merge, revision } ], next_page_pos: int | null, - down_reason?: text } + down_reason?: text, + daemon_capacity?: { admission: { pool_vacant, pool_running, + reserved_vacant, reserved_running }, + retained: { limit, ttl_ms, count, dropped, recorded, + failures: [ { slot, class, generation, owner, + received_bytes, request, reply_sent, + cause, fence_error, + diagnostic_error, age_ms, + expires_in_ms } ] } } } ``` | Field | Kind | Meaning | @@ -70,6 +78,7 @@ layerfs sandbox get [--page-size ] [--page-pos ] | `workspaces[].*` | | the daemon's read model per Workspace: id, namespace, Branch, base Commit and Layer, root, `activity`, dirty counts ([`operator/inspect.rs`](../../../../../engine/crates/layerfs-store-service/src/operator/inspect.rs) `workspace_value`) | | `next_page_pos` | int or null | next page of `workspaces` | | `down_reason` | text | the machine's worker or the forwarded daemon was not reached; `workspaces` is then `[]` | +| `daemon_capacity` | record | the daemon's `backend.daemon.connection_failures` read model ([#540](https://github.com/Ephemeral-AI-Lab/layerfs/issues/540)): control admission by slot class and the bounded retained connection failures that used to hold a slot ([`control_capacity.rs`](../../../../../engine/crates/layerfs-bridge/src/control_capacity.rs)). It is read once, best effort, and only for a Sandbox that is `running` and whose machine and forward were reached; nothing is started and no second control slot is taken, so the field is simply absent when the daemon refuses, does not answer within 2 s, or was not asked ([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)) | ## Where it runs @@ -86,10 +95,21 @@ layerfs sandbox get [--page-size ] [--page-pos ] v Unreached / Unavailable --> record + workspaces [] + down_reason NoSuchSandbox --> state "lost", workspaces [] + + running and not down, same route and the same pooled connection: + backend.daemon.connection_failures --> daemon_capacity (2 s, best effort) ``` `get` starts no machine and loads no image (#535 P4); reading never clears -an intent. See the [routing table](../../../control/routing/routing-table.md) and the +an intent. The capacity read is a second host operation on the connection +the Workspace list lent from the Sandbox's pool +([`daemon/pool.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs)), +bounded to 2 s for admission and for +the exchange and fenced at that deadline ([`daemon/fence.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs)), +so a daemon that trickles cannot hold the slot; a forwarded daemon's forward +and its machine are checked before it is dialed (`direct::check`), so a +stopped machine is `down_reason` and never a 2 s wait on a reused port. See +the [routing table](../../../control/routing/routing-table.md) and the [daemon channel](../../../control/routing/daemon-channel.md). ## Errors diff --git a/docs/architecture/control/routing/daemon-channel.md b/docs/architecture/control/routing/daemon-channel.md index a788b0c3ad..7103015780 100644 --- a/docs/architecture/control/routing/daemon-channel.md +++ b/docs/architecture/control/routing/daemon-channel.md @@ -82,7 +82,7 @@ opening a connection sends only Hello, so it attempts no operation. | `workspace.commit` | `Commit` | `values::commit`: Committed, Merged (with `merged`), or `Conflict` | [`daemon/workspace/commit.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/commit.rs), [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | `workspace.unmount` | `Unmount{token, discard}`, or `ForceUnmount` with `force` | `values::unmounted` (`Dirty` refusal) or `force_unmounted` | [`daemon/workspace/unmount.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/unmount.rs) | | `workspace.status` | `Status(token)`; the Branch head is read separately on the Store's route | `values::status` | [`daemon/workspace/status.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/status.rs) | -| `sandbox.get` Workspace list | SDK method `sandbox.list_workspaces`, the one daemon backend method sent | `sandbox/dispatch.rs` | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs) | +| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs)) | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | | `workspace.exec` | none on this channel: `bash -c` runs through the Sandbox's runtime | — | [`exec/run.rs`](../../../../control/crates/layerfs-sandbox/src/exec/run.rs) | `Locate` and `Cleanup` are read-only observations for a caller whose From 811f59e02f6e8dbee8115ed343da337316d04728 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:03:47 +0800 Subject: [PATCH 4/6] test(sandbox): cover the capacity read's routes, wall and pool slot (#543) `sandbox.get`'s `daemon_capacity` (#540) had one test of the shown answer and one of the 2 s bound, both on a forwarded Sandbox. This covers what the restored route decides: - `api/crates/layerfs-api/tests/sandbox_capacity.rs`: the Workspace list and the capacity read share one connection (the stand-in daemon counts them), a stopped machine with no worker record, an ended forward with a recorded process, and a Sandbox with no id are never dialed and answer in well under a second, and a daemon that reads the query and never answers still returns the get at its bound. - `control/crates/layerfs-sandbox/tests/capacity_read.rs`: a query ends its control session (`EndSession`) instead of dropping the socket, a trickling daemon is fenced at the deadline (its own write fails, so the connection was shut down), two pooled reads take the one connection the pool holds, and the host operation a machine's worker runs (`dispatch::call`, `Call::Capacity`) returns the model. - `control/crates/layerfs-sandbox/tests/host_frame.rs`: the `sandbox.host` frame carries `backend.daemon.connection_failures` and reads back as `Call::Capacity`. Production LOC: 157130 -> 157130 (delta 0) Scope and method: first-party production Rust/SQL/JavaScript in core, engine, control and api (tests, fixtures, tools and docs excluded), counted with `tools/production_loc.py` at the parent commit 88c88fbec and the committed tree; this commit changes tests only, so the production total is unchanged. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../layerfs-api/tests/sandbox_capacity.rs | 183 +++++++---- .../layerfs-sandbox/tests/capacity_read.rs | 300 ++++++++++++++++++ .../layerfs-sandbox/tests/host_frame.rs | 1 + 3 files changed, 430 insertions(+), 54 deletions(-) create mode 100644 control/crates/layerfs-sandbox/tests/capacity_read.rs diff --git a/api/crates/layerfs-api/tests/sandbox_capacity.rs b/api/crates/layerfs-api/tests/sandbox_capacity.rs index b9b813adaf..feb60971a2 100644 --- a/api/crates/layerfs-api/tests/sandbox_capacity.rs +++ b/api/crates/layerfs-api/tests/sandbox_capacity.rs @@ -1,8 +1,10 @@ -//! `sandbox.get`'s `daemon_capacity` (#540): the daemon's +//! `sandbox.get`'s `daemon_capacity` (#540, #543): the daemon's //! `backend.daemon.connection_failures` read model, shown best-effort and -//! bounded the same way the doctor reads it. Nothing runs: a stand-in -//! daemon answers the Sandbox's Workspace list and its capacity model, a -//! second one answers the list and then says nothing. +//! bounded on the connection the Workspace list lent from this process's +//! pool. Nothing runs: a stand-in daemon answers the Sandbox's Workspace list +//! and its capacity model on the same connection, a second one answers the +//! list and then says nothing, and a Sandbox whose machine or forward is +//! already down is not dialed at all. #![cfg(unix)] use layerfs_api::home::{self, paths, write_record}; use layerfs_bridge::control::{ @@ -13,14 +15,21 @@ use layerfs_bridge::duplex; use std::net::{SocketAddr, TcpListener}; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; use std::time::{Duration, Instant}; const CLIENT_PRIVATE: [u8; 32] = [2; 32]; const DAEMON_PRIVATE: [u8; 32] = [1; 32]; const DAEMON_INSTANCE: [u8; 32] = [3; 32]; +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + fn tempdir(tag: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!("lfs-scap-{tag}-{}", std::process::id())); + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("lfs-scap-{tag}-{}-{sequence}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); @@ -50,24 +59,27 @@ fn no_workspaces() -> SdkValue { ]) } -/// A stand-in daemon at its own loopback port: it answers this Sandbox's -/// Workspace list, then its capacity model with `model`. With `stall` it -/// answers the Workspace list, reads the capacity query and never answers -/// it, which is what a daemon stuck behind its own slots looks like. -fn daemon(model: SdkValue, stall: bool) -> SocketAddr { +/// A stand-in daemon at its own loopback port, and how many connections it +/// served: it answers this Sandbox's Workspace list and its capacity model +/// with `model` on the connection it was given. With `stall` it answers the +/// Workspace list, reads the capacity query and never answers it, which is +/// what a daemon stuck behind its own slots looks like. +fn daemon(model: SdkValue, stall: bool) -> (SocketAddr, Arc) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let endpoint = listener.local_addr().unwrap(); - let model = std::sync::Arc::new(model); + let model = Arc::new(model); + let served = Arc::new(AtomicUsize::new(0)); + let counted = served.clone(); std::thread::spawn(move || { for stream in listener.incoming() { let Ok(stream) = stream else { return }; + counted.fetch_add(1, Ordering::SeqCst); let model = model.clone(); - // one worker per connection: a used list connection stays open - // beside the capacity query's own + // one worker per connection: the pool keeps one and reuses it std::thread::spawn(move || serve(stream, &model, stall)); } }); - endpoint + (endpoint, served) } /// One stand-in connection: its Hello, then each call's answer until the @@ -97,8 +109,8 @@ fn serve(stream: std::net::TcpStream, model: &SdkValue, stall: bool) { } Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { if stall { - // read but never answered: the caller's own bound, not a - // closed socket, ends its wait + // read but never answered: the read model's own fence, not + // a closed socket, ends its wait std::thread::sleep(Duration::from_secs(6)); break; } @@ -126,22 +138,23 @@ fn map(fields: &[(&str, SdkValue)]) -> SdkValue { ) } -/// A home with Store `shop` (project `api`) and Sandbox `s1` on `native`, -/// recorded as a started Sandbox whose daemon this host reaches through the -/// forward `daemon` names; its keys are `endpoint`'s. -fn forwarded_sandbox(home: &Path, endpoint: SocketAddr) { +/// The home's machine, Store `shop` and project `api`; the machine's `worker` +/// record only while it runs (its absence is a stopped machine, #535 D11). +fn home_records(home: &Path, worker: bool) { write_record( &paths::machine_record(home, "native"), &map(&[("name", t("native")), ("platform", t("native"))]), ) .unwrap(); - // the forward's machine answers: a forward with no recorded process is - // not checked against one - write_record( - &paths::worker_record(home, "native"), - &map(&[("address", t("127.0.0.1:1")), ("secret", t("5e"))]), - ) - .unwrap(); + if worker { + // the forward's machine answers: a forward with no recorded process is + // not checked against one + write_record( + &paths::worker_record(home, "native"), + &map(&[("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + } write_record( &paths::store_record(home, "shop"), &map(&[ @@ -159,29 +172,33 @@ fn forwarded_sandbox(home: &Path, endpoint: SocketAddr) { &map(&[("name", t("api"))]), ) .unwrap(); - write_record( - &paths::sandbox_record(home, "s1"), - &map(&[ - ("name", t("s1")), - ("project", t("shop/api")), - ("machine", t("native")), - ("id", t("s-1")), - ( - "daemon", - map(&[ - ("name", t("lfs-x-s1")), - ( - "forward", - map(&[ - ("address", t(&endpoint.to_string())), - ("machine", t("native")), - ]), - ), - ]), - ), - ]), - ) - .unwrap(); +} + +/// Sandbox `s1` on `native`, recorded with its `id` (`None`: gone, its create +/// never recorded one) whose daemon this host reaches through the forward to +/// `endpoint`, carried by forward process `pid` (`None`: a port nothing +/// carries). +fn sandbox_record(home: &Path, endpoint: SocketAddr, id: Option<&str>, pid: Option) { + let mut forward = vec![ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]; + if let Some(pid) = pid { + forward.push(("pid", SdkValue::Int(pid))); + } + let mut fields = vec![ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ]; + if let Some(id) = id { + fields.push(("id", t(id))); + } + fields.push(( + "daemon", + map(&[("name", t("lfs-x-s1")), ("forward", map(&forward))]), + )); + write_record(&paths::sandbox_record(home, "s1"), &map(&fields)).unwrap(); write_record( &paths::sandbox_control(home, "s1"), &map(&[ @@ -233,10 +250,11 @@ fn busy() -> SdkValue { #[test] fn a_running_sandbox_s_get_shows_its_daemon_capacity() { - let endpoint = daemon(busy(), false); + let (endpoint, served) = daemon(busy(), false); let home = tempdir("shown"); home::open(&home).unwrap(); - forwarded_sandbox(&home, endpoint); + home_records(&home, true); + sandbox_record(&home, endpoint, Some("s-1"), None); let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); let capacity = got @@ -263,15 +281,19 @@ fn a_running_sandbox_s_get_shows_its_daemon_capacity() { ); assert_eq!(failures[0].get_int("generation"), Some(6)); assert_eq!(failures[0].get_text("request"), Some("unmount")); + // the Workspace list and the capacity read share the one pooled + // connection (#543): a get takes no second control slot of its own + assert_eq!(served.load(Ordering::SeqCst), 1, "{got:?}"); std::fs::remove_dir_all(home).unwrap(); } #[test] fn a_daemon_that_does_not_answer_leaves_daemon_capacity_out() { - let endpoint = daemon(busy(), true); + let (endpoint, _served) = daemon(busy(), true); let home = tempdir("stalled"); home::open(&home).unwrap(); - forwarded_sandbox(&home, endpoint); + home_records(&home, true); + sandbox_record(&home, endpoint, Some("s-1"), None); let at = Instant::now(); let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); let elapsed = at.elapsed(); @@ -288,3 +310,56 @@ fn a_daemon_that_does_not_answer_leaves_daemon_capacity_out() { assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); std::fs::remove_dir_all(home).unwrap(); } + +#[test] +fn a_stopped_machine_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("stopped"); + home::open(&home).unwrap(); + // the machine records no worker: nothing starts it (#535 D11) + home_records(&home, false); + sandbox_record(&home, endpoint, Some("s-1"), None); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + assert!(got.get("down_reason").is_some(), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + // a down machine is answered from its records: no port is dialed and + // nothing waits for a daemon that is not there + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn an_ended_forward_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("forward-ended"); + home::open(&home).unwrap(); + home_records(&home, true); + // a recorded forward process that is not alive: its port may be reused by + // another process, so it is never dialed + sandbox_record(&home, endpoint, Some("s-1"), Some(3_999_999)); + let at = Instant::now(); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + let elapsed = at.elapsed(); + assert!(got.get("down_reason").is_some(), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_sandbox_with_no_id_is_never_dialed() { + let (endpoint, served) = daemon(busy(), false); + let home = tempdir("gone"); + home::open(&home).unwrap(); + home_records(&home, true); + sandbox_record(&home, endpoint, None, None); + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(got.get_text("state"), Some("gone"), "{got:?}"); + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-sandbox/tests/capacity_read.rs b/control/crates/layerfs-sandbox/tests/capacity_read.rs new file mode 100644 index 0000000000..9b5241fbfa --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/capacity_read.rs @@ -0,0 +1,300 @@ +//! The `backend.daemon.connection_failures` read model's bounds (#540, +//! #543): one query ends its control session instead of dropping the socket, +//! a daemon that trickles is fenced at the deadline, and a read on the +//! Sandbox's pool takes no second connection. +//! +//! Nothing runs: a stand-in daemon answers the query on the connections it +//! accepts, and counts what it saw. +use layerfs_bridge::control::{ + AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request, + RetainedModel, SdkValue, CAPACITY_OPERATION, +}; +use layerfs_bridge::duplex; +use layerfs_home::home::paths; +use layerfs_home::secrets; +use layerfs_sandbox::daemon::capacity; +use layerfs_sandbox::daemon::pool::{Controls, Dial}; +use layerfs_sandbox::host::{Call as HostCall, Host, Message}; +use layerfs_sandbox::record::SandboxRecord; +use std::io::Write; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const CLIENT_PRIVATE: [u8; 32] = [2; 32]; +const DAEMON_PRIVATE: [u8; 32] = [1; 32]; +const DAEMON_INSTANCE: [u8; 32] = [3; 32]; + +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = + std::env::temp_dir().join(format!("lfs-capr-{tag}-{}-{sequence}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +/// A model of a pool with no vacant slot (and no retained failure). +fn model() -> SdkValue { + CapacityModel { + admission: AdmissionModel { + pool_vacant: 0, + pool_running: 8, + reserved_vacant: 1, + reserved_running: 0, + }, + retained: RetainedModel::default(), + } + .to_sdk() +} + +fn dial(endpoint: SocketAddr) -> Dial { + Dial { + endpoint, + secret: CLIENT_PRIVATE, + peer: duplex::public_key(&DAEMON_PRIVATE).unwrap(), + instance: DAEMON_INSTANCE, + } +} + +/// What a stand-in daemon saw, and how it treats the query. +#[derive(Default)] +struct Seen { + connections: AtomicUsize, + queries: AtomicUsize, + ended: AtomicBool, + fenced: AtomicBool, +} + +/// How the stand-in answers the capacity query. +#[derive(Clone, Copy)] +enum Behavior { + /// With the model. + Model, + /// Never: it trickles one byte every `TRICKLE` until the far side closes + /// the socket (`Seen::fenced`). + Trickle, +} + +const TRICKLE: Duration = Duration::from_millis(400); + +/// A stand-in daemon at its own loopback port. +fn daemon(answer: Behavior) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let seen = Arc::new(Seen::default()); + let counted = seen.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + counted.connections.fetch_add(1, Ordering::SeqCst); + let seen = counted.clone(); + std::thread::spawn(move || serve(stream, answer, &seen)); + } + }); + (endpoint, seen) +} + +/// One stand-in connection: its Hello, then each call's answer; a query ends +/// the session when the caller sends its end. +fn serve(stream: TcpStream, answer: Behavior, seen: &Seen) { + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + // kept for a trickling daemon: ciphertext is the channel's to write + let mut raw = stream.try_clone().ok(); + let Ok(mut channel) = duplex::accept( + stream, + &DAEMON_PRIVATE, + duplex::public_key(&CLIENT_PRIVATE).unwrap(), + ) else { + return; + }; + while let Ok(bytes) = channel.receive.receive() { + let Ok(call) = Call::decode(bytes) else { break }; + let id = call.id; + let reply = match call.request { + Request::Hello(_) => Reply::Hello(DaemonStatus { + instance: DAEMON_INSTANCE, + phase: DaemonPhase::ControlReady, + overlay_sqlite: Some("3.51.0".into()), + store_sqlite: Some("3.51.0".into()), + }), + Request::EndSession => { + seen.ended.store(true, Ordering::SeqCst); + Reply::SessionEnded + } + Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => { + seen.queries.fetch_add(1, Ordering::SeqCst); + if matches!(answer, Behavior::Trickle) { + if let Some(raw) = raw.as_mut() { + trickle(raw, seen); + } + break; + } + Reply::Sdk(model()) + } + _ => continue, + }; + if channel + .send + .send(&Answer { id, reply }.encode().unwrap()) + .is_err() + { + break; + } + } +} + +/// Keeps the far side waiting without ever completing a record, until it ends +/// the connection itself. +fn trickle(raw: &mut TcpStream, seen: &Seen) { + let byte = [0u8; 1]; + for _ in 0..20 { + std::thread::sleep(TRICKLE); + if raw.write_all(&byte).and_then(|()| raw.flush()).is_err() { + seen.fenced.store(true, Ordering::SeqCst); + return; + } + } +} + +/// Sandbox `s1`'s folder in `home`, recording the daemon keys its queries +/// read (`control.json`) and, with `forward`, the record that reaches its +/// daemon through that forward. +fn sandbox_folder(home: &Path, endpoint: SocketAddr, forward: bool) { + let dir = paths::sandbox_dir(home, "s1"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + let keys = SdkValue::map([ + ("endpoint", t(&endpoint.to_string())), + ("secret", SdkValue::Bytes(CLIENT_PRIVATE.to_vec())), + ( + "peer", + SdkValue::Bytes(duplex::public_key(&DAEMON_PRIVATE).unwrap().to_vec()), + ), + ("instance", SdkValue::Bytes(DAEMON_INSTANCE.to_vec())), + ]); + secrets::save_daemon_keys(&dir, &keys).unwrap(); + if forward { + let record = SdkValue::map([ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("native")), + ("id", t("s-1")), + ( + "daemon", + SdkValue::map([( + "forward", + SdkValue::map([ + ("address", t(&endpoint.to_string())), + ("machine", t("native")), + ]), + )]), + ), + ]); + layerfs_home::home::records::write_record(&dir.join("record.json"), &record).unwrap(); + layerfs_home::home::records::write_record( + &paths::worker_record(home, "native"), + &SdkValue::map([("address", t("127.0.0.1:1")), ("secret", t("5e"))]), + ) + .unwrap(); + } +} + +fn record(name: &str) -> SandboxRecord { + SandboxRecord::from_record(name, SdkValue::map([])) +} + +#[test] +fn a_query_ends_its_control_session() { + let (endpoint, seen) = daemon(Behavior::Model); + let home = tempdir("session"); + sandbox_folder(&home, endpoint, false); + let got = capacity::read(&home, &record("s1"), endpoint).unwrap(); + assert_eq!(got.admission.pool_running, 8); + // the one conversation ended before the socket was dropped, which the + // daemon would otherwise log as an unexpected end + assert!(seen.ended.load(Ordering::SeqCst), "no EndSession was sent"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_trickling_daemon_is_fenced_at_the_deadline() { + let (endpoint, seen) = daemon(Behavior::Trickle); + let home = tempdir("trickle"); + sandbox_folder(&home, endpoint, false); + let at = Instant::now(); + let refused = capacity::read(&home, &record("s1"), endpoint).err(); + let elapsed = at.elapsed(); + assert!(refused.is_some(), "{refused:?}"); + assert!( + elapsed < capacity::WAIT + Duration::from_millis(1500), + "the read outlived its deadline: {elapsed:?}" + ); + // the socket was shut down, not merely abandoned: the daemon's own write + // failed + let until = Instant::now() + Duration::from_secs(3); + while !seen.fenced.load(Ordering::SeqCst) && Instant::now() < until { + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + seen.fenced.load(Ordering::SeqCst), + "the fence did not close the connection" + ); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_pooled_read_takes_no_second_connection() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + let one = capacity::lent(&controls).unwrap(); + let two = capacity::lent(&controls).unwrap(); + assert_eq!(CapacityModel::parse(&one).admission.pool_vacant, 0); + assert_eq!(CapacityModel::parse(&two).admission.pool_running, 8); + // both reads went over the one connection the pool holds, so a get takes + // a single control slot for the list and the capacity read together + assert_eq!(seen.connections.load(Ordering::SeqCst), 1); + assert_eq!(seen.queries.load(Ordering::SeqCst), 2); +} + +#[test] +fn the_worker_s_side_reads_the_capacity_model() { + let (endpoint, seen) = daemon(Behavior::Model); + let home = tempdir("worker"); + std::fs::create_dir_all(&home).unwrap(); + sandbox_folder(&home, endpoint, true); + // the request a machine's worker decodes (`sandbox.host`) and the host + // operation it runs: no Store is opened and no image is loaded + let host = Host { + home: home.clone(), + store: "shop".to_owned(), + project: "api".to_owned(), + sandbox: Some("s1".to_owned()), + store_record: None, + store_path: None, + deploy: None, + postgresql: None, + message: Message::Absent, + started: None, + }; + let answer = layerfs_sandbox::dispatch::call(host, HostCall::Capacity).unwrap(); + let model = CapacityModel::parse(&answer); + assert_eq!(model.admission.pool_vacant, 0, "{answer:?}"); + assert_eq!(model.admission.reserved_vacant, 1, "{answer:?}"); + assert_eq!(seen.queries.load(Ordering::SeqCst), 1); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-sandbox/tests/host_frame.rs b/control/crates/layerfs-sandbox/tests/host_frame.rs index d43a315aa9..a940d00b22 100644 --- a/control/crates/layerfs-sandbox/tests/host_frame.rs +++ b/control/crates/layerfs-sandbox/tests/host_frame.rs @@ -74,6 +74,7 @@ fn a_frame_reads_back_to_its_request() { held: None, }, Call::ListWorkspaces { path: None }, + Call::Capacity, Call::Mount { branch: Some("dev".to_owned()), name: Some("w2".to_owned()), From de009a8b87ffe146f8275ea71337fe89ce5b774d Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:09:03 +0800 Subject: [PATCH 5/6] docs(sandbox): cite #540 plainly in sandbox.get's field table (#543) The `daemon_capacity` row linked an issue URL that the guides do not have a record of; it cites the issue the way the surrounding documents do, `(#540)`. Production LOC: 157130 -> 157130 (delta 0) Scope and method: first-party production Rust/SQL/JavaScript in core, engine, control and api (tests, fixtures, tools and docs excluded), counted with `tools/production_loc.py` at the parent commit 811f59e02 and the committed tree; this commit changes one documentation line, so the production total is unchanged. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- docs/architecture/api/sdk/sandbox/get.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/api/sdk/sandbox/get.md b/docs/architecture/api/sdk/sandbox/get.md index 684cff486d..7986a6bd07 100644 --- a/docs/architecture/api/sdk/sandbox/get.md +++ b/docs/architecture/api/sdk/sandbox/get.md @@ -78,7 +78,7 @@ layerfs sandbox get [--page-size ] [--page-pos ] | `workspaces[].*` | | the daemon's read model per Workspace: id, namespace, Branch, base Commit and Layer, root, `activity`, dirty counts ([`operator/inspect.rs`](../../../../../engine/crates/layerfs-store-service/src/operator/inspect.rs) `workspace_value`) | | `next_page_pos` | int or null | next page of `workspaces` | | `down_reason` | text | the machine's worker or the forwarded daemon was not reached; `workspaces` is then `[]` | -| `daemon_capacity` | record | the daemon's `backend.daemon.connection_failures` read model ([#540](https://github.com/Ephemeral-AI-Lab/layerfs/issues/540)): control admission by slot class and the bounded retained connection failures that used to hold a slot ([`control_capacity.rs`](../../../../../engine/crates/layerfs-bridge/src/control_capacity.rs)). It is read once, best effort, and only for a Sandbox that is `running` and whose machine and forward were reached; nothing is started and no second control slot is taken, so the field is simply absent when the daemon refuses, does not answer within 2 s, or was not asked ([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)) | +| `daemon_capacity` | record | the daemon's `backend.daemon.connection_failures` read model (#540): control admission by slot class and the bounded retained connection failures that used to hold a slot ([`control_capacity.rs`](../../../../../engine/crates/layerfs-bridge/src/control_capacity.rs)). It is read once, best effort, and only for a Sandbox that is `running` and whose machine and forward were reached; nothing is started and no second control slot is taken, so the field is simply absent when the daemon refuses, does not answer within 2 s, or was not asked ([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)) | ## Where it runs From 49e54038628f499108c33c8f2aeb9beeff9a5466 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:56:06 +0800 Subject: [PATCH 6/6] fix(sandbox): keep a fenced control connection out of the pool (#543) `capacity::lent` (the `sandbox.get` capacity read, #543) arms a fence on the pool connection it just borrowed, and `Lease::drop` decides reuse from `Control::is_failed()`. That answer was only `Control.failed`, which a fence's end never sets: `CloseHandle::close` (layerfs-bridge) sets the channel's own shared `failed` bit and shuts its socket down, and the `Control` above it learned of that only when a call it sent subsequently failed. The window, in order: 1. `trace::control_within` borrows a connection and `Fence::channel` arms it at the read's deadline. 2. `exchange` returns `Ok(answer)`: `Control.failed` is still false and the channel is healthy. 3. Before `drop(fence)` sends its stop, the fence thread's deadline expires and `CloseHandle::close()` quarantines the channel and shuts both directions down. `drop(fence)` joins that thread, so the close has completed before the lease is returned. 4. `Lease::drop` asks `Control::is_failed()`, hears false, and pushes the dead connection back into `pool.idle`. 5. The next call in the same long-lived process (a machine's worker, an SDK binding) pops it. `Sender::send` refuses at `shared.ready()` with `Quarantined` before writing a byte, but `Control::call_with` had already set `attempted = true`, so the caller is told `phase: Send, attempted: true` for an operation that never left. `capacity::lent` widened that window: its deadline was taken before `control_within(controls, WAIT)`, which re-reads the clock and may use the whole 2 s, so a read admitted at the boundary armed a fence already 0 ms from firing - a shutdown racing a reply that took microseconds. Fix, at the three points the ordering above crosses: - `layerfs-bridge`: `Connection::is_quarantined()` reads the one shared bit that every direction and close handle write. - `Control::is_failed()` is that bit or `Control.failed`, and `call_with` and `install_file` refuse on it before setting `attempted`. A quarantined connection now reports `phase: Encode, attempted: false` (nothing can have taken effect), and `Lease::drop` closes it instead of lending it again. - `capacity::lent` checks the window left after admission and refuses (`Unavailable`, nothing sent) instead of arming a spent deadline, which would shut a healthy connection down for nothing. Tests (`control/crates/layerfs-sandbox/tests/capacity_read.rs`): a fenced pool connection is not lent again (the next read opens the second connection and answers instead of reporting a quarantine), and a quarantined connection's call is `Encode` with `attempted: false`. With `is_failed` reverted to its previous form both fail; the first reproduces the reviewed symptom exactly (`phase: Send, attempted: true, cause: Channel(Quarantined)`). Docs: `docs/architecture/control/routing/daemon-channel.md` (the pool's drop line and the `Control` row) and `docs/architecture/api/sdk/sandbox/get.md` (what the fence and the spent admission window decide). Production LOC: 157130 -> 157143 (delta +13) Scope and method: first-party production Rust/SQL/JavaScript in core, engine, control and api (tests, fixtures, tools, docs, manifests and generated artifacts excluded), counted with `tools/production_loc.py` at the parent commit de009a8b8 and the committed tree. engine 73881 -> 73884 (+3, `duplex::Connection::is_quarantined`), control 38524 -> 38534 (+10, `daemon/capacity.rs` and `daemon/channel.rs`); core 39749 and api 4976 unchanged. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../layerfs-sandbox/src/daemon/capacity.rs | 19 +++++- .../layerfs-sandbox/src/daemon/channel.rs | 16 +++-- .../layerfs-sandbox/src/daemon/fence.rs | 5 +- .../crates/layerfs-sandbox/src/daemon/pool.rs | 8 ++- .../layerfs-sandbox/tests/capacity_read.rs | 59 ++++++++++++++++++- docs/architecture/api/sdk/sandbox/get.md | 4 +- .../control/routing/daemon-channel.md | 6 +- .../layerfs-bridge/src/duplex/channel.rs | 7 +++ 8 files changed, 105 insertions(+), 19 deletions(-) diff --git a/control/crates/layerfs-sandbox/src/daemon/capacity.rs b/control/crates/layerfs-sandbox/src/daemon/capacity.rs index 4bf1a3f1a1..31b41b1556 100644 --- a/control/crates/layerfs-sandbox/src/daemon/capacity.rs +++ b/control/crates/layerfs-sandbox/src/daemon/capacity.rs @@ -46,12 +46,25 @@ pub fn read( /// The daemon's answer to [`OPERATION`] on a connection the Sandbox's pool /// lends (`sandbox.get`, #543): at most [`WAIT`] for admission and for the /// exchange, which is fenced at the same deadline, so this read takes no -/// second connection and no second slot of its own. Nothing is sent again, -/// and the connection goes back to the pool (or is closed, when the fence -/// quarantined it). +/// second connection and no second slot of its own. A window the admission +/// wait used up refuses the read before it is sent, and a connection the +/// fence ends stays quarantined: the pool closes it instead of lending it +/// again. Nothing is sent again. pub fn lent(controls: &Controls) -> Result { let deadline = Instant::now() + WAIT; let mut control = trace::control_within(controls, WAIT)?; + if deadline.saturating_duration_since(Instant::now()).is_zero() { + // the admission wait used the whole window. The connection is healthy + // and returns to the pool; arming a fence for a deadline that already + // passed would shut it down for nothing, so nothing is sent. + return Err(refused( + "Unavailable", + format!( + "no control connection came free within {}s, so the capacity read was not sent", + WAIT.as_secs() + ), + )); + } let fence = Fence::channel((control.close_handle().map_err(failed))?, deadline); let answer = exchange(&mut control); drop(fence); diff --git a/control/crates/layerfs-sandbox/src/daemon/channel.rs b/control/crates/layerfs-sandbox/src/daemon/channel.rs index eabe0636e3..49c161ef81 100644 --- a/control/crates/layerfs-sandbox/src/daemon/channel.rs +++ b/control/crates/layerfs-sandbox/src/daemon/channel.rs @@ -74,10 +74,13 @@ impl Control { pub fn take_telemetry(&mut self) -> Vec { std::mem::take(&mut self.telemetry) } - /// True once an exchange on this connection failed: it is quarantined, - /// and every later call on it is refused before sending. + /// True once this connection cannot carry another call: an exchange + /// failed on it, its session ended, or an application fence + /// ([`Control::close_handle`]) shut its channel down. The pool closes it + /// instead of lending it again, and a later call is refused before + /// sending, so a quarantined connection is never reported as an attempt. pub fn is_failed(&self) -> bool { - self.failed + self.failed || self.connection.is_quarantined() } /// Sends once and awaits the original reply once. Typed remote refusals are /// replies; failed delivery retains an unknown, never replays the command. @@ -118,7 +121,10 @@ impl Control { let mut attempted = false; let mut received = None; let result = (|| { - if self.failed { + if self.is_failed() { + // a fence or an ended session may have quarantined the channel + // after this owner's last call: nothing is sent on it + self.failed = true; return Err(ControlCause::Channel(ChannelError::Quarantined)); } let record = call.encode().map_err(ControlCause::Protocol)?; @@ -219,7 +225,7 @@ impl Control { manifest: &layerfs_bridge::provision::StoreManifest, ) -> Result> { - if self.failed { + if self.is_failed() { return Err(Box::new(layerfs_project::store::InstallFailure { source: store.path.clone(), manifest: manifest.clone(), diff --git a/control/crates/layerfs-sandbox/src/daemon/fence.rs b/control/crates/layerfs-sandbox/src/daemon/fence.rs index bfb2b9e735..fdbbf4345b 100644 --- a/control/crates/layerfs-sandbox/src/daemon/fence.rs +++ b/control/crates/layerfs-sandbox/src/daemon/fence.rs @@ -27,8 +27,9 @@ impl End { Self::Socket(socket) => { let _ = socket.shutdown(Shutdown::Both); } - // the handle quarantines the channel too, so the pool closes the - // connection instead of lending it again + // the handle quarantines the channel itself, and the pool reads + // that flag, so it closes the connection instead of lending it + // again Self::Channel(handle) => { let _ = handle.close(); } diff --git a/control/crates/layerfs-sandbox/src/daemon/pool.rs b/control/crates/layerfs-sandbox/src/daemon/pool.rs index 4588a3aff8..e7eab3a0be 100644 --- a/control/crates/layerfs-sandbox/src/daemon/pool.rs +++ b/control/crates/layerfs-sandbox/src/daemon/pool.rs @@ -5,8 +5,9 @@ //! every call wait for the one before it, a 1 ms read behind a whole Commit. A //! Sandbox keeps up to [`CONNECTIONS`] connections and opens one only when //! every open one is busy. Each exchange runs whole on one connection, sent -//! once. A connection whose exchange failed is quarantined by its `Control` -//! and is closed rather than reused; the failed call is never sent again. +//! once. A connection whose exchange failed, or whose channel an application +//! fence (`daemon::fence`) shut down, is quarantined by its `Control` and is +//! closed rather than reused; the failed call is never sent again. //! When the daemon admits no more connections, the Sandbox keeps the ones it //! has and callers take turns on them, each waiting at most [`LEND_WAIT`] //! before it is refused `Busy`. Opening a connection is not an attempt @@ -411,7 +412,8 @@ impl Drop for Lease<'_> { .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); if control.is_failed() { - // Quarantined: closed, and a later call may open another. + // Quarantined: its own failure or its channel's fence ended it, + // and a later call may open another. pool.open -= 1; drop(control); } else { diff --git a/control/crates/layerfs-sandbox/tests/capacity_read.rs b/control/crates/layerfs-sandbox/tests/capacity_read.rs index 9b5241fbfa..b90709a375 100644 --- a/control/crates/layerfs-sandbox/tests/capacity_read.rs +++ b/control/crates/layerfs-sandbox/tests/capacity_read.rs @@ -1,7 +1,8 @@ //! The `backend.daemon.connection_failures` read model's bounds (#540, //! #543): one query ends its control session instead of dropping the socket, -//! a daemon that trickles is fenced at the deadline, and a read on the -//! Sandbox's pool takes no second connection. +//! a daemon that trickles is fenced at the deadline, a read on the Sandbox's +//! pool takes no second connection, and a connection a fence or a session end +//! quarantined is never lent again or reported as an attempt. //! //! Nothing runs: a stand-in daemon answers the query on the connections it //! accepts, and counts what it saw. @@ -13,9 +14,12 @@ use layerfs_bridge::duplex; use layerfs_home::home::paths; use layerfs_home::secrets; use layerfs_sandbox::daemon::capacity; +use layerfs_sandbox::daemon::channel::ControlPhase; +use layerfs_sandbox::daemon::operator; use layerfs_sandbox::daemon::pool::{Controls, Dial}; use layerfs_sandbox::host::{Call as HostCall, Host, Message}; use layerfs_sandbox::record::SandboxRecord; +use layerfs_sandbox::{OperationCause, SdkError}; use std::io::Write; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::os::unix::fs::PermissionsExt; @@ -298,3 +302,54 @@ fn the_worker_s_side_reads_the_capacity_model() { assert_eq!(seen.queries.load(Ordering::SeqCst), 1); std::fs::remove_dir_all(home).unwrap(); } + +#[test] +fn a_fenced_connection_is_not_lent_again() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + { + let lease = controls.lend().unwrap().0; + // the fence's end, outside any exchange: the channel is shut down + // while this `Control` never saw a call fail, which is the window + // between a returned reply and `drop(fence)` + lease.close_handle().unwrap().close().unwrap(); + assert!( + lease.is_failed(), + "a channel a fence closed was still reported healthy" + ); + } + // and the pool closed it rather than lending it again: the next read + // opens a connection and answers, instead of reporting the dead one + let answer = capacity::lent(&controls).unwrap(); + assert_eq!(CapacityModel::parse(&answer).admission.pool_vacant, 0); + assert_eq!(seen.connections.load(Ordering::SeqCst), 2); +} + +#[test] +fn a_quarantined_connection_attempts_nothing() { + let (endpoint, seen) = daemon(Behavior::Model); + let dial = dial(endpoint); + let first = dial.open().unwrap(); + let controls = Controls::new(first, dial); + let mut lease = controls.lend().unwrap().0; + lease.close_handle().unwrap().close().unwrap(); + // the call a pooled lease would make next: refused before sending, so it + // is not an attempt whose effect is unknown + let error = operator::call(&mut lease, CAPACITY_OPERATION, SdkValue::Null).unwrap_err(); + match error { + SdkError::Operation(failure) => match failure.cause { + OperationCause::Exchange(channel) => { + assert!( + !channel.attempted, + "a closed channel was reported attempted" + ); + assert_eq!(channel.phase, ControlPhase::Encode); + } + other => panic!("{other:?}"), + }, + other => panic!("{other:?}"), + } + assert_eq!(seen.queries.load(Ordering::SeqCst), 0); +} diff --git a/docs/architecture/api/sdk/sandbox/get.md b/docs/architecture/api/sdk/sandbox/get.md index 7986a6bd07..accb9c91d5 100644 --- a/docs/architecture/api/sdk/sandbox/get.md +++ b/docs/architecture/api/sdk/sandbox/get.md @@ -106,7 +106,9 @@ the Workspace list lent from the Sandbox's pool ([`daemon/pool.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs)), bounded to 2 s for admission and for the exchange and fenced at that deadline ([`daemon/fence.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs)), -so a daemon that trickles cannot hold the slot; a forwarded daemon's forward +so a daemon that trickles cannot hold the slot, and a connection the fence +ends is quarantined rather than lent again; a window the admission wait used +up refuses the read before it is sent. A forwarded daemon's forward and its machine are checked before it is dialed (`direct::check`), so a stopped machine is `down_reason` and never a 2 s wait on a reused port. See the [routing table](../../../control/routing/routing-table.md) and the diff --git a/docs/architecture/control/routing/daemon-channel.md b/docs/architecture/control/routing/daemon-channel.md index 7103015780..67bd11168d 100644 --- a/docs/architecture/control/routing/daemon-channel.md +++ b/docs/architecture/control/routing/daemon-channel.md @@ -33,7 +33,7 @@ side of the channel is in [Engine: Daemon](../../engine/daemon/README.md). | | | lend(): idle one --> else open one more (if open < limit) | | --> else wait for a free one, <= 60 s --> Busy | - | drop(Lease): failed Control --> closed, open -= 1 | + | drop(Lease): quarantined Control --> closed, open -= 1 | | healthy Control --> back to idle | +--------------------------------------------------------------+ | @@ -43,7 +43,7 @@ side of the channel is in [Engine: Daemon](../../engine/daemon/README.md). | Part | What it is | Source | | --- | --- | --- | -| `Control` | one sequential authenticated connection; numbers its calls; once an exchange fails it is quarantined and refuses every later call before sending | [`daemon/channel.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/channel.rs) | +| `Control` | one sequential authenticated connection; numbers its calls; once an exchange fails, or an applied fence (`close_handle`, `daemon/fence.rs`) closes its channel, it is quarantined and refuses every later call before sending | [`daemon/channel.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/channel.rs) | | `Dial` | how to open another connection to the same daemon instance: endpoint, secret, peer key, instance | [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | | `Controls` | the Sandbox's pool: up to `CONNECTIONS` (6) per process; the daemon admits `DAEMON_CONNECTIONS` (8), leaving the rest to other processes | `daemon/pool.rs`, [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) | | `Lease` | one connection lent for one call's exchanges, returned on drop | `daemon/pool.rs` | @@ -82,7 +82,7 @@ opening a connection sends only Hello, so it attempts no operation. | `workspace.commit` | `Commit` | `values::commit`: Committed, Merged (with `merged`), or `Conflict` | [`daemon/workspace/commit.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/commit.rs), [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | `workspace.unmount` | `Unmount{token, discard}`, or `ForceUnmount` with `force` | `values::unmounted` (`Dirty` refusal) or `force_unmounted` | [`daemon/workspace/unmount.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/unmount.rs) | | `workspace.status` | `Status(token)`; the Branch head is read separately on the Store's route | `values::status` | [`daemon/workspace/status.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/status.rs) | -| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs)) | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | +| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs); a window admission used up refuses the read, and a connection the fence ends is not lent again) | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | | `workspace.exec` | none on this channel: `bash -c` runs through the Sandbox's runtime | — | [`exec/run.rs`](../../../../control/crates/layerfs-sandbox/src/exec/run.rs) | `Locate` and `Cleanup` are read-only observations for a caller whose diff --git a/engine/crates/layerfs-bridge/src/duplex/channel.rs b/engine/crates/layerfs-bridge/src/duplex/channel.rs index 0e56901692..76abb4c192 100644 --- a/engine/crates/layerfs-bridge/src/duplex/channel.rs +++ b/engine/crates/layerfs-bridge/src/duplex/channel.rs @@ -62,6 +62,13 @@ impl Connection { shared: self.receive.shared.clone(), }) } + /// Whether this channel is quarantined: a failed exchange, an ended + /// session or an application fence (`CloseHandle`) shut both directions + /// down, and no record may be sent on it again. All owners read the one + /// shared flag, so a fence's close is visible to whoever borrowed it. + pub fn is_quarantined(&self) -> bool { + self.receive.shared.ready().is_err() + } /// Clears explicit handshake I/O waits on the actual shared socket before long-lived operations. /// An original policy-change error quarantines/fences this owner once. pub fn clear_io_waits(&self) -> ChannelResult<()> {