diff --git a/api/crates/layerfs-api/tests/sandbox_capacity_worker.rs b/api/crates/layerfs-api/tests/sandbox_capacity_worker.rs new file mode 100644 index 0000000000..5e74cc0f86 --- /dev/null +++ b/api/crates/layerfs-api/tests/sandbox_capacity_worker.rs @@ -0,0 +1,256 @@ +//! `sandbox.get` of a Sandbox on a machine whose daemon has no vacant +//! control slot (#649). This route runs the Workspace list in the machine's +//! worker, so the refusal its daemon gave comes back from the worker exactly +//! as it was made — `Capacity` on `daemon_control_slot`, with the vacant +//! slots and retained failures the admission notice carried, and no `code`. +//! The get reads that refusal the way it reads a forwarded daemon's: the +//! record with the Sandbox down, the refusal's own words as `down_reason`, +//! what it observed as the partial `down` counts, and no `daemon_capacity` +//! (a notice is never mapped into that model) — never a failed call. +//! +//! The worker here is a stand-in: it answers the secret, reads one request +//! and answers the refusal that its machine's daemon gave. What it does not +//! do is the worker's own dispatch, which is where the refusal is made; the +//! refusal is data from then on, and this is the caller's side of it. +#![cfg(unix)] +use layerfs_api::home::{self, paths, write_record}; +use layerfs_api::worker::{read_frame, write_frame, Hello}; +use layerfs_bridge::control::SdkValue; +use std::io::Write; +use std::net::{SocketAddr, TcpListener}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +/// Names the homes of one test process apart. +static HOMES: AtomicUsize = AtomicUsize::new(0); + +fn tempdir(tag: &str) -> PathBuf { + let sequence = HOMES.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!( + "lfs-scap-worker-{tag}-{}-{sequence}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + dir +} + +fn t(s: &str) -> SdkValue { + SdkValue::text(s) +} + +fn call(home: &Path, name: &str, args: &[(&str, SdkValue)]) -> Result { + let args = SdkValue::Map( + args.iter() + .map(|(n, v)| ((*n).to_owned(), v.clone())) + .collect(), + ); + layerfs_api::call(Some(home), name, args, &mut |_| {}).map_err(|r| (r.code, r.message)) +} + +/// A record from `fields`, in order. +fn map(fields: &[(&str, SdkValue)]) -> SdkValue { + SdkValue::Map( + fields + .iter() + .map(|(name, value)| ((*name).to_owned(), value.clone())) + .collect(), + ) +} + +/// Machine `box` (a `docker` machine, so its Sandboxes run in its worker), +/// its Worker record at `worker`, Store `shop` and project `api`. +fn home_records(home: &Path, worker: SocketAddr) { + write_record( + &paths::machine_record(home, "box"), + &map(&[("name", t("box")), ("platform", t("docker"))]), + ) + .unwrap(); + write_record( + &paths::worker_record(home, "box"), + &map(&[("address", t(&worker.to_string())), ("secret", t("c0ffee"))]), + ) + .unwrap(); + write_record( + &paths::store_record(home, "shop"), + &map(&[ + ("name", t("shop")), + ("kind", t("sqlite")), + ("binding", SdkValue::Bytes(vec![7; 16])), + ("cursor", SdkValue::Bytes(vec![8; 32])), + ("profile", t("disposable")), + ("machine", t("box")), + ]), + ) + .unwrap(); + write_record( + &paths::project_record(home, "shop", "api"), + &map(&[("name", t("api"))]), + ) + .unwrap(); +} + +/// Sandbox `s1` on `box`, recorded with its `id` and no forward: this home +/// reaches it through that machine's worker. +fn sandbox_record(home: &Path) { + let fields = [ + ("name", t("s1")), + ("project", t("shop/api")), + ("machine", t("box")), + ("id", t("s-1")), + ]; + write_record(&paths::sandbox_record(home, "s1"), &map(&fields)).unwrap(); +} + +/// The refusal a daemon whose pool has no vacant control slot gives before +/// any handshake, as it comes back from the worker that ran the call. +fn full_pool() -> SdkValue { + refused( + "Capacity", + "the Sandbox's daemon has no vacant control slot: 0 pool slot(s) and 1 \ + reserved slot(s) are vacant, with 3 retained failure record(s). The call was not \ + admitted; retrying is safe", + SdkValue::map([ + ("resource", t("daemon_control_slot")), + ("retained", SdkValue::Int(3)), + ("pool_vacant", SdkValue::Int(0)), + ("reserved_vacant", SdkValue::Int(1)), + ("retryable", SdkValue::Bool(true)), + ("admission", t("no-pool-slot")), + ]), + ) +} + +/// Refusal `code` with `detail`, in the frame a worker answers a request it +/// refused. +fn refused(code: &str, message: &str, detail: SdkValue) -> SdkValue { + SdkValue::map([( + "refused", + SdkValue::map([ + ("code", t(code)), + ("message", t(message)), + ("detail", detail), + ]), + )]) +} + +/// A worker that answers its secret, reads one request and refuses it with +/// `answer`, counting the requests it read. +fn refusing_worker(answer: SdkValue) -> (SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = listener.local_addr().unwrap(); + let asked = Arc::new(AtomicUsize::new(0)); + let counted = asked.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + let _ = stream.set_write_timeout(Some(Duration::from_secs(10))); + // the caller's secret, answered before it sends anything + if read_frame(&mut stream).is_err() { + continue; + } + if write_frame(&mut stream, &Hello::own().to_frame()).is_err() { + continue; + } + // one request, answered once: nothing is resent, and the get + // dials nothing for the capacity model + let Ok(Some(_)) = read_frame(&mut stream) else { + continue; + }; + counted.fetch_add(1, Ordering::SeqCst); + let _ = write_frame(&mut stream, &answer); + let _ = stream.flush(); + } + }); + (endpoint, asked) +} + +#[test] +fn a_full_pool_s_get_reads_the_worker_route_like_the_forwarded_one() { + let (endpoint, asked) = refusing_worker(full_pool()); + let home = tempdir("full"); + home::open(&home).unwrap(); + home_records(&home, endpoint); + sandbox_record(&home); + // the record is answered, not refused: a full pool is why a get exists + let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap(); + assert_eq!(asked.load(Ordering::SeqCst), 1, "asked {asked:?} times"); + assert_eq!(got.get_text("state"), Some("running"), "{got:?}"); + // the Workspace list is what was refused, so there is no page to show + assert_eq!( + got.get("workspaces"), + Some(&SdkValue::List(Vec::new())), + "{got:?}" + ); + let reason = got + .get_text("down_reason") + .unwrap_or_else(|| panic!("{got:?}")); + assert!(reason.contains("no vacant control slot"), "{reason}"); + // and the counts are readable, in the same partial model the forwarded + // route answers: the notice's five keys and nothing else + let down = got.get("down").unwrap_or_else(|| panic!("{got:?}")); + assert_eq!(down.get_text("code"), Some("Capacity"), "{down:?}"); + assert_eq!( + down.get_text("resource"), + Some("daemon_control_slot"), + "{down:?}" + ); + assert_eq!(down.get_int("pool_vacant"), Some(0), "{down:?}"); + assert_eq!(down.get_int("reserved_vacant"), Some(1), "{down:?}"); + assert_eq!(down.get_int("retained"), Some(3), "{down:?}"); + let SdkValue::Map(fields) = down else { + panic!("{down:?}") + }; + assert_eq!(fields.len(), 5, "{down:?}"); + // the notice is partial: it is never dressed up as the full model, and + // the get dials nothing for one + assert!(got.get("daemon_capacity").is_none(), "{got:?}"); + assert_eq!(asked.load(Ordering::SeqCst), 1, "asked {asked:?} times"); + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn another_capacity_or_a_busy_refusal_still_fails_the_get() { + // only a full pool's own refusal reads as a down Sandbox: `Capacity` on + // another resource and a refusal with another code are the refusals they + // are, so a get never hides one behind `down_reason` + for (code, detail) in [ + ("Capacity", SdkValue::map([("resource", t("store_lock"))])), + ("Busy", SdkValue::Null), + ] { + let (endpoint, asked) = refusing_worker(refused(code, "the pool says so", detail)); + let home = tempdir("other"); + home::open(&home).unwrap(); + home_records(&home, endpoint); + sandbox_record(&home); + let error = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap_err(); + assert_eq!(asked.load(Ordering::SeqCst), 1, "asked {asked:?} times"); + assert_eq!(error.0, code, "{error:?}"); + assert_eq!(error.1, "the pool says so", "{error:?}"); + std::fs::remove_dir_all(home).unwrap(); + } +} + +#[test] +fn a_capacity_on_a_slot_that_observed_no_counts_still_fails_the_get() { + // a `Capacity` an admitted connection returned is collapsed to `{phase}` + // by `values::refusal` (#547 slice 6): it names no pool and carries no + // counts, so it is not a full pool's notice and the get must fail as it + // did before rather than answer an invented down + let detail = SdkValue::map([("phase", t("receive"))]); + let (endpoint, asked) = refusing_worker(refused("Capacity", "a slot said so", detail)); + let home = tempdir("phase"); + home::open(&home).unwrap(); + home_records(&home, endpoint); + sandbox_record(&home); + let error = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap_err(); + assert_eq!(asked.load(Ordering::SeqCst), 1, "asked {asked:?} times"); + assert_eq!(error.0, "Capacity", "{error:?}"); + assert_eq!(error.1, "a slot said so", "{error:?}"); + std::fs::remove_dir_all(home).unwrap(); +} diff --git a/control/crates/layerfs-sandbox/src/direct.rs b/control/crates/layerfs-sandbox/src/direct.rs index 5f702985a8..9cc6c22d01 100644 --- a/control/crates/layerfs-sandbox/src/direct.rs +++ b/control/crates/layerfs-sandbox/src/direct.rs @@ -120,8 +120,9 @@ pub fn unanswered(id: &str, daemon: &DaemonRecord, refusal: SdkRefusal) -> SdkRe /// (#649): a `Capacity` refusal's notice counts, tagged with the refusal /// code it came from. Every other refusal keeps nothing: a legacy EOF, a /// malformed record, or a remote refusal collapsed to `{phase}` has no -/// counts, and inventing zeros would be false. -fn down_detail(code: &str, detail: SdkValue) -> SdkValue { +/// counts, and inventing zeros would be false. A worker or local route's +/// notice carries the same counts without the tag, so `get` tags it here. +pub(crate) fn down_detail(code: &str, detail: SdkValue) -> SdkValue { match code { "Capacity" => home::with_field(detail, "code", SdkValue::text("Capacity")), _ => SdkValue::Null, diff --git a/control/crates/layerfs-sandbox/src/get.rs b/control/crates/layerfs-sandbox/src/get.rs index b80783f976..a85959aaf9 100644 --- a/control/crates/layerfs-sandbox/src/get.rs +++ b/control/crates/layerfs-sandbox/src/get.rs @@ -87,10 +87,13 @@ pub struct Got { pub state: &'static str, /// The page of its Workspaces. pub workspaces: Vec, - /// Why its machine did not answer, when it did not. + /// Why its machine did not answer, or why the daemon it reached admitted + /// no control connection, when either. pub down_reason: Option, /// The structured counts behind `down_reason`, when the refusing - /// daemon's capacity notice carried them (#649). + /// daemon's capacity notice carried them (#649): read from a forwarded + /// daemon's unanswered dial, or from the pool a worker or this process + /// itself dialed. pub down: Option, /// The next page's position; `None` after the last. pub next: Option, @@ -122,6 +125,17 @@ pub fn get(home: &Path, events: &mut Events<'_>, request: &GetRequest) -> Result }; let (all, down_reason, down, gone) = match listed { Ok(Err(refusal)) if refusal.code == "NoSuchSandbox" => (Vec::new(), None, None, true), + // a route that runs the call in a machine's worker, or here, is told + // by the pool it dialed: `Capacity` on `daemon_control_slot`, with + // the counts the admission notice carried and no `code`. It reads as + // the forwarded route's down Sandbox (#649), from the same detail. + Ok(Err(refusal)) if slot_refused(&refusal) => { + let tagged = SdkRefusal { + detail: direct::down_detail("Capacity", refusal.detail.clone()), + ..refusal.clone() + }; + (Vec::new(), Some(refusal.message), Down::of(&tagged), false) + } Ok(workspaces) => match workspaces?.get("workspaces") { Some(SdkValue::List(items)) => ( items.iter().map(|i| listed_workspace(name, i)).collect(), @@ -153,6 +167,14 @@ pub fn get(home: &Path, events: &mut Events<'_>, request: &GetRequest) -> Result }) } +/// Whether `refusal` is a full pool's own refusal: the pool a worker or this +/// process dialed admitted no control connection. A `Capacity` an admitted +/// connection returned (`values::refusal`'s `{phase}`) observed no counts and +/// is not one. +fn slot_refused(refusal: &SdkRefusal) -> bool { + refusal.code == "Capacity" && refusal.detail.get_text("resource") == Some("daemon_control_slot") +} + /// The daemon's `backend.daemon.connection_failures` read model of a Sandbox /// this process still reaches (#540): read once, best effort, and omitted /// when the Sandbox is gone, is not running, or its machine or forward is diff --git a/docs/architecture/api/sdk/sandbox/README.md b/docs/architecture/api/sdk/sandbox/README.md index f4218f8a58..99715fee3a 100644 --- a/docs/architecture/api/sdk/sandbox/README.md +++ b/docs/architecture/api/sdk/sandbox/README.md @@ -36,7 +36,7 @@ unique across the home. | --- | --- | --- | --- | --- | | [`create`](create.md) | `project` (Project), `name` (Name), `machine?` (Name) | `image` (Text) | the Sandbox record with `state`; `forward_failed` when a direct daemon forward did not open | its machine: here or `sandbox.host` in the worker | | [`list`](list.md) | `project?` (Project) | `page_size`, `page_pos` (Int) | `{ items: [record], next_page_pos }` | caller's process, home records only | -| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down, `down` (partial capacity counts) when the refusal carried the daemon's notice, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing | +| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down or its daemon's pool has no vacant control slot, `down` (partial capacity counts) when the refusal carried the daemon's notice on either route, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing | | [`close`](close.md) | `name` (Name) | `force` (Bool) | `{ kind: "Closed", sandbox, unknown, closed }`, or `{ closed, was_gone: true }`, or `{ closed, was, left? }` for a record with no id | its machine: here or `sandbox.host` in the worker | The record is `SandboxRecord::to_sdk` @@ -56,7 +56,8 @@ When the Workspace list is refused at a full pool, `down` carries the partial counts from the daemon's pre-handshake capacity notice: `code`, `resource`, `pool_vacant`, `reserved_vacant` and `retained`, and no `daemon_capacity`, whose missing fields would read as zero -(#649). +(#649). A worker or local route meets that full pool as a refusal its own +dial returned, so it answers the same `down` from the same notice. ## Arguments and options diff --git a/docs/architecture/api/sdk/sandbox/get.md b/docs/architecture/api/sdk/sandbox/get.md index a87bb90b7b..31f14f8a83 100644 --- a/docs/architecture/api/sdk/sandbox/get.md +++ b/docs/architecture/api/sdk/sandbox/get.md @@ -78,8 +78,8 @@ layerfs sandbox get [--page-size ] [--page-pos ] | `workspaces[].name`, `path` | text | the mount's name or null; the mount directory | | `workspaces[].*` | | the daemon's read model per Workspace: id, namespace, Branch, base Commit and Layer, root, `activity`, dirty counts ([`operator/inspect.rs`](../../../../../engine/crates/layerfs-store-service/src/operator/inspect.rs) `workspace_value`) | | `next_page_pos` | int or null | next page of `workspaces` | -| `down_reason` | text | the machine's worker or the forwarded daemon was not reached; `workspaces` is then `[]` | -| `down` | record | the structured counts behind a `down_reason`, when the refused connection carried the daemon's pre-handshake `LAYERFS-CAPACITY` notice (#649): `code` is the refusal the notice made (`Capacity`), `resource` the exhausted resource (`daemon_control_slot`), and `pool_vacant`, `reserved_vacant` and `retained` its counts. It is a **partial** model, never `daemon_capacity`: the field is absent when the refusal had no counts — a legacy bare EOF, a malformed record, or a remote refusal collapsed to `{phase}` ([`direct.rs`](../../../../../control/crates/layerfs-sandbox/src/direct.rs) `unanswered`, [`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs) `Down`) | +| `down_reason` | text | the machine's worker or the forwarded daemon was not reached, or the daemon the route did reach admitted no control connection because its pool had no vacant slot (the same refusal on either route); `workspaces` is then `[]` | +| `down` | record | the structured counts behind a `down_reason`, when the refused connection carried the daemon's pre-handshake `LAYERFS-CAPACITY` notice (#649): `code` is the refusal the notice made (`Capacity`), `resource` the exhausted resource (`daemon_control_slot`), and `pool_vacant`, `reserved_vacant` and `retained` its counts. A forwarded daemon's unanswered dial is tagged by `direct::unanswered`; a worker's or this process's own dial returns the notice as it was made and `get` tags it the same way, so both routes answer this one model. It is a **partial** model, never `daemon_capacity`: the field is absent when the refusal had no counts — a legacy bare EOF, a malformed record, or a remote refusal collapsed to `{phase}` ([`direct.rs`](../../../../../control/crates/layerfs-sandbox/src/direct.rs) `unanswered`, [`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs) `Down`, `slot_refused`) | | `daemon_capacity` | record | the daemon's `backend.daemon.connection_failures` read model (#540): control admission by slot class and the bounded retained connection failures that used to hold a slot ([`control_capacity.rs`](../../../../../engine/crates/layerfs-bridge/src/control_capacity.rs)). It is read once, best effort, and only for a Sandbox that is `running` and whose machine and forward were reached; nothing is started and no second control slot is taken, so the field is simply absent when the daemon refuses, does not answer within 2 s, or was not asked ([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)) | ## Where it runs @@ -98,6 +98,9 @@ layerfs sandbox get [--page-size ] [--page-pos ] Unreached / Unavailable --> record + workspaces [] + down_reason | a Capacity refusal keeps its notice detail --> down (partial counts) | any other refusal (legacy EOF, {phase}) --> no down + Ok(Err Capacity on daemon_control_slot) --> record + workspaces [] + | the pool a worker, or this process, dialed: down_reason + down, + | the same partial counts, and nothing is dialed again NoSuchSandbox --> state "lost", workspaces [] running and not down, same route and the same pooled connection: @@ -109,7 +112,11 @@ an intent. At a full daemon pool a forwarded Sandbox's list is refused `Capacity` after the whole 60 s admission window; the get still answers, with `down_reason` and — when the daemon wrote its capacity notice — the partial `down` counts the capacity read itself cannot fetch in that state, -because it would need the same slot (#649). The capacity read is a second +because it would need the same slot (#649). A Sandbox whose list runs in a +machine's worker, or here, meets the same full pool on its own dial: that +refusal is data the route returns, so the get answers the same `down` from +the same counts, asks nothing twice, and omits `daemon_capacity` as it does +on the forwarded route. The capacity read is a second host operation on the connection the Workspace list lent from the Sandbox's pool ([`daemon/pool.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs)), bounded to 2 s for admission and for @@ -129,6 +136,7 @@ the [routing table](../../../control/routing/routing-table.md) and the | `InvalidInput` | bad name; folder scope | no | [`names.rs`](../../../../../control/crates/layerfs-home/src/names.rs), [`calls/scope.rs`](../../../../../api/crates/layerfs-api/src/calls/scope.rs) | | `NoSuchSandbox` | no record of that name | no | [`family.rs`](../../../../../control/crates/layerfs-sandbox/src/family.rs) | | `Busy`, `StaleSandbox` | a forwarded daemon admits no connection in time; a saved daemon is not `ControlReady` | no | [`daemon/pool.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | +| `Capacity` | a `Capacity` that is not the full pool's own notice: another resource, or a refusal an admitted connection returned (`{phase}`) | no | [`get.rs`](../../../../../control/crates/layerfs-sandbox/src/get.rs) `slot_refused` | | `Unknown` | **the worker's answer was lost after the request was sent; harmless for a read, but the answer is missing** | **yes (no effect expected)** | [`worker/client.rs`](../../../../../control/crates/layerfs-machine/src/worker/client.rs) | | `Failed` | the daemon's list failed or was not a record | no effect (a read) | [`dispatch.rs`](../../../../../control/crates/layerfs-sandbox/src/dispatch.rs) | diff --git a/docs/architecture/control/routing/daemon-channel.md b/docs/architecture/control/routing/daemon-channel.md index 5a6eced07b..d91e7d6553 100644 --- a/docs/architecture/control/routing/daemon-channel.md +++ b/docs/architecture/control/routing/daemon-channel.md @@ -82,7 +82,7 @@ opening a connection sends only Hello, so it attempts no operation. | `workspace.commit` | `Commit` | `values::commit`: Committed, Merged (with `merged`), or `Conflict` | [`daemon/workspace/commit.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/commit.rs), [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | `workspace.unmount` | `Unmount{token, discard}`, or `ForceUnmount` with `force` | `values::unmounted` (`Dirty` refusal) or `force_unmounted` | [`daemon/workspace/unmount.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/unmount.rs) | | `workspace.status` | `Status(token)`; the Branch head is read separately on the Store's route | `values::status` | [`daemon/workspace/status.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/workspace/status.rs) | -| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs); a window admission used up refuses the read, and a connection the fence ends is not lent again); a full pool's pre-handshake `LAYERFS-CAPACITY` notice survives the failed open as the partial `down` counts (#649) | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | +| `sandbox.get` | SDK method `sandbox.list_workspaces`, then, on the same pooled connection and for a running Sandbox its machine still reaches, `backend.daemon.connection_failures` (`daemon_capacity`, bounded to 2 s: admission and the exchange share one deadline, fenced by [`daemon/fence.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/fence.rs); a window admission used up refuses the read, and a connection the fence ends is not lent again); a full pool's pre-handshake `LAYERFS-CAPACITY` notice survives the failed open as the partial `down` counts (#649), on the forwarded route and on the worker/local one, where the same notice arrives as the refusal the call returned | `sandbox/dispatch.rs` (both through `daemon/operator.rs`) | [`daemon/operator.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/operator.rs), [`daemon/capacity.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs) | | `workspace.exec` | none on this channel: `bash -c` runs through the Sandbox's runtime | — | [`exec/run.rs`](../../../../control/crates/layerfs-sandbox/src/exec/run.rs) | `Locate` and `Cleanup` are read-only observations for a caller whose