Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions api/crates/layerfs-api/src/sandbox/get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ pub(crate) const CALL: Call = call(

/// Runs `sandbox.get`: its record with its derived `state`, one page of its
/// `workspaces` (each with its `path`, `name` and `ws`), why its machine did
/// not answer (`down_reason`), the daemon's `daemon_capacity` read model when
/// the Sandbox answered it (#540), and `next_page_pos`. The library reads it
/// best effort and bounded, so this layer only shows what it got.
/// not answer (`down_reason`, with the partial `down` counts when the
/// daemon's capacity notice carried them, #649), the daemon's
/// `daemon_capacity` read model when the Sandbox answered it (#540), and
/// `next_page_pos`. The library reads it best effort and bounded, so this
/// layer only shows what it got.
fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome {
let request = GetRequest {
name: args.get_text("name").unwrap_or("").to_owned(),
Expand All @@ -36,6 +38,7 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome {
state,
workspaces,
down_reason,
down,
next,
daemon_capacity,
} = get(home, events, &request)?;
Expand All @@ -50,6 +53,9 @@ fn run(context: &mut Context<'_>, args: &SdkValue) -> Outcome {
if let Some(why) = down_reason {
shown = with_field(shown, "down_reason", SdkValue::Text(why));
}
if let Some(down) = down {
shown = with_field(shown, "down", down.to_sdk());
}
if let Some(model) = daemon_capacity {
shown = with_field(shown, "daemon_capacity", model.to_sdk());
}
Expand Down
120 changes: 114 additions & 6 deletions api/crates/layerfs-api/tests/sandbox_capacity.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
//! `sandbox.get`'s `daemon_capacity` (#540, #543): the daemon's
//! `sandbox.get`'s daemon fields (#540, #543, #649): the
//! `backend.daemon.connection_failures` read model, shown best-effort and
//! bounded on the connection the Workspace list lent from this process's
//! pool. Nothing runs: a stand-in daemon answers the Sandbox's Workspace list
//! and its capacity model on the same connection, a second one answers the
//! list and then says nothing, and a Sandbox whose machine or forward is
//! already down is not dialed at all.
//! pool, and the partial `down` counts a full-pool refusal carries. Nothing
//! runs: a stand-in daemon answers the Sandbox's Workspace list and its
//! capacity model on the same connection, a second one answers the list and
//! then says nothing, a third refuses every connection with the
//! pre-handshake capacity notice (or a legacy bare close), and a Sandbox
//! whose machine or forward is already down is not dialed at all.
#![cfg(unix)]
use layerfs_api::home::{self, paths, write_record};
use layerfs_bridge::control::{
AdmissionModel, Answer, Call, DaemonPhase, DaemonStatus, Reply, Request, RetainedModel,
SdkValue, OPERATION,
SdkRefusal, SdkValue, OPERATION,
};
use layerfs_bridge::duplex;
use layerfs_bridge::duplex::admission::{self, AdmissionNotice, AdmissionReason};
use std::net::{SocketAddr, TcpListener};
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -281,6 +284,8 @@ fn a_running_sandbox_s_get_shows_its_daemon_capacity() {
);
assert_eq!(failures[0].get_int("generation"), Some(6));
assert_eq!(failures[0].get_text("request"), Some("unmount"));
// a daemon that answered carries no down counts (#649)
assert!(got.get("down").is_none(), "{got:?}");
// the Workspace list and the capacity read share the one pooled
// connection (#543): a get takes no second control slot of its own
assert_eq!(served.load(Ordering::SeqCst), 1, "{got:?}");
Expand Down Expand Up @@ -363,3 +368,106 @@ fn a_sandbox_with_no_id_is_never_dialed() {
assert_eq!(served.load(Ordering::SeqCst), 0, "{got:?}");
std::fs::remove_dir_all(home).unwrap();
}

/// A stand-in daemon that refuses every connection the way an accept loop
/// does when its pool is full: with `notice` before any handshake, or, for a
/// daemon that predates the notice, with a bare close (#540). It drains what
/// the client sent and counts the connections it refused.
fn refusing(notice: Option<AdmissionNotice>) -> (SocketAddr, Arc<AtomicUsize>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let endpoint = listener.local_addr().unwrap();
let served = Arc::new(AtomicUsize::new(0));
let counted = served.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else { return };
counted.fetch_add(1, Ordering::SeqCst);
let _ = stream.set_write_timeout(Some(Duration::from_millis(100)));
let _ = stream.set_read_timeout(Some(Duration::from_millis(50)));
if let Some(notice) = notice {
if admission::write(&mut stream, &notice).is_err() {
continue;
}
}
let mut sink = [0u8; 512];
let _ = std::io::Read::read(&mut stream, &mut sink);
}
});
(endpoint, served)
}

#[test]
fn a_full_daemon_pool_shows_the_partial_down_counts() {
let (endpoint, served) = refusing(Some(AdmissionNotice::new(
AdmissionReason::NoPoolSlot,
3,
0,
1,
)));
let home = tempdir("full-pool");
home::open(&home).unwrap();
home_records(&home, true);
sandbox_record(&home, endpoint, Some("s-1"), None);
let at = Instant::now();
let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap();
let elapsed = at.elapsed();
// This is the measured behaviour (#649): a full pool is not visible to
// the get before the whole admission window has passed.
assert!(
elapsed >= Duration::from_secs(55),
"the get answered before the admission window: {elapsed:?}"
);
assert!(served.load(Ordering::SeqCst) > 0, "{got:?}");
assert_eq!(got.get_text("state"), Some("running"), "{got:?}");
assert!(got.get_text("down_reason").is_some(), "{got:?}");
let down = got.get("down").unwrap_or_else(|| panic!("{got:?}"));
assert_eq!(down.get_text("code"), Some("Capacity"), "{down:?}");
assert_eq!(
down.get_text("resource"),
Some("daemon_control_slot"),
"{down:?}"
);
assert_eq!(down.get_int("pool_vacant"), Some(0), "{down:?}");
assert_eq!(down.get_int("reserved_vacant"), Some(1), "{down:?}");
assert_eq!(down.get_int("retained"), Some(3), "{down:?}");
// the notice is partial: it is never dressed up as the full model
assert!(got.get("daemon_capacity").is_none(), "{got:?}");
std::fs::remove_dir_all(home).unwrap();
}

#[test]
fn a_legacy_eof_has_no_down_counts() {
// a daemon that closes the connection without the capacity notice: the
// refusal says only what was observed, so the get shows down_reason and
// no `down` counts (#649)
let (endpoint, served) = refusing(None);
let home = tempdir("legacy-eof");
home::open(&home).unwrap();
home_records(&home, true);
sandbox_record(&home, endpoint, Some("s-1"), None);
let at = Instant::now();
let got = call(&home, "sandbox.get", &[("name", t("s1"))]).unwrap();
assert!(
at.elapsed() >= Duration::from_secs(55),
"a legacy EOF is retried for the admission window: {:?}",
at.elapsed()
);
assert!(served.load(Ordering::SeqCst) > 0, "{got:?}");
assert!(got.get_text("down_reason").is_some(), "{got:?}");
assert!(got.get("down").is_none(), "{got:?}");
assert!(got.get("daemon_capacity").is_none(), "{got:?}");
std::fs::remove_dir_all(home).unwrap();
}

#[test]
fn a_collapsed_remote_refusal_has_no_down_counts() {
// A refusal forwarded from a machine's worker is collapsed to
// `{phase}` by `values::refusal` (#547 slice 6): it carries no counts,
// so the get must omit `down` rather than fill zeros.
let refusal = SdkRefusal {
code: "Unavailable".to_owned(),
message: "the Sandbox's daemon ended this connection before it answered".to_owned(),
detail: SdkValue::map([("phase", SdkValue::text("receive"))]),
};
assert!(layerfs_sandbox::family::Down::of(&refusal).is_none());
}
37 changes: 29 additions & 8 deletions control/crates/layerfs-sandbox/src/direct.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,21 +90,42 @@ pub fn live(state: &SandboxState) -> Result<()> {
/// The refusal of a forwarded Sandbox `id` whose daemon did not answer a
/// new connection: `Unavailable`, naming the calls that say why. A busy
/// daemon's `Busy` and a restarted one's `StaleSandbox` stay as they are.
///
/// A `Capacity` refusal keeps its structured `detail` (#649): the
/// pre-handshake `LAYERFS-CAPACITY` notice is the only place the counts
/// exist while the pool is full, and `sandbox.get` shows them as its
/// partial `down` model. The surviving `Unavailable` detail carries the
/// underlying `code` so the get can tell it from a refusal that never had
/// counts.
pub fn unanswered(id: &str, daemon: &DaemonRecord, refusal: SdkRefusal) -> SdkRefusal {
if matches!(refusal.code.as_str(), "Busy" | "StaleSandbox") {
return refusal;
}
let at = forward_of(daemon)
.map(|a| a.to_string())
.unwrap_or_default();
refused(
"Unavailable",
format!(
"Sandbox {id}'s daemon does not answer at {at}: {}; `layerfs sandbox get {id}` \
shows whether it runs, `layerfs sandbox close {id}` closes it",
refusal.message
),
)
let message = format!(
"Sandbox {id}'s daemon does not answer at {at}: {}; `layerfs sandbox get {id}` \
shows whether it runs, `layerfs sandbox close {id}` closes it",
refusal.message
);
SdkRefusal {
code: "Unavailable".to_owned(),
message,
detail: down_detail(refusal.code.as_str(), refusal.detail),
}
}

/// The detail an unanswered get keeps from the refusal that made it
/// (#649): a `Capacity` refusal's notice counts, tagged with the refusal
/// code it came from. Every other refusal keeps nothing: a legacy EOF, a
/// malformed record, or a remote refusal collapsed to `{phase}` has no
/// counts, and inventing zeros would be false.
fn down_detail(code: &str, detail: SdkValue) -> SdkValue {
match code {
"Capacity" => home::with_field(detail, "code", SdkValue::text("Capacity")),
_ => SdkValue::Null,
}
}

/// A worker's `sandbox.create` answer `out` for Sandbox `name` under its
Expand Down
2 changes: 1 addition & 1 deletion control/crates/layerfs-sandbox/src/family.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
//! `route.rs`, and the `workspace` calls in `workspace/calls.rs`.
pub use super::close::{close, CloseRequest, Closed};
pub use super::create::{create, CreateRequest, Created};
pub use super::get::{get, GetRequest, Got, ListedWorkspace};
pub use super::get::{get, Down, GetRequest, Got, ListedWorkspace};
pub use super::list::{list, ListRequest, Listed};
use super::record::SandboxRecord;
use layerfs_bridge::control::SdkRefusal;
Expand Down
67 changes: 61 additions & 6 deletions control/crates/layerfs-sandbox/src/get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,53 @@ pub struct ListedWorkspace {
pub ws: String,
}

/// Why a get's daemon was not reached, as far as the refused connection
/// said (#649): a **partial** model, filled only from the daemon's
/// pre-handshake `LAYERFS-CAPACITY` notice. It is not
/// [`CapacityModel`]: that model's missing fields would read as zero, so a
/// notice is never mapped into `daemon_capacity`.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Down {
/// The refusal the notice produced: `Capacity`.
pub code: String,
/// The exhausted resource, e.g. `daemon_control_slot`.
pub resource: String,
/// Pool slots vacant when the daemon refused.
pub pool_vacant: i64,
/// Reserved slots vacant when the daemon refused.
pub reserved_vacant: i64,
/// Retained failure records visible when the daemon refused.
pub retained: i64,
}

impl Down {
/// The partial model a refusal's detail carries, when it carries every
/// count. A detail without the capacity fields (a legacy EOF, or a
/// remote refusal collapsed to `{phase}`) has none, and `None` is
/// returned rather than zeros.
pub fn of(refusal: &SdkRefusal) -> Option<Self> {
let detail = &refusal.detail;
Some(Self {
code: detail.get_text("code")?.to_owned(),
resource: detail.get_text("resource")?.to_owned(),
pool_vacant: detail.get_int("pool_vacant")?,
reserved_vacant: detail.get_int("reserved_vacant")?,
retained: detail.get_int("retained")?,
})
}

/// The SDK value `sandbox.get` shows as `down`.
pub fn to_sdk(&self) -> SdkValue {
SdkValue::map([
("code", SdkValue::text(self.code.as_str())),
("resource", SdkValue::text(self.resource.as_str())),
("pool_vacant", SdkValue::Int(self.pool_vacant)),
("reserved_vacant", SdkValue::Int(self.reserved_vacant)),
("retained", SdkValue::Int(self.retained)),
])
}
}

/// A Sandbox's record with its derived state and one page of its
/// Workspaces.
pub struct Got {
Expand All @@ -42,6 +89,9 @@ pub struct Got {
pub workspaces: Vec<ListedWorkspace>,
/// Why its machine did not answer, when it did not.
pub down_reason: Option<String>,
/// The structured counts behind `down_reason`, when the refusing
/// daemon's capacity notice carried them (#649).
pub down: Option<Down>,
/// The next page's position; `None` after the last.
pub next: Option<usize>,
/// Its daemon's control-capacity read model (#540), when a running
Expand Down Expand Up @@ -70,29 +120,34 @@ pub fn get(home: &Path, events: &mut Events<'_>, request: &GetRequest) -> Result
),
None => Ok(Ok(SdkValue::Null)),
};
let (all, down_reason, gone) = match listed {
Ok(Err(refusal)) if refusal.code == "NoSuchSandbox" => (Vec::new(), None, true),
let (all, down_reason, down, gone) = match listed {
Ok(Err(refusal)) if refusal.code == "NoSuchSandbox" => (Vec::new(), None, None, true),
Ok(workspaces) => match workspaces?.get("workspaces") {
Some(SdkValue::List(items)) => (
items.iter().map(|i| listed_workspace(name, i)).collect(),
None,
None,
false,
),
_ => (Vec::new(), None, false),
_ => (Vec::new(), None, None, false),
},
Err(Unreached(why)) => (Vec::new(), Some(why.message), false),
Err(Unreached(why)) => {
let down = Down::of(&why);
(Vec::new(), Some(why.message), down, false)
}
};
let (workspaces, next) = page(all, request.page);
let state = record.state(gone);
// best effort, bounded, and only for a Sandbox its daemon still serves:
// this read never fails a get (#540, #543)
let down = down_reason.is_some();
let daemon_capacity = capacity(&home, events, &record, gone, state, down);
let unreachable = down_reason.is_some();
let daemon_capacity = capacity(&home, events, &record, gone, state, unreachable);
Ok(Got {
state,
record,
workspaces,
down_reason,
down,
next,
daemon_capacity,
})
Expand Down
7 changes: 6 additions & 1 deletion docs/architecture/api/sdk/sandbox/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ unique across the home.
| --- | --- | --- | --- | --- |
| [`create`](create.md) | `project` (Project), `name` (Name), `machine?` (Name) | `image` (Text) | the Sandbox record with `state`; `forward_failed` when a direct daemon forward did not open | its machine: here or `sandbox.host` in the worker |
| [`list`](list.md) | `project?` (Project) | `page_size`, `page_pos` (Int) | `{ items: [record], next_page_pos }` | caller's process, home records only |
| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing |
| [`get`](get.md) | `name` (Name) | `page_size`, `page_pos` | the record with `state`, `workspaces`, `next_page_pos`, `down_reason` when its machine is down, `down` (partial capacity counts) when the refusal carried the daemon's notice, and `daemon_capacity` when its daemon answers the bounded capacity read | its machine or its daemon; starts nothing |
| [`close`](close.md) | `name` (Name) | `force` (Bool) | `{ kind: "Closed", sandbox, unknown, closed }`, or `{ closed, was_gone: true }`, or `{ closed, was, left? }` for a record with no id | its machine: here or `sandbox.host` in the worker |

The record is `SandboxRecord::to_sdk`
Expand All @@ -52,6 +52,11 @@ its `daemon_capacity` is the daemon's `backend.daemon.connection_failures`
read model, read best effort and bounded to 2 s on the connection the
Workspace list already lent
([`daemon/capacity.rs`](../../../../../control/crates/layerfs-sandbox/src/daemon/capacity.rs)).
When the Workspace list is refused at a full pool, `down` carries the
partial counts from the daemon's pre-handshake capacity notice: `code`,
`resource`, `pool_vacant`, `reserved_vacant` and `retained`, and no
`daemon_capacity`, whose missing fields would read as zero
(#649).

## Arguments and options

Expand Down
Loading