Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/crates/layerfs-api/tests/sandbox_capacity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
use layerfs_api::home::{self, paths, write_record};
use layerfs_bridge::control::{
AdmissionModel, Answer, Call, DaemonPhase, DaemonStatus, Reply, Request, RetainedModel,
SdkValue, CAPACITY_OPERATION,
SdkValue, OPERATION,
};
use layerfs_bridge::duplex;
use std::net::{SocketAddr, TcpListener};
Expand Down Expand Up @@ -107,7 +107,7 @@ fn serve(stream: std::net::TcpStream, model: &SdkValue, stall: bool) {
Request::Sdk(sdk) if sdk.method == "sandbox.list_workspaces" => {
Reply::Sdk(no_workspaces())
}
Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => {
Request::Sdk(sdk) if sdk.method == OPERATION => {
if stall {
// read but never answered: the read model's own fence, not
// a closed socket, ends its wait
Expand Down
4 changes: 2 additions & 2 deletions control/crates/layerfs-control/tests/doctor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
//! error.
use layerfs_bridge::control::{
AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request,
RetainedModel, SdkRefusal, SdkValue, CAPACITY_OPERATION,
RetainedModel, SdkRefusal, SdkValue, OPERATION,
};
use layerfs_bridge::duplex;
use layerfs_control::doctor::doctor;
Expand Down Expand Up @@ -727,7 +727,7 @@ fn capacity_daemon(reply: Reply) -> (SocketAddr, std::thread::JoinHandle<()>) {
let Request::Sdk(sdk) = call.request else {
panic!("expected the capacity read model");
};
assert_eq!(sdk.method, CAPACITY_OPERATION);
assert_eq!(sdk.method, OPERATION);
channel
.send
.send(&Answer { id, reply }.encode().unwrap())
Expand Down
2 changes: 1 addition & 1 deletion control/crates/layerfs-sandbox/src/daemon/capacity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use std::time::{Duration, Instant};

/// The daemon's operator read model of control admission and retained
/// connection failures.
pub const OPERATION: &str = layerfs_bridge::control::CAPACITY_OPERATION;
pub const OPERATION: &str = layerfs_bridge::control::OPERATION;

/// How long one daemon query may take: connecting, its Hello and its one
/// exchange. Short on purpose: a reader must not stall behind a daemon that
Expand Down
2 changes: 1 addition & 1 deletion control/crates/layerfs-sandbox/src/daemon/operator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pub fn call(control: &mut Control, method: &str, args: SdkValue) -> SdkResult {
match method {
// The daemon's control admission and bounded retained-failure store
// (#540): an operator read model, never a product operation.
layerfs_bridge::control::CAPACITY_OPERATION => {
layerfs_bridge::control::OPERATION => {
crate::daemon::call::exchange_call(control, method, args)
}
"sandbox.list_workspaces" => crate::daemon::call::exchange_call(control, method, args),
Expand Down
2 changes: 1 addition & 1 deletion control/crates/layerfs-sandbox/src/daemon/pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ fn not_admitted(refusal: &SdkRefusal) -> Option<(NotAdmitted, &'static str)> {
/// precise form: it names the resource and the retained record count.
fn notice_refusal(notice: AdmissionNotice) -> SdkRefusal {
match notice.reason {
AdmissionReason::NoPoolSlot | AdmissionReason::ReservedBusy => SdkRefusal {
AdmissionReason::NoPoolSlot => SdkRefusal {
code: "Capacity".to_owned(),
message: format!(
"the Sandbox's daemon has no vacant control slot: {} pool slot(s) and {} reserved slot(s) are vacant, with {} retained failure record(s). The call was not admitted; retrying is safe",
Expand Down
2 changes: 1 addition & 1 deletion control/crates/layerfs-sandbox/src/host/call.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ pub const OPS: [(Op, &str); 10] = [
(Op::SandboxCreate, "sandbox.create"),
(Op::SandboxClose, "sandbox.close"),
(Op::ListWorkspaces, "sandbox.list_workspaces"),
(Op::Capacity, layerfs_bridge::control::CAPACITY_OPERATION),
(Op::Capacity, layerfs_bridge::control::OPERATION),
(Op::Mount, "workspace.mount"),
(Op::Exec, "workspace.exec"),
(Op::ExecCancel, "workspace.exec_cancel"),
Expand Down
6 changes: 3 additions & 3 deletions control/crates/layerfs-sandbox/tests/capacity_read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
//! accepts, and counts what it saw.
use layerfs_bridge::control::{
AdmissionModel, Answer, Call, CapacityModel, DaemonPhase, DaemonStatus, Reply, Request,
RetainedModel, SdkValue, CAPACITY_OPERATION,
RetainedModel, SdkValue, OPERATION,
};
use layerfs_bridge::duplex;
use layerfs_home::home::paths;
Expand Down Expand Up @@ -138,7 +138,7 @@ fn serve(stream: TcpStream, answer: Behavior, seen: &Seen) {
seen.ended.store(true, Ordering::SeqCst);
Reply::SessionEnded
}
Request::Sdk(sdk) if sdk.method == CAPACITY_OPERATION => {
Request::Sdk(sdk) if sdk.method == OPERATION => {
seen.queries.fetch_add(1, Ordering::SeqCst);
if matches!(answer, Behavior::Trickle) {
if let Some(raw) = raw.as_mut() {
Expand Down Expand Up @@ -337,7 +337,7 @@ fn a_quarantined_connection_attempts_nothing() {
lease.close_handle().unwrap().close().unwrap();
// the call a pooled lease would make next: refused before sending, so it
// is not an attempt whose effect is unknown
let error = operator::call(&mut lease, CAPACITY_OPERATION, SdkValue::Null).unwrap_err();
let error = operator::call(&mut lease, OPERATION, SdkValue::Null).unwrap_err();
match error {
SdkError::Operation(failure) => match failure.cause {
OperationCause::Exchange(channel) => {
Expand Down
3 changes: 1 addition & 2 deletions engine/crates/layerfs-bridge/src/control.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,7 @@ fn boolean(value: u8) -> Result<bool, ControlError> {
}

pub use crate::control_capacity::{
AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel,
OPERATION as CAPACITY_OPERATION,
AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel, OPERATION,
};
pub use crate::control_forced::{
AbortDisposition, CommitKnowledge, DetachDisposition, ForceUnmounted, ForcedCleanup,
Expand Down
4 changes: 0 additions & 4 deletions engine/crates/layerfs-bridge/src/duplex/admission.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,13 @@ pub enum AdmissionReason {
NoPoolSlot = 1,
/// The daemon is still starting and admits no control yet.
Starting = 2,
/// The reserved slot is in use and only pool capacity was requested.
ReservedBusy = 3,
}

impl AdmissionReason {
fn of(byte: u8) -> Option<Self> {
match byte {
1 => Some(Self::NoPoolSlot),
2 => Some(Self::Starting),
3 => Some(Self::ReservedBusy),
_ => None,
}
}
Expand All @@ -44,7 +41,6 @@ impl AdmissionReason {
match self {
Self::NoPoolSlot => "no-pool-slot",
Self::Starting => "starting",
Self::ReservedBusy => "reserved-busy",
}
}
}
Expand Down
136 changes: 69 additions & 67 deletions engine/crates/layerfs-daemon/src/application/connection.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Bounded real authenticated connection workers and retained original failures.
use super::{
owner::{AdmissionClass, Finished, SlotClass, SlotTicket},
retained::RetainedFailure,
owner::{disposition, AdmissionClass, Finished, SlotClass, SlotCounts, SlotTicket},
retained::RetainedSnapshot,
Application, ApplicationError, ConnectionFailure,
};
use layerfs_bridge::{
Expand Down Expand Up @@ -80,15 +80,12 @@ impl SlotGuard {
return;
}
self.done = true;
// Build the bounded record before taking the state lock.
let prebuilt = result.as_ref().err().map(|failure| {
RetainedFailure::describe(
failure,
self.ticket.class,
self.ticket.generation,
&self.owner,
)
});
let now = Instant::now();
// Classify and build the bounded record before taking the state lock.
let classified = result
.as_ref()
.err()
.map(|failure| disposition(failure, self.ticket, &self.owner, now));
// Poison recovery here only releases the slot and records; normal
// admission still reports `Poisoned` rather than trusting corrupt state.
let finished = {
Expand All @@ -97,9 +94,14 @@ impl SlotGuard {
.state
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let finished = state
.slots
.finish(self.ticket, result, prebuilt, Instant::now());
let finished = state.slots.finish(self.ticket, result, classified);
let finished = match finished {
Finished::Retained(record) => {
let recorded = state.retained.record(*record, now);
Finished::Retained(Box::new(recorded))
}
other => other,
};
self.application.changed.notify_all();
finished
};
Expand Down Expand Up @@ -161,8 +163,8 @@ impl Application {
Some(ticket) => ticket,
None => {
let counts = state.slots.counts();
let retained = state.slots.visible();
state.slots.reap(Instant::now());
let retained = state.retained.len();
state.retained.reap(Instant::now());
let reason = if state.startup.phase != DaemonPhase::ControlReady {
AdmissionReason::Starting
} else {
Expand Down Expand Up @@ -332,9 +334,9 @@ impl Application {
let bodies = super::bodies::Bodies::of(&call.request, &mut connection.receive);
let (outcome, diagnostic_error, telemetry) =
if matches!(call.request, Request::Observed { .. }) {
self.observed_control(&call, slot)
self.observed_control(&call, slot, ticket.class)
} else {
(self.control(&call, slot), None, None)
(self.control(&call, ticket.class), None, None)
};
if let Err(cause) = bodies.finish() {
return Err(ConnectionFailure {
Expand Down Expand Up @@ -446,9 +448,9 @@ impl Application {
fn control(
&self,
call: &Call,
slot: usize,
class: SlotClass,
) -> Result<crate::control::Success, crate::control::Failure> {
self.control_operation(&call.request, slot)
self.control_operation(&call.request, class)
}
/// After the daemon reported ready (a `Hello` answered ControlReady was
/// sent), opens one idle Store reader on a thread of its own, so the
Expand Down Expand Up @@ -494,7 +496,7 @@ impl Application {
pub(super) fn control_operation(
&self,
request: &Request,
slot: usize,
class: SlotClass,
) -> Result<crate::control::Success, crate::control::Failure> {
use crate::control::{Failure, Success};
use layerfs_bridge::control::{ControlCode, DaemonPhase};
Expand All @@ -503,13 +505,11 @@ impl Application {
}
if let Request::Hello(request) = request {
return self
.hello(*request, slot)
.hello(*request, class)
.map(|status| Success::reply(Reply::Hello(status)));
}
let state = self.state.lock().map_err(|_| Failure::Poisoned)?;
if slot == usize::from(self.setup.limits.connections)
&& state.startup.phase != DaemonPhase::ControlReady
{
if class == SlotClass::Reserved && state.startup.phase != DaemonPhase::ControlReady {
return Err(Failure::Rejected(
ControlCode::Capacity,
"reserved installer slot",
Expand All @@ -521,7 +521,7 @@ impl Application {
// Operator-visible retained failures live with the application, not
// with the Store control Service, so answer this one here.
if let Request::Sdk(call) = request {
if call.method == CONNECTION_FAILURES {
if call.method == layerfs_bridge::control::OPERATION {
return self
.connection_failures()
.map(|value| Success::reply(Reply::Sdk(value)));
Expand All @@ -545,47 +545,53 @@ impl Application {
&self,
) -> Result<layerfs_bridge::control::SdkValue, crate::control::Failure> {
use crate::control::Failure;
use layerfs_bridge::control::{
AdmissionModel, CapacityModel, RetainedFailureModel, RetainedModel,
};
let counts = self.connection_slots().map_err(|_| Failure::Poisoned)?;
let snapshot = self
.retained_connection_failures()
.map_err(|_| Failure::Poisoned)?;
let failures = snapshot
.failures
.iter()
.map(|failure| RetainedFailureModel {
slot: failure.slot as i64,
class: failure.class.to_owned(),
generation: failure.generation as i64,
owner: failure.owner.clone(),
received_bytes: failure.received_bytes.map(|bytes| bytes as i64),
request: failure.request.map(str::to_owned),
reply_sent: failure.reply_sent,
cause: failure.cause.clone(),
fence_error: failure.fence_error.clone(),
diagnostic_error: failure.diagnostic_error.clone(),
age_ms: failure.age.as_millis() as i64,
expires_in_ms: failure.expires_in.as_millis() as i64,
});
Ok(CapacityModel {
admission: AdmissionModel {
pool_vacant: counts.pool_vacant as i64,
pool_running: counts.pool_running as i64,
reserved_vacant: counts.reserved_vacant as i64,
reserved_running: counts.reserved_running as i64,
},
retained: RetainedModel {
limit: snapshot.limit as i64,
ttl_ms: snapshot.ttl.as_millis() as i64,
count: snapshot.failures.len() as i64,
dropped: snapshot.dropped as i64,
recorded: snapshot.recorded as i64,
failures: failures.collect(),
},
}
.to_sdk())
Ok(capacity_report(counts, snapshot).to_sdk())
}
}

/// The daemon's control-capacity read model from its slot counts and its
/// bounded retained-failure snapshot (#540). Pure: no lock and no store.
fn capacity_report(
counts: SlotCounts,
snapshot: RetainedSnapshot,
) -> layerfs_bridge::control::CapacityModel {
use layerfs_bridge::control::{AdmissionModel, RetainedFailureModel, RetainedModel};
let failures = snapshot
.failures
.iter()
.map(|failure| RetainedFailureModel {
slot: failure.slot as i64,
class: failure.class.to_owned(),
generation: failure.generation as i64,
owner: failure.owner.clone(),
received_bytes: failure.received_bytes.map(|bytes| bytes as i64),
request: failure.request.map(str::to_owned),
reply_sent: failure.reply_sent,
cause: failure.cause.clone(),
fence_error: failure.fence_error.clone(),
diagnostic_error: failure.diagnostic_error.clone(),
age_ms: failure.age.as_millis() as i64,
expires_in_ms: failure.expires_in.as_millis() as i64,
});
layerfs_bridge::control::CapacityModel {
admission: AdmissionModel {
pool_vacant: counts.pool_vacant as i64,
pool_running: counts.pool_running as i64,
reserved_vacant: counts.reserved_vacant as i64,
reserved_running: counts.reserved_running as i64,
},
retained: RetainedModel {
limit: snapshot.limit as i64,
ttl_ms: snapshot.ttl.as_millis() as i64,
count: snapshot.failures.len() as i64,
dropped: snapshot.dropped as i64,
recorded: snapshot.recorded as i64,
failures: failures.collect(),
},
}
}

Expand All @@ -605,7 +611,3 @@ fn reported_ready(outcome: &Result<crate::control::Success, crate::control::Fail
Reply::Hello(status) if status.phase == layerfs_bridge::control::DaemonPhase::ControlReady
))
}

/// Operator/control method for the daemon's control admission and retained
/// connection failures.
pub(super) const CONNECTION_FAILURES: &str = layerfs_bridge::control::CAPACITY_OPERATION;
5 changes: 3 additions & 2 deletions engine/crates/layerfs-daemon/src/application/diagnostics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub(super) mod resources;
mod schema;
pub(crate) use schema::{commit_parts, namespace_values, storage_values};

use super::owner::SlotClass;
use super::Application;
use crate::control::{Failure, Success};
use layerfs_bridge::control::{Call, ControlCode, Request, SdkValue};
Expand Down Expand Up @@ -142,7 +143,7 @@ pub(super) type Observed = (
Option<SdkValue>,
);
impl Application {
pub(super) fn observed_control(&self, call: &Call, slot: usize) -> Observed {
pub(super) fn observed_control(&self, call: &Call, slot: usize, class: SlotClass) -> Observed {
let operation = match call.request.without_observation() {
Ok(operation) => operation,
Err(_) => {
Expand Down Expand Up @@ -195,7 +196,7 @@ impl Application {
let saves_before = saves.as_ref().map(|store| store.save_counters());
let observed_before = observing.elapsed();
let recording = traced.then(crate::telemetry::Recording::start);
let outcome = self.control_operation(operation, slot);
let outcome = self.control_operation(operation, class);
let reports = recording.map(crate::telemetry::Recording::finish);
let observed_after = std::time::Instant::now();
let sql = sessions
Expand Down
18 changes: 18 additions & 0 deletions engine/crates/layerfs-daemon/src/application/internals.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
//! Pure control-slot and retained-failure values, observed by the integration
//! tests in `tests/`.
//!
//! `#[doc(hidden)]` keeps this module out of rustdoc; it is not access
//! control. The values are the daemon's state machine with no I/O seam, so
//! the only place a test can reach them is a re-export: product `src/` may
//! not carry a test-only `cfg` attribute (the product-boundary guard rejects
//! it). Keep the documented `application` exports to the startup API
//! (`Application`, `ApplicationError`, `ConnectionFailure`); anything a test
//! reads belongs here, behind this module's one name.

#[doc(hidden)]
pub use super::owner::{
disposition, AdmissionClass, Disposition, Finished, SlotClass, SlotCounts, SlotTicket, Slots,
RESERVED_SLOTS,
};
#[doc(hidden)]
pub use super::retained::{RetainedFailure, RetainedFailures, RetainedSnapshot, LIMIT, TTL};
Loading