Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions control/crates/layerfs-sandbox/src/direct.rs
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,10 @@ fn served(home: &Path, record: &SandboxRecord, call: Call) -> Outcome {
/// The Workspaces forwarded Sandbox `record` holds, which its close sends
/// its worker (`held`): those its daemon still has, checked under the
/// folder's turn with no unmount in progress; when the daemon cannot be
/// reached, every Workspace recorded in the home folder.
/// reached, every Workspace recorded in the home folder. A Workspace whose
/// forced unmount left a Retained detach the kernel refused `EBUSY` is
/// re-attempted here (this process reaches the daemon directly) and the
/// close is refused `Busy` while its holder is still alive.
pub fn held(home: &Path, record: &SandboxRecord) -> Result<Vec<String>> {
let target = resolve::target(home, record.project().unwrap_or(""))?;
let host = host(home, record, &target, Message::Absent);
Expand All @@ -341,8 +344,31 @@ pub fn held(home: &Path, record: &SandboxRecord) -> Result<Vec<String>> {
.collect();
};
let _guard = layerfs_store::project::discovery::guard(&state.directory)?;
let recoverable = unmount_guard::check_all(&state)?;
let retained = unmount_guard::check_all(&state)?;
let recoverable = retained
.iter()
.map(|entry| entry.path.clone())
.collect::<Vec<_>>();
files::refresh_except(&state, &recoverable)?;
// A Retained unmount whose forced detach the kernel refused `EBUSY` is
// re-attempted here (this process reaches the daemon directly); a holder
// still alive refuses the close rather than deleting the data plane.
let still_held = unmount_guard::retry_held(&state, &retained)?;
if !still_held.is_empty() {
let held = still_held.join(", ");
let count = still_held.len();
return Err(SdkRefusal {
code: "Busy".to_owned(),
message: format!(
"Sandbox {} still holds {count} Workspace(s) whose unmount a live holder refuses: {held}; wait for the holder to exit, then close again",
record.name()
),
detail: SdkValue::map([(
"workspaces",
SdkValue::List(still_held.into_iter().map(SdkValue::Text).collect()),
)]),
});
}
let mounted = helpers::lock(&state.mounts)?
.keys()
.filter(|path| !recoverable.contains(path))
Expand Down
36 changes: 32 additions & 4 deletions control/crates/layerfs-sandbox/src/lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,9 +252,14 @@ pub fn create(host: &Host, named: Option<&str>) -> Outcome {
/// `sandbox.close(sandbox, { force })`: stops Sandbox `sandbox` now, under
/// its folder's guard. Refused `NoSuchSandbox` when nothing is recorded,
/// and, unless `force`, `MountsActive` while a Workspace is mounted in it.
/// Forced (O31), it stops the Sandbox through its backend anyway and
/// answers each Workspace it could not drain as `Unknown` (`unknown`: their
/// paths): their uncommitted changes and processes end with the Sandbox.
/// A Workspace whose forced unmount left Retained custody whose one detach
/// the kernel refused `EBUSY` is re-attempted as a clean detach first; while
/// its holder is still alive the close is refused `Busy` naming those
/// Workspaces instead of deleting the Sandbox data plane (which is
/// [`destroy`]'s explicit job). Forced (O31), it stops the Sandbox through
/// its backend anyway and answers each Workspace it could not drain as
/// `Unknown` (`unknown`: their paths): their uncommitted changes and
/// processes end with the Sandbox.
/// `held` names Workspaces the caller records (a Sandbox it reaches
/// directly, #535 G4), counted as mounted.
/// Stop, then delete, each attempted once: a failed stop is not followed by
Expand Down Expand Up @@ -286,13 +291,36 @@ pub fn close(host: &Host, force: bool, held: &[String]) -> Outcome {
// its running commands (D-10): `Busy`, or with `force` cancelled first
let unreached = &mut crate::exec::cancel::unreached;
crate::exec::cancel::settle(&state.directory, &id, None, force, unreached)?;
let recoverable = crate::unmount_guard::check_all(&state)?;
let retained = crate::unmount_guard::check_all(&state)?;
let recoverable = retained
.iter()
.map(|entry| entry.path.clone())
.collect::<Vec<_>>();
if !force {
// A daemon that refuses all control connections must refuse this close
// with its exact cause in bounded time, not hold the `op` intent while
// the SDK's ordinary 60 s admission wait runs out. One shared deadline
// bounds all Workspace probes; N Workspaces cannot multiply it.
crate::files::refresh_except_within(&state, &recoverable, CLOSE_STATUS_WAIT)?;
// A Retained unmount whose forced detach the kernel refused `EBUSY` is
// re-attempted as a clean detach first; a holder still alive refuses the
// close rather than deleting the Sandbox data plane (`sandbox.destroy`
// is the explicit data-plane-only delete).
let still_held = crate::unmount_guard::retry_held(&state, &retained)?;
if !still_held.is_empty() {
let held = still_held.join(", ");
let count = still_held.len();
return Err(SdkRefusal {
code: "Busy".to_owned(),
message: format!(
"Sandbox {id} still holds {count} Workspace(s) whose unmount a live holder refuses: {held}; wait for the holder to exit, then close again"
),
detail: SdkValue::map([(
"workspaces",
SdkValue::List(still_held.into_iter().map(SdkValue::Text).collect()),
)]),
});
}
}
// `force` is the recovery path: it skips the daemon status probe and stops
// the runtime anyway, reporting every Workspace it could not drain as
Expand Down
53 changes: 51 additions & 2 deletions control/crates/layerfs-sandbox/src/unmount_guard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
//! admission and shared-record cleanup, never during the daemon's helper wait.
//! Mount paths are unique within one daemon lifetime; Sandbox removal clears
//! the terminal records before that directory can be reused.
use crate::daemon::values;
use crate::files as discovery;
use crate::registry::SandboxState;
use crate::Unmounted;
use layerfs_bridge::control::{
AbortDisposition, DetachDisposition, SdkRefusal, SdkValue, TeardownStage, WorkspaceToken,
};
Expand Down Expand Up @@ -232,8 +234,18 @@ pub fn token(state: &SandboxState, path: &str) -> Result<WorkspaceToken, SdkRefu
format!("{}:{path} is not mounted", state.id),
))
}
/// A Workspace a terminal unmount left Retained, and whether its recorded
/// custody is a forced detach the kernel refused `EBUSY` (`retryable`),
/// which a later terminal call may re-attempt once the holder is gone.
pub struct Retained {
/// The Workspace's mount path.
pub path: String,
/// Whether a later terminal call may re-attempt its detach.
pub retryable: bool,
}

/// Closing a running Sandbox may not race a registered terminal operation.
pub fn check_all(state: &SandboxState) -> Result<Vec<String>, SdkRefusal> {
pub fn check_all(state: &SandboxState) -> Result<Vec<Retained>, SdkRefusal> {
let mut recoverable = Vec::new();
for entry in unmounts(state)? {
let entry = entry.map_err(failed)?;
Expand All @@ -249,13 +261,50 @@ pub fn check_all(state: &SandboxState) -> Result<Vec<String>, SdkRefusal> {
}
if let Some(value) = inspect(&entry.path())? {
if value.get_text("state") == Some("Retained") {
recoverable.push(layerfs_home::helpers::text(&value, "path")?.to_owned());
recoverable.push(Retained {
path: layerfs_home::helpers::text(&value, "path")?.to_owned(),
retryable: retry(&value),
});
}
}
}
}
Ok(recoverable)
}

/// Whether a terminal reply is `Missing`: the daemon already unmounted the
/// Workspace (an earlier retry detached it, so its stale record remains).
fn absent(failure: &crate::OperationFailure) -> bool {
matches!(
&failure.cause,
crate::OperationCause::Remote(refusal)
if refusal.code == layerfs_bridge::control::ControlCode::Missing
)
}

/// Re-attempts the clean detach of every Retained retryable Workspace in
/// `retained` (one normal unmount on the daemon, which re-attempts only the
/// recorded `EBUSY` detach and never repeats the abort write). Answers the
/// paths whose detach is still refused — their holder is still alive — which
/// a close must refuse rather than delete; any other failure is returned as
/// its refusal.
pub fn retry_held(state: &SandboxState, retained: &[Retained]) -> Result<Vec<String>, SdkRefusal> {
let mut held = Vec::new();
for entry in retained.iter().filter(|entry| entry.retryable) {
let token = token(state, &entry.path)?;
let mut control = crate::daemon::trace::control(&state.controls)?;
match crate::internal::workspace(&mut control).unmount_checked(token, false) {
Ok(Unmounted::Closed) => {}
Ok(Unmounted::Dirty(_)) => unreachable!("a retryable detach skips the dirty check"),
Err(failure) if absent(&failure) => {}
Err(failure) if retryable(&failure) => held.push(entry.path.clone()),
Err(failure) => {
return Err(values::refusal(&failure).unwrap_or_else(|| failed(failure)))
}
}
}
Ok(held)
}
/// Called under the Sandbox guard, after all before-effect checks.
pub fn begin(
state: &SandboxState,
Expand Down
Loading