diff --git a/control/crates/layerfs-sandbox/src/direct.rs b/control/crates/layerfs-sandbox/src/direct.rs index 5f702985a8..5d4494fe05 100644 --- a/control/crates/layerfs-sandbox/src/direct.rs +++ b/control/crates/layerfs-sandbox/src/direct.rs @@ -331,7 +331,10 @@ fn served(home: &Path, record: &SandboxRecord, call: Call) -> Outcome { /// The Workspaces forwarded Sandbox `record` holds, which its close sends /// its worker (`held`): those its daemon still has, checked under the /// folder's turn with no unmount in progress; when the daemon cannot be -/// reached, every Workspace recorded in the home folder. +/// reached, every Workspace recorded in the home folder. A Workspace whose +/// forced unmount left a Retained detach the kernel refused `EBUSY` is +/// re-attempted here (this process reaches the daemon directly) and the +/// close is refused `Busy` while its holder is still alive. pub fn held(home: &Path, record: &SandboxRecord) -> Result> { let target = resolve::target(home, record.project().unwrap_or(""))?; let host = host(home, record, &target, Message::Absent); @@ -341,8 +344,31 @@ pub fn held(home: &Path, record: &SandboxRecord) -> Result> { .collect(); }; let _guard = layerfs_store::project::discovery::guard(&state.directory)?; - let recoverable = unmount_guard::check_all(&state)?; + let retained = unmount_guard::check_all(&state)?; + let recoverable = retained + .iter() + .map(|entry| entry.path.clone()) + .collect::>(); files::refresh_except(&state, &recoverable)?; + // A Retained unmount whose forced detach the kernel refused `EBUSY` is + // re-attempted here (this process reaches the daemon directly); a holder + // still alive refuses the close rather than deleting the data plane. + let still_held = unmount_guard::retry_held(&state, &retained)?; + if !still_held.is_empty() { + let held = still_held.join(", "); + let count = still_held.len(); + return Err(SdkRefusal { + code: "Busy".to_owned(), + message: format!( + "Sandbox {} still holds {count} Workspace(s) whose unmount a live holder refuses: {held}; wait for the holder to exit, then close again", + record.name() + ), + detail: SdkValue::map([( + "workspaces", + SdkValue::List(still_held.into_iter().map(SdkValue::Text).collect()), + )]), + }); + } let mounted = helpers::lock(&state.mounts)? .keys() .filter(|path| !recoverable.contains(path)) diff --git a/control/crates/layerfs-sandbox/src/lifecycle.rs b/control/crates/layerfs-sandbox/src/lifecycle.rs index 1fbcfcb9ce..e18063bf05 100644 --- a/control/crates/layerfs-sandbox/src/lifecycle.rs +++ b/control/crates/layerfs-sandbox/src/lifecycle.rs @@ -252,9 +252,14 @@ pub fn create(host: &Host, named: Option<&str>) -> Outcome { /// `sandbox.close(sandbox, { force })`: stops Sandbox `sandbox` now, under /// its folder's guard. Refused `NoSuchSandbox` when nothing is recorded, /// and, unless `force`, `MountsActive` while a Workspace is mounted in it. -/// Forced (O31), it stops the Sandbox through its backend anyway and -/// answers each Workspace it could not drain as `Unknown` (`unknown`: their -/// paths): their uncommitted changes and processes end with the Sandbox. +/// A Workspace whose forced unmount left Retained custody whose one detach +/// the kernel refused `EBUSY` is re-attempted as a clean detach first; while +/// its holder is still alive the close is refused `Busy` naming those +/// Workspaces instead of deleting the Sandbox data plane (which is +/// [`destroy`]'s explicit job). Forced (O31), it stops the Sandbox through +/// its backend anyway and answers each Workspace it could not drain as +/// `Unknown` (`unknown`: their paths): their uncommitted changes and +/// processes end with the Sandbox. /// `held` names Workspaces the caller records (a Sandbox it reaches /// directly, #535 G4), counted as mounted. /// Stop, then delete, each attempted once: a failed stop is not followed by @@ -286,13 +291,36 @@ pub fn close(host: &Host, force: bool, held: &[String]) -> Outcome { // its running commands (D-10): `Busy`, or with `force` cancelled first let unreached = &mut crate::exec::cancel::unreached; crate::exec::cancel::settle(&state.directory, &id, None, force, unreached)?; - let recoverable = crate::unmount_guard::check_all(&state)?; + let retained = crate::unmount_guard::check_all(&state)?; + let recoverable = retained + .iter() + .map(|entry| entry.path.clone()) + .collect::>(); if !force { // A daemon that refuses all control connections must refuse this close // with its exact cause in bounded time, not hold the `op` intent while // the SDK's ordinary 60 s admission wait runs out. One shared deadline // bounds all Workspace probes; N Workspaces cannot multiply it. crate::files::refresh_except_within(&state, &recoverable, CLOSE_STATUS_WAIT)?; + // A Retained unmount whose forced detach the kernel refused `EBUSY` is + // re-attempted as a clean detach first; a holder still alive refuses the + // close rather than deleting the Sandbox data plane (`sandbox.destroy` + // is the explicit data-plane-only delete). + let still_held = crate::unmount_guard::retry_held(&state, &retained)?; + if !still_held.is_empty() { + let held = still_held.join(", "); + let count = still_held.len(); + return Err(SdkRefusal { + code: "Busy".to_owned(), + message: format!( + "Sandbox {id} still holds {count} Workspace(s) whose unmount a live holder refuses: {held}; wait for the holder to exit, then close again" + ), + detail: SdkValue::map([( + "workspaces", + SdkValue::List(still_held.into_iter().map(SdkValue::Text).collect()), + )]), + }); + } } // `force` is the recovery path: it skips the daemon status probe and stops // the runtime anyway, reporting every Workspace it could not drain as diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index e69f38d3c7..21bb5ba4d6 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -3,8 +3,10 @@ //! admission and shared-record cleanup, never during the daemon's helper wait. //! Mount paths are unique within one daemon lifetime; Sandbox removal clears //! the terminal records before that directory can be reused. +use crate::daemon::values; use crate::files as discovery; use crate::registry::SandboxState; +use crate::Unmounted; use layerfs_bridge::control::{ AbortDisposition, DetachDisposition, SdkRefusal, SdkValue, TeardownStage, WorkspaceToken, }; @@ -232,8 +234,18 @@ pub fn token(state: &SandboxState, path: &str) -> Result Result, SdkRefusal> { +pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { let mut recoverable = Vec::new(); for entry in unmounts(state)? { let entry = entry.map_err(failed)?; @@ -249,13 +261,50 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { } if let Some(value) = inspect(&entry.path())? { if value.get_text("state") == Some("Retained") { - recoverable.push(layerfs_home::helpers::text(&value, "path")?.to_owned()); + recoverable.push(Retained { + path: layerfs_home::helpers::text(&value, "path")?.to_owned(), + retryable: retry(&value), + }); } } } } Ok(recoverable) } + +/// Whether a terminal reply is `Missing`: the daemon already unmounted the +/// Workspace (an earlier retry detached it, so its stale record remains). +fn absent(failure: &crate::OperationFailure) -> bool { + matches!( + &failure.cause, + crate::OperationCause::Remote(refusal) + if refusal.code == layerfs_bridge::control::ControlCode::Missing + ) +} + +/// Re-attempts the clean detach of every Retained retryable Workspace in +/// `retained` (one normal unmount on the daemon, which re-attempts only the +/// recorded `EBUSY` detach and never repeats the abort write). Answers the +/// paths whose detach is still refused — their holder is still alive — which +/// a close must refuse rather than delete; any other failure is returned as +/// its refusal. +pub fn retry_held(state: &SandboxState, retained: &[Retained]) -> Result, SdkRefusal> { + let mut held = Vec::new(); + for entry in retained.iter().filter(|entry| entry.retryable) { + let token = token(state, &entry.path)?; + let mut control = crate::daemon::trace::control(&state.controls)?; + match crate::internal::workspace(&mut control).unmount_checked(token, false) { + Ok(Unmounted::Closed) => {} + Ok(Unmounted::Dirty(_)) => unreachable!("a retryable detach skips the dirty check"), + Err(failure) if absent(&failure) => {} + Err(failure) if retryable(&failure) => held.push(entry.path.clone()), + Err(failure) => { + return Err(values::refusal(&failure).unwrap_or_else(|| failed(failure))) + } + } + } + Ok(held) +} /// Called under the Sandbox guard, after all before-effect checks. pub fn begin( state: &SandboxState,