From 38af9c026dedcd2699c26bb53d74b777bf50df97 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:41:08 +0800 Subject: [PATCH 01/14] fix(daemon): reap timed-out unmount helpers and complete detached teardown When an NFS unmount helper exceeds its wait budget and remains in kernel wait past the post-KILL grace window, start a bounded background reaper thread on Helpers (and darwin force_unmount) that polls try_wait() until the retained queue empties and records the observed exit status instead of leaving a zombie child until the next helper launch. Verify the mount table after a failed or interrupted unmount/force helper when the mount was tracked at Attach: if the mount has already disappeared from the mount table, proceed with export teardown (stopping and joining the NFS Server, releasing Kernel holdings, revoking the engine mount, and closing the Workspace) rather than unconditionally returning Retained(Detach). Keep conservative Retained(Detach) custody when the mount remains present, and allow explicit ForceUnmount to complete teardown of an unforced ExportKept(Detach) entry while reporting real Server loop exit and join counts. Ref: layerfs-project#136 Production LOC: 156,309 -> 156,485 (delta +176) Method: python3 tools/production_loc.py (58c548b3e -> HEAD) Subtotals: core 39,749 -> 39,749 (delta 0); engine 73,079 -> 73,215 (delta +136); control 38,511 -> 38,551 (delta +40); api 4,970 -> 4,970 (delta 0); reference absent (retired R9c; last counted 65,417 at tree 498dd1917812); dsh 0 -> 0 (delta 0). --- .../src/local/os/macos/darwin.rs | 58 +++- .../crates/layerfs-sandbox/src/registry.rs | 8 +- .../layerfs-sandbox/src/unmount_guard.rs | 8 + .../layerfs-sandbox/src/workspace/serve.rs | 12 +- .../layerfs-daemon/src/control/export.rs | 137 +++++---- .../src/control/export_helper.rs | 151 +++++++-- .../layerfs-daemon/src/control/force.rs | 3 + engine/crates/layerfs-nfs/src/server.rs | 15 +- .../layerfs-nfs/tests/helper_deadline.rs | 291 +++++++++++++++++- 9 files changed, 573 insertions(+), 110 deletions(-) diff --git a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs index 3ac6af51bb..d76fe26104 100644 --- a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs +++ b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs @@ -8,7 +8,10 @@ use std::{ io::{self, Read}, path::{Path, PathBuf}, process::{Child, Command, Stdio}, - sync::Mutex, + sync::{ + atomic::{AtomicBool, Ordering}, + Mutex, + }, time::{Duration, Instant}, }; @@ -48,13 +51,18 @@ fn mount_point(line: &str) -> Option { Some(PathBuf::from(point)) } +static RETAINED: Mutex> = Mutex::new(Vec::new()); +static REAPING: AtomicBool = AtomicBool::new(false); + +fn reap_retained() -> bool { + let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); + held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)) | Err(_))); + held.is_empty() +} + /// Forces the mount at `point` off once; a point no longer mounted is done. pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { - static RETAINED: Mutex> = Mutex::new(Vec::new()); - { - let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); - held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); - } + reap_retained(); let mut child = Command::new(UMOUNT) .arg("-f") .arg(point) @@ -98,21 +106,39 @@ pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { let reaped = loop { unmount_output(&mut stderr, &mut output); match child.try_wait() { - Ok(Some(_)) => break true, + Ok(Some(status)) => break Some(status), Ok(None) if Instant::now() < deadline => { std::thread::sleep(Duration::from_millis(5)); } - Ok(None) | Err(_) => break false, + Ok(None) | Err(_) => break None, } }; - let disposition = if reaped { - "exit reaped" - } else { - RETAINED - .lock() - .unwrap_or_else(|error| error.into_inner()) - .push(child); - "exit not observed after 5s; child custody retained (possible kernel wait)" + let disposition = match reaped { + Some(status) => format!("exit reaped ({status})"), + None => { + RETAINED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + if !REAPING.swap(true, Ordering::AcqRel) + && std::thread::Builder::new() + .name("layerfs-darwin-reap".into()) + .spawn(|| loop { + if reap_retained() { + REAPING.store(false, Ordering::Release); + if reap_retained() || REAPING.swap(true, Ordering::AcqRel) { + break; + } + } + std::thread::sleep(Duration::from_millis(20)); + }) + .is_err() + { + REAPING.store(false, Ordering::Release); + } + "exit not observed after 5s; child custody retained (possible kernel wait)" + .into() + } }; return Err(io::Error::new( error.kind(), diff --git a/control/crates/layerfs-sandbox/src/registry.rs b/control/crates/layerfs-sandbox/src/registry.rs index 6864944940..2e4edc064b 100644 --- a/control/crates/layerfs-sandbox/src/registry.rs +++ b/control/crates/layerfs-sandbox/src/registry.rs @@ -106,11 +106,18 @@ impl SandboxState { /// The token of the Workspace mounted at `path`. pub fn token(&self, path: &str) -> Result { + crate::unmount_guard::check(self, path)?; + self.mounted(path, |mount| mount.workspace.bound.token) + } + + /// The token of the Workspace at `path` after unmount admission checked it. + pub fn unmount_token(&self, path: &str) -> Result { self.mounted(path, |mount| mount.workspace.bound.token) } /// The kernel directory of the Workspace mounted at `path`. pub fn directory(&self, path: &str) -> Result { + crate::unmount_guard::check(self, path)?; self.mounted(path, |mount| mount.workspace.ready.directory.clone()) } @@ -120,7 +127,6 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { - crate::unmount_guard::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index 66f25d91d3..b733bc4ca2 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -51,6 +51,7 @@ pub struct Attempt { _lease: Lease, path: PathBuf, started: i64, + forced: bool, } fn result(value: &SdkValue) -> Outcome { if value.get_text("state") == Some("Unmounted") { @@ -220,11 +221,16 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { } Ok(recoverable) } +pub fn forceable(value: &SdkValue) -> bool { + value.get_text("state") == Some("Retained") + && value.get("forced") != Some(&SdkValue::Bool(true)) +} /// Called under the Sandbox guard, after all before-effect checks. pub fn begin( state: &SandboxState, token: WorkspaceToken, path: &str, + forced: bool, ) -> Result { let path_record = record(state, token); let file = lease(&path_record.with_extension("unmount-lock"), true)? @@ -233,6 +239,7 @@ pub fn begin( _lease: file, path: path_record, started: millis(), + forced, }; layerfs_store::project::discovery::save( &attempt.path, @@ -267,6 +274,7 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd ("state", SdkValue::text(state)), ("startedMs", SdkValue::Int(attempt.started)), ("path", SdkValue::text(path)), + ("forced", SdkValue::Bool(attempt.forced)), ("code", SdkValue::text(code)), ("message", SdkValue::text(message)), ("detail", detail), diff --git a/control/crates/layerfs-sandbox/src/workspace/serve.rs b/control/crates/layerfs-sandbox/src/workspace/serve.rs index b4378700ad..62af20fcd7 100644 --- a/control/crates/layerfs-sandbox/src/workspace/serve.rs +++ b/control/crates/layerfs-sandbox/src/workspace/serve.rs @@ -245,22 +245,26 @@ pub fn unmount(host: &Host, path: Option<&str>, discard: bool, force: bool) -> O let state = sandbox(host)?; let path = required(path, "path")?; if let Some(value) = crate::unmount_guard::find(&state, path)? { - return crate::unmount_guard::repeat(&value); + if !(force && crate::unmount_guard::forceable(&value)) { + return crate::unmount_guard::repeat(&value); + } } let guard = layerfs_store::project::discovery::guard(&state.directory)?; // Another process may have registered while this call waited for the // short admission section. Check again before sending anything. if let Some(value) = crate::unmount_guard::find(&state, path)? { - return crate::unmount_guard::repeat(&value); + if !(force && crate::unmount_guard::forceable(&value)) { + return crate::unmount_guard::repeat(&value); + } } - let token = state.token(path)?; + let token = state.unmount_token(path)?; // its running commands (D-10): `Busy`, or with `force` cancelled first let wsid = layerfs_bridge::sdk_values::hex(&token.workspace.to_bytes()); let unreached = &mut crate::exec::cancel::unreached; crate::exec::cancel::settle(&state.directory, &state.id, Some(&wsid), force, unreached)?; crate::files::refresh(&state)?; // the daemon's kernel probe answers Busy for a process still holding it - let attempt = crate::unmount_guard::begin(&state, token, path)?; + let attempt = crate::unmount_guard::begin(&state, token, path, force)?; drop(guard); // Only this Workspace's lease remains held. Other Workspaces and // processes can take Sandbox ownership while the daemon helper waits. diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index c92759b3b7..630f5777ec 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -21,7 +21,7 @@ //! the export and its engine mount in the entry, as FUSE custody is kept. //! - ForceUnmount runs the forced helper once, then the same teardown. use super::{ - export_helper::{Helpers, Ran}, + export_helper::{mounted_under, Helpers, Ran}, kernel::{bounded, Kernel}, registry::{bound_mut, idle}, Failure, Service, Success, @@ -34,8 +34,6 @@ use layerfs_bridge::control::{ }; use layerfs_nfs::{ExportConfig, Released, Server}; use layerfs_overlay::{CleanupState, KernelMount}; -#[cfg(target_os = "macos")] -use std::process::{Command, Stdio}; use std::{ fmt, net::{IpAddr, Ipv4Addr, TcpListener}, @@ -48,7 +46,7 @@ use std::{ /// `rsize`/`wsize` it asks the host client to mount with. One source: the /// receipt, the mount options and FSINFO cannot drift apart. const WINDOW: u32 = layerfs_nfs::TRANSFER_WINDOW; -pub use super::export_helper::{Helper, HELPER_WAIT}; +pub use super::export_helper::{Helper, HELPER_WAIT, REAP_WAIT}; /// Stale inodes one install refreshes through the mount point at most. pub const REFRESHED: usize = 4096; @@ -143,6 +141,9 @@ pub struct ExportServingConfig { pub refuse_apple_double: bool, /// Lookup references one export keeps before collecting. pub lookups_held: usize, + pub helper_wait: std::time::Duration, + pub reap_wait: std::time::Duration, + pub mount_table: Option, } impl ExportServingConfig { pub fn new(mounts: PathBuf, command_uid: u32, command_gid: u32, mounter: Mounter) -> Self { @@ -154,6 +155,9 @@ impl ExportServingConfig { mounter, refuse_apple_double: true, lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + helper_wait: HELPER_WAIT, + reap_wait: REAP_WAIT, + mount_table: None, } } } @@ -162,28 +166,34 @@ impl ExportServingConfig { #[derive(Debug)] pub struct ExportServing { config: ExportServingConfig, - helpers: Helpers, + helpers: Arc, } impl ExportServing { pub fn start(config: ExportServingConfig) -> Self { - Self { - config, - helpers: Helpers::default(), - } + let helpers = Helpers::new(config.helper_wait, config.reap_wait); + Self { config, helpers } } pub fn config(&self) -> &ExportServingConfig { &self.config } + pub fn reaped_helpers(&self) -> Vec<(u32, String)> { + self.helpers.reaped() + } fn directory(&self, token: WorkspaceToken) -> PathBuf { self.config.mounts.join(token.namespace.to_string()) } + fn is_mounted(&self, directory: &str) -> std::io::Result { + let target = std::path::Path::new(directory); + let points = mounted_under(&self.config.mounts, self.config.mount_table.as_deref())?; + Ok(points.iter().any(|point| point == target)) + } /// At startup, before any Attach: every mount the host still has at or /// under the mount home is one a daemon that ended without unmounting /// left (its export is gone, so the client would hang on it). Each is /// forced off once with the force helper; one line per mount says what /// happened. Nothing is retried. pub fn reclaim(&self) -> Vec { - let points = match mounted_under(&self.config.mounts) { + let points = match mounted_under(&self.config.mounts, self.config.mount_table.as_deref()) { Ok(points) => points, Err(error) => return vec![format!("mount table unreadable: {error}")], }; @@ -200,41 +210,12 @@ impl ExportServing { .collect() } } -/// Mount points at or under `home`, deepest first, from the host's table: -/// `mount`'s ` on ()` lines on macOS, -/// `/proc/self/mountinfo` (field 5, octal escapes) on Linux. -fn mounted_under(home: &std::path::Path) -> std::io::Result> { - #[cfg(target_os = "macos")] - let points: Vec = { - let out = Command::new("/sbin/mount") - .env("LC_ALL", "C") - .stdin(Stdio::null()) - .output()?; - (String::from_utf8_lossy(&out.stdout).lines()) - .filter_map(|line| { - Some(PathBuf::from( - line.split_once(" on ")?.1.rsplit_once(" (")?.0, - )) - }) - .collect() - }; - #[cfg(not(target_os = "macos"))] - let points: Vec = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) - .filter_map(|line| line.split(' ').nth(4)) - .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) - .collect(); - let mut found: Vec = (points.into_iter()) - .filter(|point| point.starts_with(home)) - .collect(); - found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); - found.dedup(); - Ok(found) -} /// A serving export and the mount the host client holds of it. pub(super) struct Exported { pub server: Arc, pub mount: KernelMount, pub ready: ReadyMount, + pub tracked: bool, } /// A terminal operation owns the export; status still observes it. pub(super) struct ExportLeaving { @@ -249,10 +230,17 @@ pub(super) struct ExportKept { pub stage: TeardownStage, pub detail: String, pub work: Option, + pub exported: Option>, pub evidence: Box, pub forced: Option, } impl ExportKept { + pub(super) fn forceable(&self) -> bool { + self.exported.is_some() + && self.stage == TeardownStage::Detach + && self.forced.is_none() + && self.ready.is_some() + } pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { TeardownCustody { token, @@ -266,12 +254,16 @@ impl ExportKept { } impl fmt::Debug for ExportKept { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let evidence: &dyn fmt::Debug = self + .exported + .as_ref() + .map_or(&*self.evidence as &dyn fmt::Debug, |e| e); f.debug_struct("ExportKept") .field("ready", &self.ready) .field("stage", &self.stage) .field("detail", &self.detail) .field("work", &self.work) - .field("evidence", &self.evidence) + .field("evidence", &evidence) .field("forced", &self.forced) .finish() } @@ -292,8 +284,8 @@ pub(super) fn work(server: &Server) -> MountWork { MountWork { loops_configured: 1, loops_entered: 1, - loops_exited: 0, - loops_joined: 0, + loops_exited: u16::from(server.exited()), + loops_joined: u16::from(server.joined()), active: 0, retained: u32::try_from(held.retained).unwrap_or(u32::MAX), completed: served, @@ -518,11 +510,13 @@ impl Service { stage: TeardownStage::Detach, detail, work: Some(work(&server)), - evidence: Box::new(Exported { + exported: Some(Box::new(Exported { server: Arc::new(server), mount, ready: receipt(token, text, mount, port), - }), + tracked: false, + })), + evidence: Box::new(()), forced: None, }))); } @@ -540,7 +534,8 @@ impl Service { Ok(Exported { server: Arc::new(server), mount, - ready: receipt(token, text, mount, port), + ready: receipt(token, text.clone(), mount, port), + tracked: serving.is_mounted(&text).unwrap_or(false), }) } /// No client mount and no export of this attempt remains: revoke its @@ -561,6 +556,7 @@ impl Service { stage: TeardownStage::Revoke, detail: revoke.wire().detail, work: None, + exported: None, evidence: Box::new((failure, mount, released, revoke)), forced: None, })), @@ -681,6 +677,7 @@ impl Service { format!("client reported the mount busy; no terminal effect: {detail}"), )); } + Ran::Failed(_) | Ran::Interrupted(_) if self.detached_export(&exported) => {} Ran::Failed(detail) | Ran::Interrupted(detail) => { return Err(self.keep_export( token, @@ -694,6 +691,11 @@ impl Service { self.teardown_export(token, &workspace, exported, None)?; self.close(token, &workspace, None) } + fn detached_export(&self, exported: &Exported) -> bool { + exported.tracked + && (self.export.as_ref()) + .is_some_and(|s| matches!(s.is_mounted(&exported.ready.directory), Ok(false))) + } /// After the client mount is gone: stop the export, revoke the engine /// mount, remove the directory and leave the entry Unattached for Close. fn teardown_export( @@ -702,15 +704,16 @@ impl Service { workspace: &BoundWorkspace, exported: Exported, forced: Option, - ) -> Result<(), Failure> { + ) -> Result { self.export_phase(token, MountPhase::Draining, Some(work(&exported.server))); let Exported { server, mount, ready, + tracked, } = exported; - let released = match Arc::try_unwrap(server) { - Ok(server) => server.stop(), + let (released, stopped) = match Arc::try_unwrap(server) { + Ok(mut server) => (server.shutdown(), work(&server)), Err(server) => { return Err(self.keep_export( token, @@ -720,6 +723,7 @@ impl Service { server, mount, ready, + tracked, }, forced, )) @@ -733,6 +737,7 @@ impl Service { stage: TeardownStage::Revoke, detail: failure.wire().detail, work: None, + exported: None, evidence: Box::new((failure, mount, released)), forced, }; @@ -754,7 +759,7 @@ impl Service { .lock() .map(|mut last| *last = Some((token, released))) .ok(); - Ok(()) + Ok(stopped) } fn keep_export( &self, @@ -769,7 +774,8 @@ impl Service { stage, detail, work: Some(work(&exported.server)), - evidence: Box::new(exported), + exported: Some(Box::new(exported)), + evidence: Box::new(()), forced, }; let custody = kept.custody(token); @@ -794,6 +800,9 @@ impl Service { } /// Describes retained export custody for an operator. Observation only. pub fn retained_export(&self, token: WorkspaceToken) -> Result, Failure> { + if let Some(serving) = &self.export { + serving.helpers.reap(); + } let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; Ok(match &bound_mut(&mut entries, token)?.kernel { Kernel::ExportKept(kept) => Some(format!("{kept:?}")), @@ -852,7 +861,9 @@ impl Service { None => Ran::Failed("export serving not assembled".into()), }; match ran { - Ran::Done => facts.detach = DetachDisposition::Detached, + Ran::Done => {} + Ran::Busy(_) | Ran::Failed(_) | Ran::Interrupted(_) + if self.detached_export(&exported) => {} Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { facts.detach = DetachDisposition::Failed; return Err(self.keep_export( @@ -864,8 +875,8 @@ impl Service { )); } } - let work = work(&exported.server); - self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; + facts.detach = DetachDisposition::Detached; + let work = self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; let kernel = self.closed(token, &workspace, None, Some(facts.clone()))?; let (cleanup, observation_failure) = match self .owner @@ -905,8 +916,17 @@ impl Service { /// Terminal admission over an export-owning entry; the entry observes the /// export as leaving until the operation settles it. pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { - match kernel { - Kernel::Exported(_) => {} + let exported = match kernel { + Kernel::Exported(_) => match std::mem::replace(kernel, Kernel::Unattached) { + Kernel::Exported(exported) => *exported, + _ => unreachable!("checked export"), + }, + Kernel::ExportKept(kept) if phase == MountPhase::Stopping && kept.forceable() => { + match std::mem::replace(kernel, Kernel::Unattached) { + Kernel::ExportKept(mut kept) => *kept.exported.take().expect("forceable export"), + _ => unreachable!("checked retained export"), + } + } Kernel::ExportLeaving(_) => { return Err(kernel_failure( ControlCode::Busy, @@ -921,14 +941,11 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result>); +pub(super) struct Helpers { + held: Mutex>, + reaped: Mutex>, + reaping: AtomicBool, + pub helper_wait: Duration, + pub reap_wait: Duration, +} impl Helpers { - pub fn run(&self, helper: &Helper, port: u16, directory: &str) -> Ran { - { - let mut held = self.0.lock().unwrap_or_else(|error| error.into_inner()); - held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); + pub fn new(helper_wait: Duration, reap_wait: Duration) -> Arc { + Arc::new(Self { + helper_wait, + reap_wait, + ..Self::default() + }) + } + pub fn reap(&self) -> bool { + let mut held = self.held.lock().unwrap_or_else(|e| e.into_inner()); + let mut reaped = self.reaped.lock().unwrap_or_else(|e| e.into_inner()); + held.retain_mut(|child| match child.try_wait() { + Ok(Some(s)) => { + reaped.push((child.id(), s.to_string())); + false + } + Ok(None) => true, + Err(e) => { + reaped.push((child.id(), format!("wait error: {e}"))); + false + } + }); + let excess = reaped.len().saturating_sub(64); + reaped.drain(..excess); + held.is_empty() + } + pub fn reaped(&self) -> Vec<(u32, String)> { + self.reap(); + self.reaped + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + fn retain(self: &Arc, child: Child) { + self.held + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(child); + if !self.reaping.swap(true, Ordering::AcqRel) { + let queue = self.clone(); + if std::thread::Builder::new() + .name("layerfs-helper-reap".into()) + .spawn(move || loop { + if queue.reap() { + queue.reaping.store(false, Ordering::Release); + if queue.reap() || queue.reaping.swap(true, Ordering::AcqRel) { + break; + } + } + std::thread::sleep(Duration::from_millis(20)); + }) + .is_err() + { + self.reaping.store(false, Ordering::Release); + } } + } + pub fn run(self: &Arc, helper: &Helper, port: u16, directory: &str) -> Ran { + self.reap(); let Some(mut command) = helper.command(port, directory) else { return Ran::Failed("empty helper command".into()); }; @@ -86,7 +149,7 @@ impl Helpers { } } let mut output = Vec::new(); - let deadline = Instant::now() + HELPER_WAIT; + let deadline = Instant::now() + self.helper_wait; let status = loop { drain(&mut pipes, &mut output); if let Some(error) = pipe_error.take() { @@ -95,7 +158,7 @@ impl Helpers { match child.try_wait() { Ok(Some(status)) => break Ok(status), Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), - Ok(None) => break Err(format!("helper did not finish in {HELPER_WAIT:?}")), + Ok(None) => break Err(format!("helper did not finish in {:?}", self.helper_wait)), Err(error) => break Err(format!("helper wait: {error}")), } }; @@ -103,23 +166,31 @@ impl Helpers { Ok(status) => Ok(status), Err(detail) => { let kill = child.kill(); - let deadline = Instant::now() + REAP_WAIT; - let reaped = loop { - match child.try_wait() { - Ok(Some(_)) => break true, - Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), - Ok(None) | Err(_) => break false, + let deadline = Instant::now() + self.reap_wait; + let reaped = if self.reap_wait.is_zero() { + None + } else { + loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) | Err(_) => break None, + } } }; let pid = child.id(); - let disposition = if reaped { - "exit reaped".to_owned() - } else { - self.0 - .lock() - .unwrap_or_else(|error| error.into_inner()) - .push(child); - "exit not observed after 5s; helper may remain in kernel wait; child custody retained".into() + let disposition = match reaped { + Some(status) => { + self.reaped + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push((pid, status.to_string())); + format!("exit reaped ({status})") + } + None => { + self.retain(child); + format!("exit not observed after {:?}; helper may remain in kernel wait; child custody retained", self.reap_wait) + } }; Err(format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}")) } @@ -141,6 +212,38 @@ impl Helpers { } } +/// Mount points at or under `home`, deepest first, from `table` or the host's +/// mount table (`mount` on macOS, `/proc/self/mountinfo` on Linux). +pub(super) fn mounted_under(home: &Path, table: Option<&Path>) -> io::Result> { + let points: Vec = if let Some(table) = table { + let text = std::fs::read_to_string(table).unwrap_or_default(); + (text.lines().map(str::trim).filter(|l| !l.is_empty())) + .map(PathBuf::from) + .collect() + } else { + #[cfg(target_os = "macos")] + let points = { + let out = Command::new("/sbin/mount") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .output()?; + (String::from_utf8_lossy(&out.stdout).lines()) + .filter_map(|l| Some(PathBuf::from(l.split_once(" on ")?.1.rsplit_once(" (")?.0))) + .collect() + }; + #[cfg(not(target_os = "macos"))] + let points = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) + .filter_map(|line| line.split(' ').nth(4)) + .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) + .collect(); + points + }; + let mut found: Vec = points.into_iter().filter(|p| p.starts_with(home)).collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} + fn drain(pipes: &mut [Box], output: &mut Vec) { for pipe in pipes { let mut bytes = [0; OUTPUT]; diff --git a/engine/crates/layerfs-daemon/src/control/force.rs b/engine/crates/layerfs-daemon/src/control/force.rs index 1699cdc211..f4e7d24dd3 100644 --- a/engine/crates/layerfs-daemon/src/control/force.rs +++ b/engine/crates/layerfs-daemon/src/control/force.rs @@ -54,6 +54,9 @@ pub(super) fn guard( // it. The one exception is a normal unmount stalled at Join, admitted // idle with no Commit custody, whose abort control is still unused. if let Kernel::ExportKept(kept) = &value.kernel { + if kept.forceable() { + return Ok(CommitKnowledge::Absent); + } return Err(Failure::Retained(Box::new(kept.custody(token)))); } #[cfg(target_os = "linux")] diff --git a/engine/crates/layerfs-nfs/src/server.rs b/engine/crates/layerfs-nfs/src/server.rs index 60add78dff..2db8d8f569 100644 --- a/engine/crates/layerfs-nfs/src/server.rs +++ b/engine/crates/layerfs-nfs/src/server.rs @@ -120,6 +120,8 @@ struct Shared { config: ExportConfig, counters: Counters, stopping: AtomicBool, + exited: AtomicBool, + joined: AtomicBool, connections: Mutex)>>, } /// A running export. @@ -145,6 +147,8 @@ impl Server { config, counters: Counters::default(), stopping: AtomicBool::new(false), + exited: AtomicBool::new(false), + joined: AtomicBool::new(false), connections: Mutex::default(), }); let accepting = shared.clone(); @@ -200,13 +204,19 @@ impl Server { pub fn verifier(&self) -> [u8; 8] { self.shared.verifier } + pub fn exited(&self) -> bool { + self.shared.exited.load(Ordering::Acquire) + } + pub fn joined(&self) -> bool { + self.shared.joined.load(Ordering::Acquire) + } /// Stops accepting, closes every connection, joins every thread, then /// releases the synthetic kernel's descriptors and references. The engine /// mount itself stays with the caller, who revokes it afterwards. pub fn stop(mut self) -> Released { self.shutdown() } - fn shutdown(&mut self) -> Released { + pub fn shutdown(&mut self) -> Released { let Some(accept) = self.accept.take() else { return Released::default(); }; @@ -214,6 +224,7 @@ impl Server { // Wake the blocking accept; it sees the flag and returns. drop(TcpStream::connect(self.address)); drop(accept.join()); + self.shared.exited.store(true, Ordering::Release); let connections = std::mem::take( &mut *self .shared @@ -227,6 +238,7 @@ impl Server { for (_, thread) in connections { drop(thread.join()); } + self.shared.joined.store(true, Ordering::Release); self.shared.kernel.release() } } @@ -238,6 +250,7 @@ impl Drop for Server { fn accept_loop(listener: &TcpListener, shared: &Arc) { for stream in listener.incoming() { if shared.stopping.load(Ordering::Acquire) { + shared.exited.store(true, Ordering::Release); return; } let Ok(stream) = stream else { continue }; diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs index 781e82710a..429b8966ee 100644 --- a/engine/crates/layerfs-nfs/tests/helper_deadline.rs +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -2,15 +2,20 @@ #[allow(dead_code)] #[path = "support/rig.rs"] mod rig; -use layerfs_bridge::control::{MountPhase, Reply, Request, UnmountControl}; -use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; +use layerfs_bridge::control::{ + CleanupObservation, DetachDisposition, ForcedCleanup, MountPhase, Reply, Request, + TeardownStage, UnmountControl, +}; +use layerfs_daemon::control::{ + ExportServing, ExportServingConfig, Failure, Helper, Mounter, HELPER_WAIT, +}; use layerfs_history::WorkspaceId; use std::{ fs, - path::PathBuf, + path::{Path, PathBuf}, process::Command, sync::{mpsc, Arc}, - time::Duration, + time::{Duration, Instant}, }; fn base(label: &str) -> PathBuf { @@ -140,3 +145,281 @@ fn verbose_helper_preserves_busy_refusal_without_pipe_deadlock() { assert!(reply.to_ascii_lowercase().contains("busy"), "{reply}"); assert_eq!(phase, MountPhase::Ready); } + +fn wait_reaped(serving: &ExportServing, pid: u32, limit: Duration) -> String { + let deadline = Instant::now() + limit; + loop { + if let Some((_, status)) = serving + .reaped_helpers() + .into_iter() + .find(|(p, _)| *p == pid) + { + return status; + } + assert!( + Instant::now() < deadline, + "timed-out helper {pid} was not reaped within {limit:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn assert_not_zombie(pid: u32) { + let out = Command::new("ps") + .args(["-o", "state=", "-p", &pid.to_string()]) + .output() + .unwrap(); + let state = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + assert!( + !state.starts_with('Z'), + "helper pid {pid} remained a zombie: state={state:?}" + ); +} + +#[test] +fn timed_out_helper_when_mount_remains_stays_retained_reaps_zombie_and_forces_cleanly() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("mount-remains"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let pidfile = base.join("unmount.pid"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo $$ > {}; sleep 5", pidfile.display()), + ]), + force: Helper(vec![ + "sh".into(), + "-c".into(), + format!("rm -f {}", table.display()), + ]), + flush: None, + }, + ); + config.helper_wait = Duration::from_millis(100); + config.reap_wait = Duration::ZERO; + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export.clone()); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([34; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // Mount remains in table when unmount helper times out: keep Retained(Detach). + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + let work = custody.work.expect("mount work"); + assert_eq!(work.loops_entered, 1); + assert_eq!(work.loops_exited, 0); + assert_eq!(work.loops_joined, 0); + } + other => panic!("expected Retained, got {other:?}"), + } + + // With no new Helpers::run call, background reaper reaps the child and records exit. + let pid: u32 = fs::read_to_string(&pidfile) + .unwrap() + .trim() + .parse() + .unwrap(); + let status = wait_reaped(&export, pid, Duration::from_secs(2)); + assert!( + !status.is_empty(), + "recorded exit status should not be empty" + ); + assert_not_zombie(pid); + + // Explicit ForceUnmount on ExportKept(Detach) finishes teardown and closes Workspace. + match service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => { + assert_eq!(done.outcome.facts.detach, DetachDisposition::Detached); + assert_eq!(done.outcome.work.loops_entered, 1); + assert_eq!(done.outcome.work.loops_exited, 1); + assert_eq!(done.outcome.work.loops_joined, 1); + assert!(matches!( + done.outcome.cleanup, + ForcedCleanup::Queued | ForcedCleanup::Gone + )); + } + other => panic!("expected ForceUnmounted, got {other:?}"), + } + let released = service.export_released(token).expect("export released"); + assert_eq!(released.retained, 0); + assert!(!Path::new(&ready.directory).exists()); + assert!(service.execute_control(&Request::Status(token)).is_err()); + match service + .execute_control(&Request::Cleanup(token)) + .unwrap() + .reply + { + Reply::Cleanup { state, .. } => assert_eq!(state, CleanupObservation::Gone), + other => panic!("expected Cleanup, got {other:?}"), + } + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + +#[test] +fn timed_out_helper_when_mount_already_gone_completes_teardown() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("mount-gone"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let pidfile = base.join("unmount.pid"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec![ + "sh".into(), + "-c".into(), + format!( + "rm -f {}; echo $$ > {}; sleep 5", + table.display(), + pidfile.display() + ), + ]), + force: Helper::words("true").unwrap(), + flush: None, + }, + ); + config.helper_wait = Duration::from_millis(100); + config.reap_wait = Duration::ZERO; + config.mount_table = Some(table); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export.clone()); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([35; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // Helper timed out, but mount table confirms the mount is gone: teardown completes. + let reply = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .expect("unmount should complete teardown when mount is gone") + .reply; + assert_eq!(reply, Reply::Unmounted(token)); + let released = service.export_released(token).expect("export released"); + assert_eq!(released.retained, 0); + assert!(!Path::new(&ready.directory).exists()); + assert!(service.execute_control(&Request::Status(token)).is_err()); + match service + .execute_control(&Request::Cleanup(token)) + .unwrap() + .reply + { + Reply::Cleanup { state, .. } => assert_eq!(state, CleanupObservation::Gone), + other => panic!("expected Cleanup, got {other:?}"), + } + + // The timed-out helper is still reaped in the background without becoming a zombie. + let pid: u32 = fs::read_to_string(&pidfile) + .unwrap() + .trim() + .parse() + .unwrap(); + let status = wait_reaped(&export, pid, Duration::from_secs(2)); + assert!(!status.is_empty()); + assert_not_zombie(pid); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} From 7e501eb2868d6002f40fdd0321f0c27e23a9e061 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:19:33 +0800 Subject: [PATCH 02/14] fix(daemon,sandbox): harden mount table checks and unmount guard forceable rules Address review findings M1, M2, and M3 on native NFS unmount teardown: - M1: check /sbin/mount exit status in mounted_under on macOS and treat only NotFound as an empty table file so mount-listing failures never falsely mark a still-mounted Workspace as detached. - M2: match both literal and canonicalized paths in mounted_under and ExportServing::is_mounted so symlinked macOS mount roots (/var, /tmp -> /private/var, /private/tmp) are tracked and reconciled accurately. - M3: record forced in unmount_guard::begin and inspect, and restrict unmount_guard::forceable to completed non-forced Failed records so crashed or Unknown unmounts are never re-sent as ForceUnmount. Ref: layerfs-project#136 Production LOC: 156,485 -> 156,510 (delta +25) Method: python3 tools/production_loc.py (38af9c026 -> HEAD) Subtotals: core 39,749 -> 39,749 (delta 0); engine 73,215 -> 73,231 (delta +16); control 38,551 -> 38,560 (delta +9); api 4,970 -> 4,970 (delta 0); reference absent (retired R9c; last counted 65,417 at tree 498dd1917812); dsh 0 -> 0 (delta 0). --- .../layerfs-sandbox/src/unmount_guard.rs | 13 ++- .../layerfs-sandbox/tests/record_codec.rs | 78 ++++++++++++++ .../layerfs-daemon/src/control/export.rs | 5 +- .../src/control/export_helper.rs | 17 ++- .../layerfs-nfs/tests/helper_deadline.rs | 102 ++++++++++++++++++ 5 files changed, 210 insertions(+), 5 deletions(-) diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index b733bc4ca2..5964a76466 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -130,6 +130,13 @@ fn inspect(path: &std::path::Path) -> Result, SdkRefusal> { value.get("startedMs").cloned().unwrap_or_default(), ), ("path", value.get("path").cloned().unwrap_or_default()), + ( + "forced", + value + .get("forced") + .cloned() + .unwrap_or(SdkValue::Bool(false)), + ), ("code", SdkValue::text(error.code)), ("message", SdkValue::text(error.message)), ("detail", error.detail), @@ -184,7 +191,7 @@ pub fn check_in(directory: &std::path::Path, path: &str) -> Result<(), SdkRefusa Ok(()) } /// [`find`] by scanning Sandbox folder `directory`'s unmount records. -fn find_in(directory: &std::path::Path, path: &str) -> Result, SdkRefusal> { +pub fn find_in(directory: &std::path::Path, path: &str) -> Result, SdkRefusal> { for entry in unmounts_in(directory)? { let entry = entry.map_err(failed)?; if entry.path().extension().is_some_and(|ext| ext == "unmount") { @@ -223,7 +230,8 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { } pub fn forceable(value: &SdkValue) -> bool { value.get_text("state") == Some("Retained") - && value.get("forced") != Some(&SdkValue::Bool(true)) + && value.get("forced") == Some(&SdkValue::Bool(false)) + && value.get_text("code") == Some("Failed") } /// Called under the Sandbox guard, after all before-effect checks. pub fn begin( @@ -247,6 +255,7 @@ pub fn begin( ("state", SdkValue::text("Unmounting")), ("startedMs", SdkValue::Int(attempt.started)), ("path", SdkValue::text(path)), + ("forced", SdkValue::Bool(attempt.forced)), ]), )?; Ok(attempt) diff --git a/control/crates/layerfs-sandbox/tests/record_codec.rs b/control/crates/layerfs-sandbox/tests/record_codec.rs index 1e4355b8e9..e84c104c8d 100644 --- a/control/crates/layerfs-sandbox/tests/record_codec.rs +++ b/control/crates/layerfs-sandbox/tests/record_codec.rs @@ -88,3 +88,81 @@ fn a_workspace_record_is_written_in_its_order() { assert!(!std::fs::read_to_string(&file).unwrap().contains("\"name\"")); std::fs::remove_dir_all(dir).unwrap(); } + +#[test] +fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { + use std::os::unix::fs::PermissionsExt; + use std::sync::mpsc; + use std::time::Duration; + + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let dir = std::env::temp_dir().join(format!("lfs-unmount-guard-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let ws = dir.join("workspace"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&ws, std::fs::Permissions::from_mode(0o700)).unwrap(); + + let write_rec = |name: &str, text: &str| { + let p = ws.join(name); + let bytes = layerfs_bridge::control::Answer { + id: 1, + reply: layerfs_bridge::control::Reply::Sdk(value(text)), + } + .encode() + .unwrap(); + std::fs::write(&p, bytes).unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600)).unwrap(); + }; + // 1. Crashed force unmount (state=Unmounting, forced=true, no active lock -> code=Unknown) + write_rec( + "01.unmount", + r#"{"state":"Unmounting","startedMs":1,"path":"/w/1","forced":true}"#, + ); + let r1 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/1") + .unwrap() + .unwrap(); + assert_eq!(r1.get_text("code"), Some("Unknown")); + assert_eq!(r1.get("forced"), Some(&SdkValue::Bool(true))); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r1)); + + // 2. Crashed normal unmount (state=Unmounting, forced=false, no active lock -> code=Unknown) + write_rec( + "02.unmount", + r#"{"state":"Unmounting","startedMs":1,"path":"/w/2","forced":false}"#, + ); + let r2 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/2") + .unwrap() + .unwrap(); + assert_eq!(r2.get_text("code"), Some("Unknown")); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r2)); + + // 3. Completed forced unmount that remained Retained (forced=true, code=Failed) + write_rec( + "03.unmount", + r#"{"state":"Retained","startedMs":1,"path":"/w/3","forced":true,"code":"Failed","message":"m","detail":null}"#, + ); + let r3 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/3") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r3)); + + // 4. Completed normal unmount that returned Retained (forced=false, code=Failed) -> forceable! + write_rec( + "04.unmount", + r#"{"state":"Retained","startedMs":1,"path":"/w/4","forced":false,"code":"Failed","message":"m","detail":null}"#, + ); + let r4 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/4") + .unwrap() + .unwrap(); + assert!(layerfs_sandbox::unmount_guard::forceable(&r4)); + + std::fs::remove_dir_all(dir).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(5)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 630f5777ec..eae4811ba6 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -184,8 +184,11 @@ impl ExportServing { } fn is_mounted(&self, directory: &str) -> std::io::Result { let target = std::path::Path::new(directory); + let real = target.canonicalize().ok(); let points = mounted_under(&self.config.mounts, self.config.mount_table.as_deref())?; - Ok(points.iter().any(|point| point == target)) + Ok(points + .iter() + .any(|p| p == target || real.as_deref().is_some_and(|r| p == r))) } /// At startup, before any Attach: every mount the host still has at or /// under the mount home is one a daemon that ended without unmounting diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index 15a7933ff7..5a1a95ff84 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -216,7 +216,11 @@ impl Helpers { /// mount table (`mount` on macOS, `/proc/self/mountinfo` on Linux). pub(super) fn mounted_under(home: &Path, table: Option<&Path>) -> io::Result> { let points: Vec = if let Some(table) = table { - let text = std::fs::read_to_string(table).unwrap_or_default(); + let text = match std::fs::read_to_string(table) { + Ok(text) => text, + Err(error) if error.kind() == io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(error), + }; (text.lines().map(str::trim).filter(|l| !l.is_empty())) .map(PathBuf::from) .collect() @@ -227,6 +231,12 @@ pub(super) fn mounted_under(home: &Path, table: Option<&Path>) -> io::Result) -> io::Result = points.into_iter().filter(|p| p.starts_with(home)).collect(); + let real = home.canonicalize().ok(); + let mut found: Vec = (points.into_iter()) + .filter(|p| p.starts_with(home) || real.as_deref().is_some_and(|r| p.starts_with(r))) + .collect(); found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); found.dedup(); Ok(found) diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs index 429b8966ee..4cc628a8a6 100644 --- a/engine/crates/layerfs-nfs/tests/helper_deadline.rs +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -423,3 +423,105 @@ fn timed_out_helper_when_mount_already_gone_completes_teardown() { .expect("bounded test timed out"); worker.join().unwrap(); } + +#[test] +fn canonical_mount_paths_and_unreadable_table_preserve_detach_safety() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("canon-table"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("real_mounts")).unwrap(); + let link_mounts = base.join("link_mounts"); + std::os::unix::fs::symlink(base.join("real_mounts"), &link_mounts).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + link_mounts, + 501, + 20, + Mounter { + // Record the canonicalized path (like macOS /sbin/mount printing /private/...). + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("cd '{{dir}}' && pwd -P > {}", table.display()), + ]), + unmount: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + force: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + flush: None, + }, + ); + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([36; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // M1: Replace mount.table with a directory so reading fails with a non-NotFound error. + // Unmount must NOT treat the read error as an empty mount table; it must keep Retained. + fs::remove_file(&table).unwrap(); + fs::create_dir(&table).unwrap(); + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + assert!( + matches!(err, Failure::Retained(_)), + "expected Retained, got {err:?}" + ); + assert!(Path::new(&ready.directory).exists()); + + // M2: Once mount.table is removed (NotFound -> mount gone), even if force helper exits 1 + // ("not currently mounted"), canonicalized tracked path recognizes the mount is gone. + fs::remove_dir(&table).unwrap(); + match service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => { + assert_eq!(done.outcome.facts.detach, DetachDisposition::Detached); + assert_eq!(done.outcome.work.loops_joined, 1); + } + other => panic!("expected ForceUnmounted, got {other:?}"), + } + assert!(!Path::new(&ready.directory).exists()); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} From 03204bb92bf9fc72163650ab98971cb4d365f77f Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:43:49 +0000 Subject: [PATCH 03/14] refactor(bridge): share one retained-child reaper The bounded background reaper that polls a helper child past its KILL and post-KILL grace window existed twice: in the daemon's NFS `Helpers` and, with its own statics and its own copy of the loop, in the macOS driver's `umount -f`. Move the queue, the polling thread and the record of observed exits into `layerfs_bridge::process::Retained`, one implementation both callers hold. Semantics are unchanged: nothing waits for a retained child, nothing signals it twice, and at most 64 observed exits are kept. `Helpers` keeps its own deadlines and its `reap`/`reaped` accessors over the shared queue; the driver keeps only its `umount -f` deadline, stderr drain and mount re-check. Production LOC: 160178 -> 160198 (delta +20) Method: python3 tools/production_loc.py, combined scope, first parent 09f9e4b34 and this commit's staged tree; core 39787 -> 39787 (0), engine 74571 -> 74618 (+47), control 39844 -> 39817 (-27), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- .../src/local/os/macos/darwin.rs | 43 ++----- .../crates/layerfs-bridge/src/process/mod.rs | 6 +- .../layerfs-bridge/src/process/retained.rs | 115 ++++++++++++++++++ .../src/control/export_helper.rs | 61 ++-------- 4 files changed, 136 insertions(+), 89 deletions(-) create mode 100644 engine/crates/layerfs-bridge/src/process/retained.rs diff --git a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs index d76fe26104..f22e5b9e35 100644 --- a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs +++ b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs @@ -3,15 +3,13 @@ //! run with a fixed path and the C locale, read once; nothing is retried. //! The process facts (`proc_pidinfo`, `sysctl`) are `layerfs_bridge::process` //! (layerfs-project#112), below the drivers and the home. +use layerfs_bridge::process::Retained; use nix::fcntl::{fcntl, FcntlArg, OFlag}; use std::{ io::{self, Read}, path::{Path, PathBuf}, - process::{Child, Command, Stdio}, - sync::{ - atomic::{AtomicBool, Ordering}, - Mutex, - }, + process::{Command, Stdio}, + sync::LazyLock, time::{Duration, Instant}, }; @@ -51,18 +49,14 @@ fn mount_point(line: &str) -> Option { Some(PathBuf::from(point)) } -static RETAINED: Mutex> = Mutex::new(Vec::new()); -static REAPING: AtomicBool = AtomicBool::new(false); - -fn reap_retained() -> bool { - let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); - held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)) | Err(_))); - held.is_empty() -} +/// Children a `umount` KILL did not release. Their exits are reaped by the +/// shared retained queue ([`layerfs_bridge::process::Retained`]), never by a +/// second wait here. +static RETAINED: LazyLock = LazyLock::new(Retained::default); /// Forces the mount at `point` off once; a point no longer mounted is done. pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { - reap_retained(); + RETAINED.reap(); let mut child = Command::new(UMOUNT) .arg("-f") .arg(point) @@ -116,26 +110,7 @@ pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { let disposition = match reaped { Some(status) => format!("exit reaped ({status})"), None => { - RETAINED - .lock() - .unwrap_or_else(|error| error.into_inner()) - .push(child); - if !REAPING.swap(true, Ordering::AcqRel) - && std::thread::Builder::new() - .name("layerfs-darwin-reap".into()) - .spawn(|| loop { - if reap_retained() { - REAPING.store(false, Ordering::Release); - if reap_retained() || REAPING.swap(true, Ordering::AcqRel) { - break; - } - } - std::thread::sleep(Duration::from_millis(20)); - }) - .is_err() - { - REAPING.store(false, Ordering::Release); - } + RETAINED.retain(child); "exit not observed after 5s; child custody retained (possible kernel wait)" .into() } diff --git a/engine/crates/layerfs-bridge/src/process/mod.rs b/engine/crates/layerfs-bridge/src/process/mod.rs index f06582ef97..d72efbe6c9 100644 --- a/engine/crates/layerfs-bridge/src/process/mod.rs +++ b/engine/crates/layerfs-bridge/src/process/mod.rs @@ -1,7 +1,9 @@ //! A local process as records and Sandboxes name it (moved from the Sandbox //! driver, layerfs-project#112): its id, kernel start time and boot, read //! from `/proc` on Linux (`linux.rs`) and `proc_pidinfo`/`sysctl` on macOS -//! (`darwin.rs`, this crate's one audited `unsafe` file). It sits below the +//! (`darwin.rs`, this crate's one audited `unsafe` file), and the retained +//! child custody both the daemon's NFS helpers and the macOS driver's +//! `umount -f` use after a KILL (`retained.rs`). It sits below the //! control layer so the home and the drivers share it without an edge //! between them. #[cfg(target_os = "macos")] @@ -9,8 +11,10 @@ mod darwin; mod identity; #[cfg(target_os = "linux")] mod linux; +mod retained; #[cfg(target_os = "macos")] pub use darwin::{boot, processes, stat}; pub use identity::{pid, LocalProcess, Stat}; #[cfg(target_os = "linux")] pub use linux::{boot, processes, stat}; +pub use retained::Retained; diff --git a/engine/crates/layerfs-bridge/src/process/retained.rs b/engine/crates/layerfs-bridge/src/process/retained.rs new file mode 100644 index 0000000000..e9f063d759 --- /dev/null +++ b/engine/crates/layerfs-bridge/src/process/retained.rs @@ -0,0 +1,115 @@ +//! Retained child custody, one implementation for every caller that stops a +//! helper with a deadline: the daemon's NFS mount and unmount helpers and the +//! macOS driver's `umount -f` (layerfs-project#136). A child that survived its +//! KILL and the post-KILL grace window is never waited on synchronously and is +//! never signalled twice. One bounded background thread polls it instead and +//! records the exit it observes. +use std::{ + fmt, + process::Child, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +/// Between two looks at the retained queue. +const PAUSE: Duration = Duration::from_millis(20); +/// Observed exits kept for diagnostics; the oldest go first. +const RECORDED: usize = 64; + +/// Children a caller could not observe exiting, and the exits seen since. +/// `retain`, `reap`, `exits` and `record` are the whole surface: nothing here +/// signals a child, and no call waits for one. +#[derive(Clone, Debug, Default)] +pub struct Retained(Arc); + +#[derive(Debug, Default)] +struct Inner { + children: Mutex>, + exits: Mutex>, + reaping: AtomicBool, +} + +impl Retained { + /// Keeps `child` and makes sure one background reaper is polling for it. + pub fn retain(&self, child: Child) { + self.reap(); + self.0 + .children + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(child); + if self.0.reaping.swap(true, Ordering::AcqRel) { + return; + } + let queue = self.0.clone(); + if std::thread::Builder::new() + .name("layerfs-child-reap".into()) + .spawn(move || loop { + if queue.reap() { + queue.reaping.store(false, Ordering::Release); + if queue.reap() || queue.reaping.swap(true, Ordering::AcqRel) { + break; + } + } + std::thread::sleep(PAUSE); + }) + .is_err() + { + self.0.reaping.store(false, Ordering::Release); + } + } + + /// Reaps every child that has exited; true when none remain. + pub fn reap(&self) -> bool { + self.0.reap() + } + + /// The exits observed so far, oldest first. + pub fn exits(&self) -> Vec<(u32, String)> { + self.reap(); + self.0 + .exits + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } + + /// Records an exit the caller observed itself with its own `try_wait`. + pub fn record(&self, child: u32, status: impl fmt::Display) { + let mut exits = self + .0 + .exits + .lock() + .unwrap_or_else(|error| error.into_inner()); + exits.push((child, status.to_string())); + let excess = exits.len().saturating_sub(RECORDED); + exits.drain(..excess); + } +} + +impl Inner { + fn reap(&self) -> bool { + let mut children = self + .children + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut exits = self.exits.lock().unwrap_or_else(|error| error.into_inner()); + children.retain_mut(|child| match child.try_wait() { + Ok(Some(status)) => { + exits.push((child.id(), status.to_string())); + false + } + Ok(None) => true, + Err(error) => { + exits.push((child.id(), format!("wait error: {error}"))); + false + } + }); + let excess = exits.len().saturating_sub(RECORDED); + exits.drain(..excess); + children.is_empty() + } +} diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index 5a1a95ff84..9d2f471d7e 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -1,13 +1,11 @@ //! One bounded NFS client helper attempt, including post-KILL observation. +use layerfs_bridge::process::Retained; use nix::fcntl::{fcntl, FcntlArg, OFlag}; use std::{ io::{self, Read}, path::{Path, PathBuf}, process::{Child, Command, Stdio}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, - }, + sync::Arc, time::{Duration, Instant}, }; @@ -49,9 +47,7 @@ pub(super) enum Ran { /// A bounded background reaper reaps their exits and records their status. #[derive(Debug, Default)] pub(super) struct Helpers { - held: Mutex>, - reaped: Mutex>, - reaping: AtomicBool, + retained: Retained, pub helper_wait: Duration, pub reap_wait: Duration, } @@ -64,53 +60,13 @@ impl Helpers { }) } pub fn reap(&self) -> bool { - let mut held = self.held.lock().unwrap_or_else(|e| e.into_inner()); - let mut reaped = self.reaped.lock().unwrap_or_else(|e| e.into_inner()); - held.retain_mut(|child| match child.try_wait() { - Ok(Some(s)) => { - reaped.push((child.id(), s.to_string())); - false - } - Ok(None) => true, - Err(e) => { - reaped.push((child.id(), format!("wait error: {e}"))); - false - } - }); - let excess = reaped.len().saturating_sub(64); - reaped.drain(..excess); - held.is_empty() + self.retained.reap() } pub fn reaped(&self) -> Vec<(u32, String)> { - self.reap(); - self.reaped - .lock() - .unwrap_or_else(|e| e.into_inner()) - .clone() + self.retained.exits() } fn retain(self: &Arc, child: Child) { - self.held - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(child); - if !self.reaping.swap(true, Ordering::AcqRel) { - let queue = self.clone(); - if std::thread::Builder::new() - .name("layerfs-helper-reap".into()) - .spawn(move || loop { - if queue.reap() { - queue.reaping.store(false, Ordering::Release); - if queue.reap() || queue.reaping.swap(true, Ordering::AcqRel) { - break; - } - } - std::thread::sleep(Duration::from_millis(20)); - }) - .is_err() - { - self.reaping.store(false, Ordering::Release); - } - } + self.retained.retain(child) } pub fn run(self: &Arc, helper: &Helper, port: u16, directory: &str) -> Ran { self.reap(); @@ -181,10 +137,7 @@ impl Helpers { let pid = child.id(); let disposition = match reaped { Some(status) => { - self.reaped - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push((pid, status.to_string())); + self.retained.record(pid, status); format!("exit reaped ({status})") } None => { From 20b4faf4393bd6c3b51236e0233bd8cbc2d9b0cc Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:45:53 +0000 Subject: [PATCH 04/14] fix(daemon): never resolve the mount point to read the mount table `ExportServing::is_mounted` canonicalized the mount directory to compare it with the host's table. That lookup goes through the mount itself, so against a wedged NFS client it can block an unmount with no bound, and it fails (EIO, ETIMEDOUT) exactly while the mount is in trouble: the literal `/var/...` then never matches the table's `/private/var/...`, `is_mounted` answered `Ok(false)`, and `detached_export` tore down the export and the Workspace while the kernel still held the mount. Resolve only the mount home, which the daemon owns, and compare the entry name below it (`export_helper::listed`); the configured path is still compared as it is. The mount table itself is unchanged: one read per check, an unreadable table still keeping `Retained(Detach)`. Test: `a_listed_canonical_mount_stays_retained_when_the_mount_point_does_not_resolve` (a symlinked mount home like macOS `/var` -> `/private/var`, the mount point replaced by a dangling link, the table still listing the resolved point). It fails before this change: the reply was `Unmounted` and the entry was torn down while the table still listed the mount. Production LOC: 160198 -> 160207 (delta +9) Method: python3 tools/production_loc.py, combined scope, first parent 03204bb92 and this commit's staged tree; core 39787 -> 39787 (0), engine 74618 -> 74627 (+9), control 39817 -> 39817 (0), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- .../layerfs-daemon/src/control/export.rs | 34 ++++-- .../src/control/export_helper.rs | 17 +++ .../layerfs-nfs/tests/helper_deadline.rs | 104 ++++++++++++++++++ 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index eae4811ba6..cb55f78a8e 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -19,9 +19,14 @@ //! every reference and descriptor given back), revokes the engine mount //! and runs the one logical Close. A step that is not established keeps //! the export and its engine mount in the entry, as FUSE custody is kept. -//! - ForceUnmount runs the forced helper once, then the same teardown. +//! A helper that failed or ran out of time is judged by the host's mount +//! table, never by the mount itself: only a mount Attach saw listed there +//! can be known gone (`export_helper::listed`). +//! - ForceUnmount runs the forced helper once, then the same teardown. It is +//! also how an export kept at Detach for a mount still listed is finished. +//! A helper that outlives its KILL goes to the bounded retained queue. use super::{ - export_helper::{mounted_under, Helpers, Ran}, + export_helper::{listed, mounted_under, Helpers, Ran}, kernel::{bounded, Kernel}, registry::{bound_mut, idle}, Failure, Service, Success, @@ -141,8 +146,17 @@ pub struct ExportServingConfig { pub refuse_apple_double: bool, /// Lookup references one export keeps before collecting. pub lookups_held: usize, + /// How long one client helper may run before it is killed and reported + /// as unknown. Production value [`HELPER_WAIT`]. pub helper_wait: std::time::Duration, + /// The post-KILL window in which the helper's own exit is still observed + /// synchronously; past it the child goes to the retained queue. Zero + /// skips the window, which bounded tests use to reach the retained path + /// at once. Production value [`REAP_WAIT`]. pub reap_wait: std::time::Duration, + /// Test hook, `None` in production: a file read line by line in place of + /// the host's mount table, so a test states which mounts exist without + /// mounting anything. pub mount_table: Option, } impl ExportServingConfig { @@ -176,19 +190,25 @@ impl ExportServing { pub fn config(&self) -> &ExportServingConfig { &self.config } + /// Test and diagnostic accessor: the helper exits the bounded reaper has + /// observed since the daemon started (newest last, at most 64 kept). + /// Production code reports a retained child inside its own refusal text + /// instead; nothing in the daemon reads this. pub fn reaped_helpers(&self) -> Vec<(u32, String)> { self.helpers.reaped() } fn directory(&self, token: WorkspaceToken) -> PathBuf { self.config.mounts.join(token.namespace.to_string()) } + /// Whether the host's table still lists this mount. The table comes from + /// the home the daemon owns, never from the mount itself: see [`listed`]. fn is_mounted(&self, directory: &str) -> std::io::Result { - let target = std::path::Path::new(directory); - let real = target.canonicalize().ok(); let points = mounted_under(&self.config.mounts, self.config.mount_table.as_deref())?; - Ok(points - .iter() - .any(|p| p == target || real.as_deref().is_some_and(|r| p == r))) + Ok(listed( + &points, + &self.config.mounts, + std::path::Path::new(directory), + )) } /// At startup, before any Attach: every mount the host still has at or /// under the mount home is one a daemon that ended without unmounting diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index 9d2f471d7e..828dc6a5c0 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -165,6 +165,23 @@ impl Helpers { } } +/// Whether the host's table still lists `directory`: the path as configured, +/// or the same name below the canonical mount home (`/var` and `/tmp` are +/// `/private/var` and `/private/tmp` on macOS, which is what the table +/// prints). Only the home is resolved. Resolving the mount directory itself +/// would look the path up through the mount: against a wedged NFS client that +/// lookup can block with no bound, and it fails (EIO, ETIMEDOUT) exactly while +/// the mount is in trouble, which would read as "not mounted". +pub(super) fn listed(points: &[PathBuf], home: &Path, directory: &Path) -> bool { + if points.iter().any(|point| point == directory) { + return true; + } + let (Some(name), Ok(real)) = (directory.file_name(), home.canonicalize()) else { + return false; + }; + points.iter().any(|point| point == &real.join(name)) +} + /// Mount points at or under `home`, deepest first, from `table` or the host's /// mount table (`mount` on macOS, `/proc/self/mountinfo` on Linux). pub(super) fn mounted_under(home: &Path, table: Option<&Path>) -> io::Result> { diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs index 4cc628a8a6..13f72d617f 100644 --- a/engine/crates/layerfs-nfs/tests/helper_deadline.rs +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -525,3 +525,107 @@ fn canonical_mount_paths_and_unreadable_table_preserve_detach_safety() { .expect("bounded test timed out"); worker.join().unwrap(); } + +/// The mount home is a symbolic link (macOS `/var`, `/tmp` are +/// `/private/...`), so the host's table prints the resolved point. A still +/// mounted Workspace must stay `Retained(Detach)` even when the mount +/// directory itself cannot be resolved: that lookup would go through the +/// mount, which is exactly what a wedged client makes fail. +#[test] +fn a_listed_canonical_mount_stays_retained_when_the_mount_point_does_not_resolve() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("canon-listed"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("real_mounts")).unwrap(); + let link_mounts = base.join("link_mounts"); + std::os::unix::fs::symlink(base.join("real_mounts"), &link_mounts).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + link_mounts, + 501, + 20, + Mounter { + // The host prints the resolved path, as `mount` does on macOS. + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("cd '{{dir}}' && pwd -P > {}", table.display()), + ]), + unmount: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + force: Helper::words("true").unwrap(), + flush: None, + }, + ); + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([37; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + let name = ready.directory.rsplit('/').next().unwrap().to_owned(); + let canonical = base.canonicalize().unwrap().join("real_mounts").join(&name); + let listed = || fs::read_to_string(&table).unwrap().trim().to_owned(); + assert_eq!( + listed(), + canonical.display().to_string(), + "the table lists the resolved point, never the configured home" + ); + + // The mount point stops resolving (a wedged mount answers EIO here). + fs::remove_dir(&ready.directory).unwrap(); + std::os::unix::fs::symlink(base.join("gone"), &ready.directory).unwrap(); + + // The table still lists it, so the mount is not gone: keep custody. + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + } + other => panic!("expected Retained, got {other:?}"), + } + assert_eq!( + listed(), + canonical.display().to_string(), + "an unresolved mount point must not be read as a removed mount" + ); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} From 523462afcb84df43e0f3acf4120086b2f0e5f2ae Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:47:46 +0000 Subject: [PATCH 05/14] fix(sandbox): keep the earlier unmount record a forced attempt replaced A forced attempt admitted over a completed `Failed` record wrote its own `Unmounting` record, and a `Busy` or `Dirty` answer to that force deleted the file: the earlier record - possibly one whose answer is "do not resend" - was lost with the attempt that changed nothing. `begin` now keeps the record it replaces, and `restore` puts it back exactly when an attempt has no effect of its own: a `Busy` or `Dirty` finish, and a call that sent nothing (`cancel`). With no earlier record both cases still remove the file, as before. The forceable rule is unchanged and now says what it is: a completed `Failed` record without `forced: true`, because the daemon's own guard is the real gate; a record that is still `Unmounting`, one of unknown outcome and one an earlier force produced keep their answer. A record written before `forced` was stored has no such field, so it is answered as it stands and is not forceable - the description claimed otherwise and is corrected. Tests (`tests/unmount_record.rs`): a busy force, a dirty force and a call that sent nothing each leave the earlier record as it was, still forceable; a first attempt that does not run still leaves no record. Production LOC: 160207 -> 160215 (delta +8) Method: python3 tools/production_loc.py, combined scope, first parent 20b4faf43 and this commit's staged tree; core 39787 -> 39787 (0), engine 74627 -> 74627 (0), control 39817 -> 39825 (+8), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- .../layerfs-sandbox/src/unmount_guard.rs | 35 +++++- .../layerfs-sandbox/tests/unmount_record.rs | 110 ++++++++++++++++++ 2 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 control/crates/layerfs-sandbox/tests/unmount_record.rs diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index 5964a76466..bb796136b7 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -52,6 +52,9 @@ pub struct Attempt { path: PathBuf, started: i64, forced: bool, + /// The record this attempt replaced, put back when the attempt has no + /// effect of its own (see [`restore`]). + prior: Option, } fn result(value: &SdkValue) -> Outcome { if value.get_text("state") == Some("Unmounted") { @@ -229,6 +232,13 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { Ok(recoverable) } pub fn forceable(value: &SdkValue) -> bool { + // A completed `Failed` record, forced or not, is the one shape a force + // may replace: the daemon's own guard is the real gate on whether force + // may proceed, and a refusal answers with the record it holds. A record + // that is still `Unmounting` (its caller ended without publishing), + // `Unknown` or already `forced` keeps its answer. A record written before + // this rule (no `forced` field) is answered as it stands and is not + // forceable. value.get_text("state") == Some("Retained") && value.get("forced") == Some(&SdkValue::Bool(false)) && value.get_text("code") == Some("Failed") @@ -240,11 +250,17 @@ pub fn begin( path: &str, forced: bool, ) -> Result { - let path_record = record(state, token); + begin_at(record(state, token), path, forced) +} +/// [`begin`] at the record file `path_record` (a Workspace's +/// `workspace/.unmount`), holding its lease and replacing the record it +/// found with this attempt's `Unmounting` record. +pub fn begin_at(path_record: PathBuf, path: &str, forced: bool) -> Result { let file = lease(&path_record.with_extension("unmount-lock"), true)? .ok_or_else(|| refused("Busy", "Workspace is Unmounting"))?; let attempt = Attempt { _lease: file, + prior: observed(&path_record)?, path: path_record, started: millis(), forced, @@ -260,13 +276,22 @@ pub fn begin( )?; Ok(attempt) } +/// Undo this attempt's own record: what it replaced comes back, or the file +/// goes when there was none. A forced attempt admitted over an earlier record +/// that answers `Busy` or `Dirty` changed nothing, so the earlier record — +/// possibly one that says "do not resend" — must not be lost with it. A call +/// that sent nothing is the same case. +fn restore(attempt: &Attempt) -> Result<(), SdkRefusal> { + match &attempt.prior { + Some(prior) => layerfs_store::project::discovery::save(&attempt.path, prior.clone()), + None => std::fs::remove_file(&attempt.path).map_err(failed), + } +} /// Publish while the Workspace lease prevents another terminal request. /// No Sandbox guard is needed to replace this attempt's own result file. pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), SdkRefusal> { match outcome { - Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => { - std::fs::remove_file(&attempt.path).map_err(failed)?; - } + Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => restore(attempt)?, _ => { let (state, code, message, detail) = match outcome { Ok(_) => ("Unmounted", "", "", SdkValue::Null), @@ -294,7 +319,7 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd Ok(()) } pub fn cancel(attempt: &Attempt) -> Result<(), SdkRefusal> { - std::fs::remove_file(&attempt.path).map_err(failed) + restore(attempt) } pub fn cleanup_failure(mut outcome: Outcome, error: SdkRefusal) -> Outcome { // Do not replace a known daemon success with a cleanup Busy/Failed. diff --git a/control/crates/layerfs-sandbox/tests/unmount_record.rs b/control/crates/layerfs-sandbox/tests/unmount_record.rs new file mode 100644 index 0000000000..009d9d481e --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/unmount_record.rs @@ -0,0 +1,110 @@ +//! The unmount guard's record file across a forced attempt (layerfs-project#136): +//! an attempt that changed nothing must leave the earlier record as it was. +use layerfs_bridge::control::{SdkRefusal, SdkValue}; +use layerfs_sandbox::unmount_guard::{begin_at, cancel, finish, forceable}; +use layerfs_store::project::discovery; +use std::{ + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +/// A Sandbox folder with its `workspace/` subfolder, both private. +fn fixture(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("lfs-unmount-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let workspace = dir.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + for path in [&dir, &workspace] { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + dir +} + +fn record_file(dir: &Path) -> PathBuf { + dir.join("workspace").join("01.unmount") +} + +/// A completed record a force may replace. +fn completed() -> SdkValue { + SdkValue::map([ + ("state", SdkValue::text("Retained")), + ("startedMs", SdkValue::Int(7)), + ("path", SdkValue::text("/w")), + ("forced", SdkValue::Bool(false)), + ("code", SdkValue::text("Failed")), + ("message", SdkValue::text("detach not established")), + ("detail", SdkValue::Null), + ]) +} + +fn refused(code: &str) -> SdkRefusal { + SdkRefusal { + code: code.into(), + message: code.into(), + detail: SdkValue::Null, + } +} + +/// The earlier record comes back exactly, and still answers as it did. +fn assert_restored(file: &Path, prior: &SdkValue) { + let kept = discovery::load(file).unwrap(); + assert_eq!(kept.get_text("state"), prior.get_text("state")); + assert_eq!(kept.get_text("code"), prior.get_text("code")); + assert_eq!(kept.get("forced"), prior.get("forced")); + assert_eq!(kept.get_int("startedMs"), prior.get_int("startedMs")); + assert!(forceable(&kept), "the earlier record answers as it did"); +} + +#[test] +fn a_busy_force_keeps_the_record_it_replaced() { + let dir = fixture("busy"); + let file = record_file(&dir); + let prior = completed(); + discovery::save(&file, prior.clone()).unwrap(); + let attempt = begin_at(file.clone(), "/w", true).unwrap(); + assert_eq!( + discovery::load(&file).unwrap().get_text("state"), + Some("Unmounting") + ); + finish(&attempt, "/w", &Err(refused("Busy"))).unwrap(); + assert_restored(&file, &prior); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_dirty_force_keeps_the_record_it_replaced() { + let dir = fixture("dirty"); + let file = record_file(&dir); + let prior = completed(); + discovery::save(&file, prior.clone()).unwrap(); + let attempt = begin_at(file.clone(), "/w", true).unwrap(); + finish(&attempt, "/w", &Err(refused("Dirty"))).unwrap(); + assert_restored(&file, &prior); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_call_that_sent_nothing_keeps_the_record_it_replaced() { + let dir = fixture("cancel"); + let file = record_file(&dir); + let prior = completed(); + discovery::save(&file, prior.clone()).unwrap(); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + cancel(&attempt).unwrap(); + assert_restored(&file, &prior); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_first_attempt_that_does_not_run_leaves_no_record() { + let dir = fixture("first"); + let file = record_file(&dir); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + finish(&attempt, "/w", &Err(refused("Busy"))).unwrap(); + assert!(!file.exists()); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} From a7ca6c91132bc0bbbf77f4ee3b7be0ddf9f38be2 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:48:56 +0000 Subject: [PATCH 06/14] docs(daemon,sandbox): document the native NFS forced teardown The native route's forced teardown and the mount-table checks the review asked for were only in source. Document them where the guides already are: - `engine/daemon/forced-teardown.md`: a "Native NFS route" section - the one forced client unmount standing in for the abort write, the two shapes of `Retained(Detach)` and when the mount counts as gone, why the mount point is never resolved, and the shared retained queue; the guard step 1 and the rule table name the forceable export custody; the test-only hooks (`mount_table`, `reap_wait`, `reaped_helpers`) are named as test-only. - `engine/daemon/control-calls.md`: the NFS route's Attach (table read once), Unmount (failure settled by the table) and Helper run rows. - `engine/kernel-routes/nfs.md`: the Attach-state read and its one extra `/sbin/mount` run (or `/proc/self/mountinfo` read) per Attach, the mount point never being resolved, the failure/deadline rule and the retained queue. - `api/sdk/workspace/unmount.md`: `force` also finishes an entry kept at Detach on the native NFS route; the recorded-outcome rule says which record a force may replace and that an older record without `forced` is not forceable. - `force.rs`: the guard comment names both exceptions to "stored custody answers first", and the module header names the NFS abort. Production LOC: 160215 -> 160215 (delta 0); comments and docs only. Method: python3 tools/production_loc.py, combined scope, first parent 523462afc and this commit's staged tree; core 39787 -> 39787 (0), engine 74627 -> 74627 (0), control 39825 -> 39825 (0), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- .../architecture/api/sdk/workspace/unmount.md | 14 ++++-- .../engine/daemon/control-calls.md | 8 ++-- .../engine/daemon/forced-teardown.md | 48 ++++++++++++++++++- docs/architecture/engine/kernel-routes/nfs.md | 16 ++++++- .../layerfs-daemon/src/control/force.rs | 11 +++-- 5 files changed, 84 insertions(+), 13 deletions(-) diff --git a/docs/architecture/api/sdk/workspace/unmount.md b/docs/architecture/api/sdk/workspace/unmount.md index 2762168564..43ea189497 100644 --- a/docs/architecture/api/sdk/workspace/unmount.md +++ b/docs/architecture/api/sdk/workspace/unmount.md @@ -44,7 +44,7 @@ layerfs workspace unmount [--discard] [--force] | Name | Kind | Default | Rule | Source | | --- | --- | --- | --- | --- | | `discard` | Bool | false | drop uncommitted changes instead of refusing `Dirty` | [`workspace/serve.rs`](../../../../../control/crates/layerfs-sandbox/src/workspace/serve.rs) | -| `force` | Bool | false | each running command is cancelled first ([workspace.exec_cancel](exec_cancel.md), 5000 ms grace); one abort write ends the connection, also after a normal unmount stalled at Join; uncommitted changes are dropped | same, [`exec/cancel.rs`](../../../../../control/crates/layerfs-sandbox/src/exec/cancel.rs) `settle` | +| `force` | Bool | false | each running command is cancelled first ([workspace.exec_cancel](exec_cancel.md), 5000 ms grace); one abort write ends the connection, also after a normal unmount stalled at Join; on the native NFS route one forced client unmount takes its place, and also finishes an entry kept at Detach when a normal unmount's helper failed or ran out of time while the client mount stayed; uncommitted changes are dropped | same, [`exec/cancel.rs`](../../../../../control/crates/layerfs-sandbox/src/exec/cancel.rs) `settle`, [`force.rs`](../../../../../engine/crates/layerfs-daemon/src/control/force.rs) | ## Returns @@ -85,7 +85,13 @@ A Workspace the daemon no longer knows (`Missing`) answers `Unmounted`. ``` The recorded outcome makes a repeated unmount answer what the first one -established, without dialling the daemon again. The daemon's kernel probe +established, without dialling the daemon again. One completed record is the +exception: a `Failed` unmount that was not itself forced, which a `force` +call may replace with a forced attempt (the daemon's own guard still decides +whether that force may proceed). A record that is still in progress, one of +an unknown outcome, and one that a force already produced are answered as +they stand; a record written before `forced` was recorded (no such field) is +answered as it stands too, and is not forceable. The daemon's kernel probe answers `Busy` while a process still holds the mount. See the [routing table](../../../control/routing/routing-table.md), [Engine: daemon](../../../engine/daemon/README.md) and the @@ -107,7 +113,9 @@ answers `Busy` while a process still holds the mount. See the **After an uncertain unmount**, call `workspace.unmount` again: a recorded outcome is answered as it was, and a Workspace already gone is `NotMounted` or `Unmounted`. [sandbox.get](../sandbox/get.md) lists what is -still mounted. A stalled normal unmount can be ended with `force`. +still mounted. A stalled normal unmount can be ended with `force`, and so can +an unmount kept `Retained` at detach on the native NFS route while the client +mount stays ([engine: forced teardown](../../../engine/daemon/forced-teardown.md)). ## Examples diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index 44eed0c3e0..4abe773eb1 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -86,11 +86,11 @@ differs. | Step | NFS route | Source | | --- | --- | --- | -| Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the host client mounts it with the mount helper; Ready only after the helper reports the mount. An NFS receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the host client mounts it with the mount helper; Ready only after the helper reports the mount, and the host's mount table is read once to record whether the mount is listed. An NFS receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Commit | Flush first (flush helper, or `sync(2)`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | -| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| ForceUnmount | Force helper once, then the same teardown | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| Helper run | One attempt, at most `HELPER_WAIT` (60 s), then killed and reported | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close. A failed or timed-out helper is settled by the mount table: a mount listed at Attach and gone now completes the teardown, a still listed one (or an unreadable table) keeps `Retained(Detach)` | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| ForceUnmount | Force helper once, then the same teardown; also finishes an export kept at `Detach` whose client mount stayed | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Helper run | One attempt, at most `HELPER_WAIT` (60 s), then killed and reported; a child still unreaped past `REAP_WAIT` goes to the bounded retained queue | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | Kernel states `Exported`, `ExportLeaving` and `ExportKept` mirror `Ready`, `Leaving` and `Retained`. Details: [Kernel routes: NFS](../kernel-routes/nfs.md). diff --git a/docs/architecture/engine/daemon/forced-teardown.md b/docs/architecture/engine/daemon/forced-teardown.md index 5ad59278d3..905914907c 100644 --- a/docs/architecture/engine/daemon/forced-teardown.md +++ b/docs/architecture/engine/daemon/forced-teardown.md @@ -20,7 +20,8 @@ process. Each step is made once. ```text registry lock: guard (in this order; a refusal changes nothing) 1 ExportKept / Retained custody -> Retained (except a forceable - normal unmount stalled at Join, which is admitted) + normal unmount stalled at Join, or a forceable export kept at + Detach, which are admitted) 2 Committing | Attaching | Closing -> Busy (force:admission) 3 no serving connection -> Invalid / Unknown / Busy 4 Commit knowledge: Idle -> Absent; LocalFailure + published -> @@ -52,11 +53,56 @@ reply ForceUnmounted { token, ForcedOutcome { facts, work, cleanup } } | --- | --- | | The abort write uses the connection's own control, bound from the daemon's `fusectl` tree at Attach. Without it, Force is refused before any effect. | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`application/filesystem.rs`](../../../../engine/crates/layerfs-daemon/src/application/filesystem.rs) | | A normal unmount stalled at `Join` with its abort control unused is forced from its retained session; its plain detach counts as the one detach. | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`serving.rs`](../../../../engine/crates/layerfs-daemon/src/control/serving.rs) `Kept::forceable` | +| An export kept at `Detach` whose export and receipt are still held, and whose forced facts are unset, is forced by the one forced client unmount. | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `ExportKept::forceable` | | `relinquish_unknown` allows local disposal of stopped unknown Commit custody; the receipt keeps it `Unknown`. | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`control_request.rs`](../../../../engine/crates/layerfs-bridge/src/control_request.rs) | | On the NFS route the force helper is the abort; the same teardown follows. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Outside Linux without NFS, the guard runs and the call is refused (`kernel serving requires Linux`). | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | | No daemon-wide graceful drain exists. | [engine guide](../../../../engine/AGENTS.md) | +## Native NFS route + +An export-owning entry reaches `ForceUnmounted` through the same guard, with +the forced client unmount standing in for the abort write. Nothing about +the client mount is repeated: one `umount -f` (macOS), or one +`umount -f -l` (Linux, in the daemon's container). + +```text +registry lock: guard; take the export out of the entry (Kernel::ExportLeaving) +----------------------------------------------------------------- +force helper, once -- Busy/Interrupted, mount still listed --> facts.detach + Failed, Retained(Detach) +then the normal teardown: Server::stop, kernel_revoke, remove the directory, +the one Close, observe_cleanup, depart +``` + +Normal `Unmount` reaches `Retained(Detach)` only when the helper's failure or +deadline cannot be settled, and it keeps that custody in two shapes: + +| Shape | Meaning | Source | +| --- | --- | --- | +| Attach never saw the mount listed | The client may or may not hold it; nothing may be revoked through it | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `Exported::tracked` | +| The mount is still listed, or the table cannot be read | Conservative custody; the export keeps serving | same, `detached_export` | + +The mount is only known gone when Attach saw it listed (`tracked`) and the +host's table no longer has it; teardown then completes as a normal unmount +and the reply is `Unmounted`. The table is the host's (`mount` on macOS, +`/proc/self/mountinfo` on Linux), read once. The mount point itself is never +resolved: that lookup goes through the mount, so against a wedged client it +can block with no bound and it fails exactly while the mount is in trouble, +which would read as "not mounted". The check therefore compares the path as +configured and the same name below the canonical mount home. + +A helper that outlives its `KILL` and the post-KILL window enters a bounded +retained queue, whose one background thread reaps it and records the exit; +nothing waits for it and nothing signals it twice. The same custody serves +the macOS driver's `umount -f` ([`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs)). + +For tests only, `ExportServingConfig` carries a substitute mount table +(`mount_table`) and a zero post-KILL window (`reap_wait`, with the bounded +`helper_wait`), and `ExportServing::reaped_helpers` reports the exits observed +so far. Production wiring sets none of them, and no daemon path reads the +report. + ## Cleanup | `CleanupState` / `ForcedCleanup` | Meaning | Source | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 7b64bf49ed..c4e9e72e51 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -207,11 +207,22 @@ it starts with `._` and has at least one more byte. bind 127.0.0.1:0 -> Server::start(...) port P helper: mount_nfs -o vers=3,tcp,port=P,mountport=P,... 127.0.0.1:/ client --MOUNT MNT "/"--> root handle + AUTH_UNIX + read the host's mount table once -> tracked: is listed? daemon Commit: flush helper (sync(2) by default), close_files, Commit - daemon Unmount: unmount helper (busy is reversible), Server::stop, + daemon Unmount: unmount helper (busy is reversible); a failure or deadline + is settled by the mount table (see below); Server::stop, revoke the engine mount, Close + daemon ForceUnmount: one forced client unmount (umount -f, or umount -f -l + on Linux), then the same teardown ``` +| Rule | Source | +| --- | --- | +| Attach reads the host's mount table once, after the mount helper returned, so a later failed or timed-out unmount helper can be judged. The cost is one `/sbin/mount` run (macOS) or one `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `Exported::tracked`, [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `mounted_under` | +| The mount point itself is never resolved when the table is read or compared: that lookup goes through the mount. The comparison is the path as configured, plus the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | +| A failed or timed-out unmount helper completes teardown only when Attach saw the mount listed and the table no longer lists it; a still listed mount, or an unreadable table, keeps `Retained(Detach)` and the export serving. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `detached_export` | +| A helper that outlives its `KILL` and the post-KILL window is reaped in the background by the shared retained queue; nothing waits for it and it is not signalled twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | + | MOUNT procedure | Answer | Source | | --- | --- | --- | | 0 `NULL`, 4 `UMNTALL` | Empty success | [`mount.rs`](../../../../engine/crates/layerfs-nfs/src/mount.rs) | @@ -225,7 +236,8 @@ The client mount options are the daemon's `Mounter::host`. On macOS: elsewhere `vers=3,proto=tcp,port={port},mountport={port},mountproto=tcp,nolock,actimeo=60` ([`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs)). Helper deadlines and teardown belong to the -[daemon](../daemon/README.md). +[daemon](../daemon/README.md); the forced route is +[forced teardown](../daemon/forced-teardown.md#native-nfs-route). ## Related diff --git a/engine/crates/layerfs-daemon/src/control/force.rs b/engine/crates/layerfs-daemon/src/control/force.rs index f4e7d24dd3..2ad6b6254a 100644 --- a/engine/crates/layerfs-daemon/src/control/force.rs +++ b/engine/crates/layerfs-daemon/src/control/force.rs @@ -1,7 +1,8 @@ //! Forced terminal unmount: a before-effect guard, one abort write, the local //! drain with its one plain detach, then Revoke and Close. Nothing is repeated. //! A normal unmount stalled at Join is forced the same way, its plain detach -//! counting as the one detach. +//! counting as the one detach; on the NFS route the one forced client unmount +//! is the abort. #[cfg(target_os = "linux")] use super::serving::{stage, work, Attached, Evidence, Leaving}; use super::{ @@ -51,8 +52,12 @@ pub(super) fn guard( relinquish_unknown: bool, ) -> Result { // Stored custody answers first: no second abort or detach ever follows - // it. The one exception is a normal unmount stalled at Join, admitted - // idle with no Commit custody, whose abort control is still unused. + // it. Two shapes are the exception, both admitted with nothing yet + // aborted or detached: an export whose client unmount is not established + // (`ExportKept` at `Detach`, export still held, see + // `ExportKept::forceable`), where the one forced client unmount is the + // abort; and a normal unmount stalled at Join, admitted idle with no + // Commit custody, whose abort control is still unused. if let Kernel::ExportKept(kept) = &value.kernel { if kept.forceable() { return Ok(CommitKnowledge::Absent); From 6dc8a81dc2ec9762d56dc1bb818c2bebc0675fba Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 04:11:04 +0000 Subject: [PATCH 07/14] docs(daemon,nfs,api): tighten the forced teardown and mount-table wording Say which record a forced unmount may replace in one sentence, give the native route's Alpine flow one witness per step, and cite each mount-rule row's source consistently. Production LOC: 160215 -> 160215 (delta 0); docs only. Method: python3 tools/production_loc.py, combined scope, first parent a7ca6c911 and this commit's staged tree; core 39787 -> 39787 (0), engine 74627 -> 74627 (0), control 39825 -> 39825 (0), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- docs/architecture/api/sdk/workspace/unmount.md | 16 ++++++++-------- .../engine/daemon/forced-teardown.md | 8 +++++--- docs/architecture/engine/kernel-routes/nfs.md | 7 ++++--- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/architecture/api/sdk/workspace/unmount.md b/docs/architecture/api/sdk/workspace/unmount.md index 43ea189497..689d9c0fc8 100644 --- a/docs/architecture/api/sdk/workspace/unmount.md +++ b/docs/architecture/api/sdk/workspace/unmount.md @@ -85,14 +85,14 @@ A Workspace the daemon no longer knows (`Missing`) answers `Unmounted`. ``` The recorded outcome makes a repeated unmount answer what the first one -established, without dialling the daemon again. One completed record is the -exception: a `Failed` unmount that was not itself forced, which a `force` -call may replace with a forced attempt (the daemon's own guard still decides -whether that force may proceed). A record that is still in progress, one of -an unknown outcome, and one that a force already produced are answered as -they stand; a record written before `forced` was recorded (no such field) is -answered as it stands too, and is not forceable. The daemon's kernel probe -answers `Busy` while a process still holds the mount. See the +established, without dialling the daemon again. The exception is a completed +`Failed` record that was not itself forced: a `force` call may replace it +with a forced attempt, and the daemon's own guard still decides whether that +force may proceed. A record that is still in progress, one of an unknown +outcome, and one a force already produced are answered as they stand; so is a +record written before `forced` was stored (no such field), which is therefore +not forceable. The daemon's kernel probe answers `Busy` while a process still +holds the mount. See the [routing table](../../../control/routing/routing-table.md), [Engine: daemon](../../../engine/daemon/README.md) and the [engine teardown pages](../../../engine/kernel-routes/README.md). diff --git a/docs/architecture/engine/daemon/forced-teardown.md b/docs/architecture/engine/daemon/forced-teardown.md index 905914907c..bbac8a92ef 100644 --- a/docs/architecture/engine/daemon/forced-teardown.md +++ b/docs/architecture/engine/daemon/forced-teardown.md @@ -68,9 +68,11 @@ the client mount is repeated: one `umount -f` (macOS), or one ```text registry lock: guard; take the export out of the entry (Kernel::ExportLeaving) ------------------------------------------------------------------ -force helper, once -- Busy/Interrupted, mount still listed --> facts.detach - Failed, Retained(Detach) +------------------------------------------------------------------ +one force helper run + Done, or the mount is no longer listed -> facts.detach = Detached + the mount is still listed --------------> facts.detach = Failed, + Retained(Detach), facts kept then the normal teardown: Server::stop, kernel_revoke, remove the directory, the one Close, observe_cleanup, depart ``` diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index c4e9e72e51..b80255c5bd 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -218,10 +218,11 @@ it starts with `._` and has at least one more byte. | Rule | Source | | --- | --- | -| Attach reads the host's mount table once, after the mount helper returned, so a later failed or timed-out unmount helper can be judged. The cost is one `/sbin/mount` run (macOS) or one `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `Exported::tracked`, [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `mounted_under` | +| Every step above is one attempt; the mount table is the only witness for a helper that failed or ran out of time. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach reads the host's mount table once, after the mount helper returned, so a later failed or timed-out unmount helper can be judged. The cost is one `/sbin/mount` run (macOS) or one `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` | | The mount point itself is never resolved when the table is read or compared: that lookup goes through the mount. The comparison is the path as configured, plus the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | -| A failed or timed-out unmount helper completes teardown only when Attach saw the mount listed and the table no longer lists it; a still listed mount, or an unreadable table, keeps `Retained(Detach)` and the export serving. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `detached_export` | -| A helper that outlives its `KILL` and the post-KILL window is reaped in the background by the shared retained queue; nothing waits for it and it is not signalled twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| A failed or timed-out unmount helper completes teardown only when Attach saw the mount listed and the table no longer lists it; a still listed mount, or an unreadable table, keeps `Retained(Detach)` and the export serving. | `export.rs` `detached_export` | +| A helper that outlives its `KILL` and the post-KILL window is reaped in the background by the shared retained queue; nothing waits for it and it is not signalled twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | MOUNT procedure | Answer | Source | | --- | --- | --- | From bbd15c3a72b5a7f0f529eea5dc5a88d35ef7eec7 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:41:51 +0000 Subject: [PATCH 08/14] docs(daemon,nfs,api): port native NFS forced teardown docs to v2.0 layout Port the #644 native NFS mount-table, retained helper queue, ExportKept force, and unmount_guard record-retention updates into the reorganized upstream architecture docs: - engine/daemon/control-calls.md: Export table (Attach mount-table read, Unmount mount-table settlement, ForceUnmount for Detach-kept exports, Helper run retained queue), Force diagram step 1, and Force rules (ExportKept::forceable, Retained(Detach) custody shapes, listed without resolving mount point, shared retained queue, test-only ExportServingConfig/reaped_helpers hooks). - engine/kernel-routes/nfs.md: Mount diagram and rules (Attach mount-table read and cost, listed without resolving mount point, Unmount/ForceUnmount settlement, shared retained queue, link to Control calls Force). - api/sdk/workspace.md: workspace.unmount force option on the native NFS route, and recorded-outcome replacement/restore rules for workspace/.unmount. Docs: updated Daemon, Kernel routes (NFS), and SDK workspace topics. Production LOC: 160215 -> 160215 (delta 0); docs only. Method: python3 tools/production_loc.py, combined scope, first parent 9a9012e3d and this commit's staged tree; core 39787 -> 39787 (0), engine 74627 -> 74627 (0), control 39825 -> 39825 (0), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- docs/architecture/api/sdk/workspace.md | 11 +++++--- .../engine/daemon/control-calls.md | 14 +++++++---- docs/architecture/engine/kernel-routes/nfs.md | 25 ++++++++++++++----- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/docs/architecture/api/sdk/workspace.md b/docs/architecture/api/sdk/workspace.md index 4fac960411..8407bca6a8 100644 --- a/docs/architecture/api/sdk/workspace.md +++ b/docs/architecture/api/sdk/workspace.md @@ -208,7 +208,7 @@ CLI layerfs workspace unmount [--discard] [--force] | Option | Effect | | --- | --- | | `discard` | drop uncommitted changes instead of refusing `Dirty` | -| `force` | cancel each running command first (5000 ms grace), then one abort write ends the connection, also after a normal unmount stalled; uncommitted changes are dropped | +| `force` | cancel each running command first (5000 ms grace), then one abort write ends the connection (also after a normal unmount stalled); on the native NFS route one forced client unmount takes its place and also finishes an entry kept at `Detach` when a normal unmount's helper failed or timed out while the client mount stayed; uncommitted changes are dropped | ```text normal: { kind: "Unmounted" } @@ -218,8 +218,13 @@ either may add cleanup_error (daemon succeeded; a local record stayed) The machine writes `workspace/.unmount` before it calls the daemon and records the outcome after, so a repeated unmount answers what the first one -established without dialling again; a Workspace the daemon no longer knows -answers `Unmounted` +established without dialling again. The exception is a completed `Failed` +record that was not itself forced: `force` may replace it with a forced +attempt (the daemon's guard remains the real gate, and `Busy`, `Dirty`, or +an unsent call restores the earlier record); an in-progress, `Unknown`, +already-forced, or pre-`forced` record (no `forced` field) keeps its answer +and is not forceable. A Workspace the daemon no longer knows answers +`Unmounted` ([`unmount_guard.rs`](../../../../control/crates/layerfs-sandbox/src/unmount_guard.rs)). | Error | When | diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index 5064020a1e..d3a128ebe4 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -96,11 +96,11 @@ differs. `Exported`, `ExportLeaving` and `ExportKept` mirror `Ready`, | Step | NFS route | Source | | --- | --- | --- | -| Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount, and the host's mount table is read once to record whether the mount is listed. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Commit | Flush first (flush helper, or `sync(2)`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | -| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| ForceUnmount | Force helper once, then the same teardown | same | -| Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close. A failed or timed-out helper is settled by the mount table: a mount listed at Attach and gone now completes teardown, while a still-listed one or an unreadable table keeps `Retained(Detach)` | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| ForceUnmount | Force helper once, then the same teardown; also finishes an export kept at `Detach` whose client mount stayed | same | +| Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported; a child still unreaped past `REAP_WAIT` 5 s enters the bounded retained queue | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | ## Unmount @@ -156,7 +156,8 @@ each step is made once. ```text under the registry lock (a refusal changes nothing): 1 ExportKept / Retained -> Retained (a normal unmount stalled at - Join is admitted instead) + Join, or a forceable export + kept at Detach, is admitted) 2 Committing | Attaching | Closing -> Busy force:admission 3 no serving connection -> Invalid / Unknown / Busy 4 Commit knowledge: Idle -> Absent; LocalFailure + published -> @@ -191,6 +192,9 @@ reply ForceUnmounted { token, ForcedOutcome { facts, work, cleanup } } | --- | --- | | The abort write uses the connection's own control, bound from the daemon's `fusectl` tree at Attach; without it Force is refused before any effect | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`application/filesystem.rs`](../../../../engine/crates/layerfs-daemon/src/application/filesystem.rs) | | A normal unmount stalled at `Join` with its abort unused is forced from its retained session; its plain detach counts as the one detach | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`serving.rs`](../../../../engine/crates/layerfs-daemon/src/control/serving.rs) | +| An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `ExportKept::forceable` | +| Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), or when the table still lists it or cannot be read (`detached_export`); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| A helper past `KILL` and `REAP_WAIT` enters the bounded retained queue, whose one background thread reaps and records its exit (shared with the macOS driver's `umount -f`). For tests only, `ExportServingConfig` adds `mount_table`, `reap_wait` and `helper_wait`, and `ExportServing::reaped_helpers` reads observed exits | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | `relinquish_unknown` permits local disposal of stopped unknown Commit custody; the receipt keeps it `Unknown` | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | | On the NFS route the force helper is the abort | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Outside Linux without NFS the guard runs and the call is refused (`kernel serving requires Linux`) | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 67e53bb052..03b5548eab 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -172,14 +172,25 @@ reaches the Workspace or a Commit; everything is dropped with the export. ## Mount (`mount.rs`, daemon `export.rs`) ```text - Attach: bind 127.0.0.1:0 -> Server::start -> port P - helper mount_nfs -o vers=3,tcp,port=P,mountport=P,... - client MOUNT MNT "/" -> root handle, AUTH_UNIX - Commit: flush helper (sync(2) by default), close_files, Commit - Unmount: unmount helper (busy is reversible), Server::stop, - revoke the engine mount, Close + Attach: bind 127.0.0.1:0 -> Server::start -> port P + helper mount_nfs -o vers=3,tcp,port=P,mountport=P,... + client MOUNT MNT "/" -> root handle, AUTH_UNIX + read host mount table once -> tracked: is listed? + Commit: flush helper (sync(2) by default), close_files, Commit + Unmount: unmount helper (busy is reversible); failure or deadline + settled by the mount table; Server::stop, revoke, Close + ForceUnmount: one forced client unmount (umount -f, or umount -f -l on + Linux), then the same teardown ``` +| Mount-table and helper rule | Source | +| --- | --- | +| Every step above is one attempt; the mount table is the only witness for a failed or timed-out unmount helper. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| The mount point itself is never resolved (a lookup through a wedged mount can block or fail); `listed` compares the configured path and the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | +| A failed or timed-out unmount helper completes teardown only when Attach saw it listed and the table no longer does; a still-listed mount or unreadable table keeps `Retained(Detach)` and the export serving. | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| A helper past `KILL` and `REAP_WAIT` is reaped in the background by the shared retained queue, with nothing waiting for it or signalling twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | + | MOUNT procedure | Answer | | --- | --- | | 0 `NULL`, 4 `UMNTALL` | Empty success | @@ -192,6 +203,8 @@ Client options are the daemon's `Mounter::host`. macOS: `vers=3,proto=tcp,port={port},mountport={port},mountproto=tcp,nolock,actimeo=60` ([`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs)). An NFS ready receipt has `abi_major == 0` and its `mount_id` is the port. +Forced teardown on this route is in +[Daemon: Control calls](../daemon/control-calls.md#force). ## Application SQLite databases on the NFS route From b49690d1194699401445c1493ab8a664136baae8 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:42:59 +0000 Subject: [PATCH 09/14] fix(sandbox): align forceable comment and record macOS synchronous reap Address the two optional P3 items from the #644 re-review: - unmount_guard.rs: remove the contradictory "forced or not" phrase from the forceable() comment (completed Failed records with forced == false only). - darwin.rs: call RETAINED.record(pid, status) in the 5 s post-KILL synchronous reap branch, matching the background queue branch and daemon export_helper.rs. Docs: no architecture impact (comment wording and in-memory diagnostic exit record alignment). Production LOC: 160215 -> 160218 (delta +3) Method: python3 tools/production_loc.py, combined scope, first parent bbd15c3a7 and this commit's staged tree; core 39787 -> 39787 (0), engine 74627 -> 74627 (0), control 39825 -> 39828 (+3), api 5976 -> 5976 (0), reference absent (retired R9c; last counted 65,417 at tree 498dd1917812), dsh 0 -> 0 (0). Ref: layerfs-project#136 --- .../layerfs-sandbox-backend/src/local/os/macos/darwin.rs | 5 ++++- control/crates/layerfs-sandbox/src/unmount_guard.rs | 7 +++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs index f22e5b9e35..e0a39e7441 100644 --- a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs +++ b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs @@ -108,7 +108,10 @@ pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { } }; let disposition = match reaped { - Some(status) => format!("exit reaped ({status})"), + Some(status) => { + RETAINED.record(pid, status); + format!("exit reaped ({status})") + } None => { RETAINED.retain(child); "exit not observed after 5s; child custody retained (possible kernel wait)" diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index bb796136b7..09dcdc91f0 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -232,10 +232,9 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { Ok(recoverable) } pub fn forceable(value: &SdkValue) -> bool { - // A completed `Failed` record, forced or not, is the one shape a force - // may replace: the daemon's own guard is the real gate on whether force - // may proceed, and a refusal answers with the record it holds. A record - // that is still `Unmounting` (its caller ended without publishing), + // A completed `Failed` record is the one shape a force may replace: the + // daemon's own guard is the real gate on whether force may proceed, and a + // refusal answers with the record it holds. A record that is still `Unmounting` (its caller ended without publishing), // `Unknown` or already `forced` keeps its answer. A record written before // this rule (no `forced` field) is answered as it stands and is not // forceable. From e710662d9995a8b6d45d011703d1ac8f1e406122 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:22:09 +0000 Subject: [PATCH 10/14] fix(daemon): never trust a lazy detach as proof the NFS mount is gone Review item 1 (blocking): on Linux the forced helper is `umount -f -l`. Its lazy detach removes the mount from mountinfo while the kernel still holds the superblock, so a vanished table entry never proves the client's mount ended; `detached_export` treated it as done, then stopped the NFS server and closed the Workspace. - `Helper::lazy()` reads the helper's own words (`-l`, `--lazy`, or a combined short word such as `-fl`). - `detached_export` now requires three things: Attach saw the mount listed (`tracked`), the helper cannot detach lazily, and the table does not list the mount now. A failed or timed-out lazy helper keeps `Retained(Detach)` with the export still serving, on Unmount and on ForceUnmount. - The ordinary helper (no `-l`) and macOS keep the table as witness, so the existing exit-and-table tests are unchanged. - Docs: control-calls.md (Force and Unmount rules) and core nfs.md state that a lazy detach is never proof. Test: `a_lazily_detached_force_helper_never_completes_teardown` (engine/crates/layerfs-nfs/tests/helper_deadline.rs): the forced helper removes the table entry, fails, and carries `-l`; ForceUnmount must answer `Retained(Detach)` with `detach: Failed`, keep the Workspace `Retained`, the mount directory and the export, and release nothing. Reversed (lazy guard removed) the same test fails: ForceUnmount answers `ForceUnmounted` with `detach: Detached` and `cleanup: Queued`. Checks: `cargo +1.85.1 test --locked -p layerfs-nfs --test helper_deadline` 8 passed, 0 failed. Durable: NOT_RUN - disabled by owner until explicit reauthorization. Production LOC: 163746 -> 163755 (delta +9); parent 8bc995f3f. core 39787, engine 75078 -> 75087, control 42804, api 6077. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root . --- .../engine/daemon/control-calls.md | 4 +- docs/architecture/engine/kernel-routes/nfs.md | 2 +- .../layerfs-daemon/src/control/export.rs | 19 ++- .../src/control/export_helper.rs | 12 ++ .../layerfs-nfs/tests/helper_deadline.rs | 126 ++++++++++++++++++ 5 files changed, 154 insertions(+), 9 deletions(-) diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index ef37fa584e..0588a4e625 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -199,8 +199,8 @@ reply ForceUnmounted { token, ForcedOutcome { facts, work, cleanup } } | --- | --- | | The abort write uses the connection's own control, bound from the daemon's `fusectl` tree at Attach; without it Force is refused before any effect | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`application/filesystem.rs`](../../../../engine/crates/layerfs-daemon/src/application/filesystem.rs) | | A normal unmount stalled at `Join` with its abort unused is forced from its retained session; its plain detach counts as the one detach | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`serving.rs`](../../../../engine/crates/layerfs-daemon/src/control/serving.rs) | -| An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `ExportKept::forceable` | -| Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), or when the table still lists it or cannot be read (`detached_export`); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts. On Linux that helper detaches lazily, so its failure never counts as a detach even when the table no longer lists the mount | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `ExportKept::forceable` | +| Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), when the table still lists it or cannot be read (`detached_export`), or when the helper detaches lazily (`-l`/`--lazy` in its own words: the mount leaves the table before the kernel gives its superblock back, so the table proves nothing); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | | A helper past `KILL` and `REAP_WAIT` enters the bounded retained queue, whose one background thread reaps and records its exit (shared with the macOS driver's `umount -f`). For tests only, `ExportServingConfig` adds `mount_table`, `reap_wait` and `helper_wait`, and `ExportServing::reaped_helpers` reads observed exits | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | `relinquish_unknown` permits local disposal of stopped unknown Commit custody; the receipt keeps it `Unknown` | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | | On the NFS route the force helper is the abort | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 535a003714..4eccecd3d5 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -200,7 +200,7 @@ reaches the Workspace or a Commit; everything is dropped with the export. | Every step above is one attempt; the mount table is the only witness for a failed or timed-out unmount helper. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | The mount point itself is never resolved (a lookup through a wedged mount can block or fail); `listed` compares the configured path and the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | -| A failed or timed-out unmount helper completes teardown only when Attach saw it listed and the table no longer does; a still-listed mount or unreadable table keeps `Retained(Detach)` and the export serving. | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| A failed or timed-out unmount helper completes teardown only when Attach saw it listed and the table no longer does; a still-listed mount, an unreadable table, or a helper that detaches lazily (`-l`) keeps `Retained(Detach)` and the export serving. | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | A helper past `KILL` and `REAP_WAIT` is reaped in the background by the shared retained queue, with nothing waiting for it or signalling twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | MOUNT procedure | Answer | diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 521737eb0c..2c55498aca 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -21,8 +21,9 @@ //! and runs the one logical Close. A step that is not established keeps //! the export and its engine mount in the entry, as FUSE custody is kept. //! A helper that failed or ran out of time is judged by the host's mount -//! table, never by the mount itself: only a mount Attach saw listed there -//! can be known gone (`export_helper::listed`). +//! table, never by the mount itself: only a mount Attach saw listed there, +//! and only an ordinary (not lazy) helper, can be known gone +//! (`export_helper::listed`, `Helper::lazy`). //! - ForceUnmount runs the forced helper once, then the same teardown. It is //! also how an export kept at Detach for a mount still listed is finished. //! A helper that outlives its KILL goes to the bounded retained queue. @@ -706,7 +707,7 @@ impl Service { format!("client reported the mount busy; no terminal effect: {detail}"), )); } - Ran::Failed(_) | Ran::Interrupted(_) if self.detached_export(&exported) => {} + Ran::Failed(_) | Ran::Interrupted(_) if self.detached_export(&exported, helper.as_ref()) => {} Ran::Failed(detail) | Ran::Interrupted(detail) => { return Err(self.keep_export( token, @@ -720,8 +721,14 @@ impl Service { self.teardown_export(token, &workspace, exported, None)?; self.close(token, &workspace, None) } - fn detached_export(&self, exported: &Exported) -> bool { - exported.tracked + /// Whether a failed or timed-out helper proves the client mount is gone: + /// Attach saw the mount listed (`tracked`), the helper cannot detach + /// lazily, and the host's table does not list it now. A lazy detach + /// (`umount -l`, the Linux force helper) removes the table entry while the + /// kernel still holds the superblock, so it is never proof. + fn detached_export(&self, exported: &Exported, helper: Option<&Helper>) -> bool { + !helper.is_some_and(Helper::lazy) + && exported.tracked && (self.export.as_ref()) .is_some_and(|s| matches!(s.is_mounted(&exported.ready.directory), Ok(false))) } @@ -892,7 +899,7 @@ impl Service { match ran { Ran::Done => {} Ran::Busy(_) | Ran::Failed(_) | Ran::Interrupted(_) - if self.detached_export(&exported) => {} + if self.detached_export(&exported, helper.as_ref()) => {} Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { facts.detach = DetachDisposition::Failed; return Err(self.keep_export( diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index 828dc6a5c0..7219e048db 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -24,6 +24,18 @@ impl Helper { let words: Vec = line.split_whitespace().map(str::to_owned).collect(); (!words.is_empty()).then_some(Self(words)) } + /// Whether this helper can detach lazily: `-l`/`--lazy` (also inside a + /// combined word such as `-fl`) asks the kernel to drop the mount from the + /// host's table before its superblock goes. The table then cannot witness + /// that the client's mount ended, so a vanished entry proves nothing. + pub fn lazy(&self) -> bool { + self.0.iter().any(|word| { + word == "--lazy" + || (word.starts_with('-') + && !word.starts_with("--") + && word.contains('l')) + }) + } fn command(&self, port: u16, directory: &str) -> Option { let mut words = self.0.iter().map(|word| { word.replace("{port}", &port.to_string()) diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs index 13f72d617f..490a0e1a3e 100644 --- a/engine/crates/layerfs-nfs/tests/helper_deadline.rs +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -424,6 +424,132 @@ fn timed_out_helper_when_mount_already_gone_completes_teardown() { worker.join().unwrap(); } +#[test] +fn a_lazily_detached_force_helper_never_completes_teardown() { + // `umount -f -l` detaches lazily: the mount leaves the host's table while + // the kernel still holds its superblock. A vanished entry is therefore no + // proof that the client's mount ended, and Force must keep the custody. + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("lazy-force"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + // `-l` is what the Linux force helper (`umount -f -l`) passes: + // the helper fails, yet the entry is gone from the table. + force: Helper(vec![ + "sh".into(), + "-c".into(), + format!("rm -f {}; exit 1", table.display()), + "-l".into(), + ]), + flush: None, + }, + ); + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([38; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // The plain helper fails while the table still lists the mount. + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + } + other => panic!("expected Retained, got {other:?}"), + } + + // The forced helper fails too, and the entry is gone from the table: + // a lazy detach, so the mount is not known gone. + let err = service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + assert_eq!( + custody.forced.as_ref().map(|facts| facts.detach), + Some(DetachDisposition::Failed) + ); + } + other => panic!("expected Retained, got {other:?}"), + } + assert!(Path::new(&ready.directory).exists()); + match service.execute_control(&Request::Status(token)) { + Ok(done) => match done.reply { + Reply::Status(status) => assert_eq!( + status.kernel.map(|kernel| kernel.phase), + Some(MountPhase::Retained) + ), + other => panic!("{other:?}"), + }, + Err(error) => panic!("{error:?}"), + } + assert!( + service.retained_export(token).unwrap().is_some(), + "the export stays in custody" + ); + assert!(service.export_released(token).is_none()); + assert!(!table.exists(), "the lazy helper removed the table entry"); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + #[test] fn canonical_mount_paths_and_unreadable_table_preserve_detach_safety() { let (send, receive) = mpsc::channel(); From 6f3cf7714b757befe60e107e856a80aa325edbac Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:27:44 +0000 Subject: [PATCH 11/14] fix(sandbox): force only a record the daemon answered Retained at Detach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review item 2 (blocking): `unmount_guard::forceable` admitted every completed, never-forced `Retained`/`Failed` record on every platform. That made two wrong records forceable: a failed or lost control exchange (code `Failed`, never answered by the daemon) where a later `force` that gets `Missing` is taken as unmounted and deletes the Workspace record; and on Linux FUSE any failed plain unmount, which reaches the daemon's Join-stall exception again although the plain unmount already made its one abort. - The daemon's `Retained` reply now travels with the custody's stage: `lifecycle::operation` maps it to code `Failed` (unchanged) with `detail.stage` and `detail.detached` (`daemon/values.rs`). - `unmount_guard::finish` persists that stage in the record as `stage` (`Null` when the failure never got a daemon answer), and `forceable` admits only `state: Retained`, `forced: false`, `code: Failed`, `stage: Detach` — an NFS export still held, where the one forced client unmount is what finishes it. Every other stage, a staged-less failure, an `Unmounting`/`Unknown` record and an already-forced record keep their answer. - Docs: api/sdk/workspace.md states the narrowed rule and what a `Failed` answer without a stage is. Tests: - `a_record_force_cannot_replace_keeps_its_answer` (unmount_record.rs): `Join`, `Revoke`, `Close`, `Abort` and a record without `stage` are not forceable and repeat their `Retained`/`Failed` answer. - `a_retained_answer_carries_its_stage_into_the_record` (unmount_record.rs): a `Retained(Detach)` cause becomes code `Failed` with `detail.stage: Detach`, and the written record keeps that stage while a `Join` one is not forceable. - record_codec.rs: the forceable fixture carries `stage: Detach`; a new case proves the same record without a stage is not forceable. Reverse checks (each run, then reverted): with the `stage` condition removed from `forceable`, `a_record_force_cannot_replace_keeps_its_answer` fails at the `Join` case (`stage: Join` is forceable) and record_codec fails at the staged-less case; with the `Retained` branch removed from `values::refusal`, `a_retained_answer_carries_its_stage_into_the_record` fails with `left: None, right: Some("Detach")`. Checks: `cargo +1.85.1 test --locked -p layerfs-sandbox --test unmount_record --test record_codec`: 6 passed and 5 passed, 0 failed. Durable: NOT_RUN - disabled by owner until explicit reauthorization. Production LOC: 163755 -> 163788 (delta +33); parent e710662d9. core 39787, engine 75087, control 42804 -> 42837, api 6077. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root . --- .../layerfs-sandbox/src/daemon/values.rs | 35 +++++- .../layerfs-sandbox/src/unmount_guard.rs | 28 ++++- .../layerfs-sandbox/tests/record_codec.rs | 17 ++- .../layerfs-sandbox/tests/unmount_record.rs | 114 +++++++++++++++++- docs/architecture/api/sdk/workspace.md | 16 +-- 5 files changed, 191 insertions(+), 19 deletions(-) diff --git a/control/crates/layerfs-sandbox/src/daemon/values.rs b/control/crates/layerfs-sandbox/src/daemon/values.rs index e0dad01add..389e912513 100644 --- a/control/crates/layerfs-sandbox/src/daemon/values.rs +++ b/control/crates/layerfs-sandbox/src/daemon/values.rs @@ -6,7 +6,9 @@ //! refusals with stable codes. use super::workspace::{CommitReply, MountedWorkspace, Unmounted}; use crate::{OperationCause, OperationFailure}; -use layerfs_bridge::control::{ForcedOutcome, SdkRefusal, SdkValue, WorkspaceStatus}; +use layerfs_bridge::control::{ + ForcedOutcome, SdkRefusal, SdkValue, TeardownStage, WorkspaceStatus, +}; use layerfs_bridge::sdk_values::{ commit_value, conflicts, conflicts_truncated, hex, lww, lww_truncated, }; @@ -24,6 +26,22 @@ pub fn refusal(failure: &OperationFailure) -> Option { message: refused.detail.clone(), detail: SdkValue::map([("phase", SdkValue::text(refused.phase.as_str()))]), }), + // A terminal step the daemon stopped after effects. Its stage is the + // one fact a later `force` is judged by, so it travels in the detail + // the machine writes into `workspace/.unmount`; a failure without + // such an answer (a send or connect failure, or a refusal before any + // effect) has no stage. + OperationCause::Retained(custody) => Some(SdkRefusal { + code: "Failed".to_owned(), + message: format!( + "teardown custody retained at {}; do not resend", + stage_text(custody.stage) + ), + detail: SdkValue::map([ + ("stage", SdkValue::text(stage_text(custody.stage))), + ("detached", SdkValue::Bool(custody.detached)), + ]), + }), // The request was sent but no reply arrived (the daemon reclaimed or // expired the worker, or it died mid-operation): its effect is // unknown, so the caller observes `Unavailable { outcome: unknown }` @@ -45,6 +63,21 @@ pub fn refusal(failure: &OperationFailure) -> Option { } } +/// A retained teardown stage as the machine's unmount record writes it. +fn stage_text(stage: TeardownStage) -> &'static str { + match stage { + TeardownStage::Detach => "Detach", + TeardownStage::Join => "Join", + TeardownStage::Owner => "Owner", + TeardownStage::Requests => "Requests", + TeardownStage::Lane => "Lane", + TeardownStage::Revoke => "Revoke", + TeardownStage::Close => "Close", + TeardownStage::Registry => "Registry", + TeardownStage::Abort => "Abort", + } +} + /// A mounted Workspace: its path and binding. pub fn mounted(workspace: &MountedWorkspace, branch: &str) -> SdkValue { let binding = &workspace.bound.binding; diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index 09dcdc91f0..6c7b8597b6 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -232,16 +232,25 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { Ok(recoverable) } pub fn forceable(value: &SdkValue) -> bool { - // A completed `Failed` record is the one shape a force may replace: the - // daemon's own guard is the real gate on whether force may proceed, and a - // refusal answers with the record it holds. A record that is still `Unmounting` (its caller ended without publishing), - // `Unknown` or already `forced` keeps its answer. A record written before - // this rule (no `forced` field) is answered as it stands and is not - // forceable. + // Only a record the daemon itself answered `Retained` at `Detach` may be + // replaced by `force`: it holds an export whose client unmount was not + // established, and the one forced client unmount is what finishes it. The + // daemon's own guard remains the real gate, and a refusal answers with + // the record it holds. A record that is still `Unmounting` (its caller + // ended without publishing), one the machine wrote after a send or + // connect failure or a refusal before any effect (code `Failed` without + // the daemon's `stage`), one kept at another stage, one already forced, + // and a record written before this field existed keep their answer. value.get_text("state") == Some("Retained") && value.get("forced") == Some(&SdkValue::Bool(false)) && value.get_text("code") == Some("Failed") + && value.get_text("stage") == Some(FORCEABLE_STAGE) } +/// The daemon's retained stage one forced client unmount can still finish: +/// an NFS export kept because its client unmount was not established +/// (`ExportKept::forceable` in the daemon). Any other stage keeps its answer. +const FORCEABLE_STAGE: &str = "Detach"; + /// Called under the Sandbox guard, after all before-effect checks. pub fn begin( state: &SandboxState, @@ -301,6 +310,12 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd error.detail.clone(), ), }; + // The daemon's own retained stage, when it answered one; `Null` + // for a failure that never got that answer (see [`forceable`]). + let stage = match outcome { + Ok(_) => SdkValue::Null, + Err(error) => error.detail.get("stage").cloned().unwrap_or(SdkValue::Null), + }; layerfs_store::project::discovery::save( &attempt.path, SdkValue::map([ @@ -308,6 +323,7 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd ("startedMs", SdkValue::Int(attempt.started)), ("path", SdkValue::text(path)), ("forced", SdkValue::Bool(attempt.forced)), + ("stage", stage), ("code", SdkValue::text(code)), ("message", SdkValue::text(message)), ("detail", detail), diff --git a/control/crates/layerfs-sandbox/tests/record_codec.rs b/control/crates/layerfs-sandbox/tests/record_codec.rs index e84c104c8d..fda87800ea 100644 --- a/control/crates/layerfs-sandbox/tests/record_codec.rs +++ b/control/crates/layerfs-sandbox/tests/record_codec.rs @@ -148,16 +148,29 @@ fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { .unwrap(); assert!(!layerfs_sandbox::unmount_guard::forceable(&r3)); - // 4. Completed normal unmount that returned Retained (forced=false, code=Failed) -> forceable! + // 4. Completed normal unmount the daemon answered Retained at Detach + // (forced=false, code=Failed, stage=Detach) -> forceable! write_rec( "04.unmount", - r#"{"state":"Retained","startedMs":1,"path":"/w/4","forced":false,"code":"Failed","message":"m","detail":null}"#, + r#"{"state":"Retained","startedMs":1,"path":"/w/4","forced":false,"stage":"Detach","code":"Failed","message":"m","detail":null}"#, ); let r4 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/4") .unwrap() .unwrap(); assert!(layerfs_sandbox::unmount_guard::forceable(&r4)); + // 5. The same completed `Retained` answer without the daemon's stage: + // the machine wrote it after a send/connect failure, so force must + // not replace it. + write_rec( + "05.unmount", + r#"{"state":"Retained","startedMs":1,"path":"/w/5","forced":false,"code":"Failed","message":"m","detail":null}"#, + ); + let r5 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/5") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r5)); + std::fs::remove_dir_all(dir).unwrap(); let _ = send.send(()); }); diff --git a/control/crates/layerfs-sandbox/tests/unmount_record.rs b/control/crates/layerfs-sandbox/tests/unmount_record.rs index 009d9d481e..8699a41707 100644 --- a/control/crates/layerfs-sandbox/tests/unmount_record.rs +++ b/control/crates/layerfs-sandbox/tests/unmount_record.rs @@ -1,7 +1,11 @@ //! The unmount guard's record file across a forced attempt (layerfs-project#136): //! an attempt that changed nothing must leave the earlier record as it was. -use layerfs_bridge::control::{SdkRefusal, SdkValue}; -use layerfs_sandbox::unmount_guard::{begin_at, cancel, finish, forceable}; +use layerfs_bridge::control::{ + Request, SdkRefusal, SdkValue, TeardownCustody, TeardownStage, UnmountControl, WorkspaceToken, +}; +use layerfs_history::WorkspaceId; +use layerfs_sandbox::unmount_guard::{begin_at, cancel, finish, forceable, repeat}; +use layerfs_sandbox::{lifecycle, OperationCause, OperationFailure}; use layerfs_store::project::discovery; use std::{ os::unix::fs::PermissionsExt, @@ -24,19 +28,46 @@ fn record_file(dir: &Path) -> PathBuf { dir.join("workspace").join("01.unmount") } -/// A completed record a force may replace. +/// A completed record a force may replace: the daemon answered `Retained` +/// with its export still held at `Detach`. fn completed() -> SdkValue { SdkValue::map([ ("state", SdkValue::text("Retained")), ("startedMs", SdkValue::Int(7)), ("path", SdkValue::text("/w")), ("forced", SdkValue::Bool(false)), + ("stage", SdkValue::text("Detach")), ("code", SdkValue::text("Failed")), ("message", SdkValue::text("detach not established")), ("detail", SdkValue::Null), ]) } +/// The same shape with another daemon stage, or with no stage at all (a send +/// or connect failure, or a refusal before any effect). +fn completed_at(stage: Option<&str>) -> SdkValue { + let mut fields = Vec::new(); + for (name, value) in [ + ("state", SdkValue::text("Retained")), + ("startedMs", SdkValue::Int(7)), + ("path", SdkValue::text("/w")), + ("forced", SdkValue::Bool(false)), + ] { + fields.push((name.to_owned(), value)); + } + if let Some(stage) = stage { + fields.push(("stage".to_owned(), SdkValue::text(stage))); + } + for (name, value) in [ + ("code", SdkValue::text("Failed")), + ("message", SdkValue::text("detach not established")), + ("detail", SdkValue::Null), + ] { + fields.push((name.to_owned(), value)); + } + SdkValue::Map(fields) +} + fn refused(code: &str) -> SdkRefusal { SdkRefusal { code: code.into(), @@ -108,3 +139,80 @@ fn a_first_attempt_that_does_not_run_leaves_no_record() { drop(attempt); std::fs::remove_dir_all(dir).unwrap(); } + +/// Only the daemon's own `Retained(Detach)` answer is replaceable. A stage +/// another step stopped at, or a failure that never got a stage, is answered +/// as it stands: the daemon would refuse it, and `force` must not overwrite +/// the earlier answer with an attempt of its own. +#[test] +fn a_record_force_cannot_replace_keeps_its_answer() { + for (label, stage) in [ + ("join", Some("Join")), + ("revoke", Some("Revoke")), + ("close", Some("Close")), + ("abort", Some("Abort")), + ("no-stage", None), + ] { + let value = completed_at(stage); + assert!(!forceable(&value), "{label}: {value:?}"); + let answer = repeat(&value); + let error = answer.expect_err("a retained record answers Retained"); + assert_eq!(error.code, "Failed"); + assert_eq!(error.detail.get_text("state"), Some("Retained")); + } +} + +/// The daemon's own answer carries the stage the record keeps: a `Retained` +/// reply becomes `Failed` with `detail.stage`, while a refusal or a lost +/// exchange has no stage at all. +#[test] +fn a_retained_answer_carries_its_stage_into_the_record() { + let token = WorkspaceToken { + workspace: WorkspaceId::from_authority([9; 32]).unwrap(), + namespace: 3, + }; + let failure = |stage: TeardownStage| { + Box::new(OperationFailure { + request: Request::Unmount(UnmountControl { + token, + discard: true, + }), + cause: OperationCause::Retained(Box::new(TeardownCustody { + token, + stage, + detached: false, + work: None, + detail: "client unmount failed".into(), + forced: None, + })), + }) + }; + let refusal = lifecycle::operation(failure(TeardownStage::Detach)); + assert_eq!(refusal.code, "Failed"); + assert_eq!(refusal.detail.get_text("stage"), Some("Detach")); + assert!(!refusal.detail.get_bool("detached")); + + // The same record shape the attempt publishes: the stage it keeps is the + // daemon's, so this one is forceable and a `Join` one is not. + for (stage, forceable_stage) in [("Detach", true), ("Join", false)] { + let dir = fixture(&format!("stage-{stage}")); + let file = record_file(&dir); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + finish(&attempt, "/w", &Err(refusal_at(stage))).unwrap(); + let kept = discovery::load(&file).unwrap(); + assert_eq!(kept.get_text("stage"), Some(stage)); + assert_eq!(forceable(&kept), forceable_stage, "{stage}: {kept:?}"); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); + } +} + +/// The refusal shape `lifecycle::operation` builds from a daemon `Retained` +/// answer at `stage`. +fn refusal_at(stage: &str) -> SdkRefusal { + SdkRefusal { + code: "Failed".into(), + message: "retained".into(), + detail: SdkValue::map([("stage", SdkValue::text(stage))]), + } +} diff --git a/docs/architecture/api/sdk/workspace.md b/docs/architecture/api/sdk/workspace.md index e7adfb8232..dadecb8000 100644 --- a/docs/architecture/api/sdk/workspace.md +++ b/docs/architecture/api/sdk/workspace.md @@ -218,13 +218,15 @@ either may add cleanup_error (daemon succeeded; a local record stayed) The machine writes `workspace/.unmount` before it calls the daemon and records the outcome after, so a repeated unmount answers what the first one -established without dialling again. The exception is a completed `Failed` -record that was not itself forced: `force` may replace it with a forced -attempt (the daemon's guard remains the real gate, and `Busy`, `Dirty`, or -an unsent call restores the earlier record); an in-progress, `Unknown`, -already-forced, or pre-`forced` record (no `forced` field) keeps its answer -and is not forceable. A Workspace the daemon no longer knows answers -`Unmounted` +established without dialling again. The one exception is a record the daemon +itself answered `Retained` at stage `Detach`: its export is still held and one +forced client unmount is what finishes it, so `force` may replace that record +with a forced attempt (the daemon's guard remains the real gate, and `Busy`, +`Dirty`, or an unsent call restores the earlier record). A `Failed` answer +without a stage — a send or connect failure, or a refusal before any effect — +and an in-progress, `Unknown`, already-forced, other-stage or pre-`forced` +record keeps its answer and is not forceable. A Workspace the daemon no longer +knows answers `Unmounted` ([`unmount_guard.rs`](../../../../control/crates/layerfs-sandbox/src/unmount_guard.rs)). | Error | When | From 3468655dea12fb50e892f8be4bd3aa92e1751552 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:30:30 +0000 Subject: [PATCH 12/14] fix(sandbox): version the unmount record and prove the old record still reads Review item 3: the unmount record gained a persisted field (`forced`) with no stated version and no compatibility evidence, and the PR body claimed pre-`forced` records were treated as unforced while the code answered them as not forceable. - The record now names its version: `v: 1`, written with every attempt and every published result. Version 0 (no `v`) is the format before #644: `state`, `startedMs`, `path`, `code`, `message`, `detail`. Version 1 adds `forced` and, on a result, `stage`. The module doc in `unmount_guard.rs` states the format and the rules. - Compatibility, both directions: - A version-0 record is read as before and answered as before; it is not forceable (it has no daemon `stage`), which is what the code did and what the docs now say. - A version-1 record keeps every version-0 field with the values a version-0 writer wrote; the added fields are ignored by an older reader (all reads are by field name), and the repeated answer is identical. - A record from a newer version (`v` above 1) is answered as it stands and is never forceable: this build does not take the only step that could destroy unknown custody. - `forceable` requires `v == 1`; the `Unmounting` record written by `begin_at` and the crashed-caller record `inspect` synthesizes carry `v`. - Docs: api/sdk/workspace.md names the record version and the version-0 field set. Tests (control/crates/layerfs-sandbox/tests/unmount_record.rs): - `a_version_zero_record_is_answered_and_never_forced`: a record without `v`/`forced`/`stage` is found by `find_in`, is not forceable, and repeats its own `Failed` message unchanged. - `a_current_record_answers_every_version_zero_field`: a written version-1 record answers `state`, `startedMs`, `path`, `code`, `message` and `detail` exactly as a version-0 reader reads them, and repeats the same answer when only those fields are kept. - `a_newer_record_is_answered_and_not_forced`: a `v: 2` record with a `Detach` stage is answered and not forceable. - record_codec.rs adds the version-0 and version-2 read cases; every raw fixture now names `v: 1` so each case tests only its own condition. Reverse checks (each run, then reverted): without the `v` write in `finish` and the `v == 1` condition, `a_current_record_answers_every_version_zero_field` fails with `left: None, right: Some(1)` and `a_newer_record_is_answered_and_not_forced` fails (a `v: 2` record becomes forceable); with `find_in` skipping records that have no `v`, `a_version_zero_record_is_answered_and_never_forced` fails at `find_in(...).expect("version-0 record")`. Checks: `cargo +1.85.1 test --locked -p layerfs-sandbox --test unmount_record --test record_codec`: 9 passed and 5 passed, 0 failed. Durable: NOT_RUN - disabled by owner until explicit reauthorization. Production LOC: 163788 -> 163793 (delta +5); parent 6f3cf7714. core 39787, engine 75087, control 42837 -> 42842, api 6077. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root . --- .../layerfs-sandbox/src/unmount_guard.rs | 22 +++++ .../layerfs-sandbox/tests/record_codec.rs | 30 +++++- .../layerfs-sandbox/tests/unmount_record.rs | 91 ++++++++++++++++++- docs/architecture/api/sdk/workspace.md | 8 +- 4 files changed, 143 insertions(+), 8 deletions(-) diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index 6c7b8597b6..5c26b354ec 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -3,6 +3,21 @@ //! admission and shared-record cleanup, never during the daemon's helper wait. //! Mount paths are unique within one daemon lifetime; Sandbox removal clears //! the terminal records before that directory can be reused. +//! +//! The record is JSON in the version its `v` field names. +//! +//! - Version 0 (no `v`) has `state`, `startedMs`, `path`, `code`, `message` +//! and `detail`, and nothing else. It predates any forced attempt. +//! - Version 1 ([`VERSION`]) adds `forced`, whether this attempt was started +//! with `force`, and — in a published result — `stage`, the daemon's own +//! retained teardown stage (`Null` when the failure never got a daemon +//! answer). Both are extra fields: a version-0 reader reads the version-0 +//! fields unchanged and ignores them, which is what +//! `tests/unmount_record.rs` checks. +//! - A record whose `v` is above [`VERSION`] is answered as it stands and is +//! never forceable: the only step that could destroy unknown custody is the +//! one this version does not take. Every reader keeps answering version-0 +//! records exactly as before. use crate::files as discovery; use crate::registry::SandboxState; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; @@ -14,6 +29,9 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +/// The record version this build writes and the newest one it forces. +const VERSION: i64 = 1; + fn millis() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -127,6 +145,7 @@ fn inspect(path: &std::path::Path) -> Result, SdkRefusal> { Some(_) => { let error = progress(&value, false); Ok(Some(SdkValue::map([ + ("v", value.get("v").cloned().unwrap_or_default()), ("state", SdkValue::text("Retained")), ( "startedMs", @@ -242,6 +261,7 @@ pub fn forceable(value: &SdkValue) -> bool { // the daemon's `stage`), one kept at another stage, one already forced, // and a record written before this field existed keep their answer. value.get_text("state") == Some("Retained") + && value.get_int("v") == Some(VERSION) && value.get("forced") == Some(&SdkValue::Bool(false)) && value.get_text("code") == Some("Failed") && value.get_text("stage") == Some(FORCEABLE_STAGE) @@ -276,6 +296,7 @@ pub fn begin_at(path_record: PathBuf, path: &str, forced: bool) -> Result Result<(), Sd layerfs_store::project::discovery::save( &attempt.path, SdkValue::map([ + ("v", SdkValue::Int(VERSION)), ("state", SdkValue::text(state)), ("startedMs", SdkValue::Int(attempt.started)), ("path", SdkValue::text(path)), diff --git a/control/crates/layerfs-sandbox/tests/record_codec.rs b/control/crates/layerfs-sandbox/tests/record_codec.rs index fda87800ea..95b6c884e1 100644 --- a/control/crates/layerfs-sandbox/tests/record_codec.rs +++ b/control/crates/layerfs-sandbox/tests/record_codec.rs @@ -118,7 +118,7 @@ fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { // 1. Crashed force unmount (state=Unmounting, forced=true, no active lock -> code=Unknown) write_rec( "01.unmount", - r#"{"state":"Unmounting","startedMs":1,"path":"/w/1","forced":true}"#, + r#"{"v":1,"state":"Unmounting","startedMs":1,"path":"/w/1","forced":true}"#, ); let r1 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/1") .unwrap() @@ -130,7 +130,7 @@ fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { // 2. Crashed normal unmount (state=Unmounting, forced=false, no active lock -> code=Unknown) write_rec( "02.unmount", - r#"{"state":"Unmounting","startedMs":1,"path":"/w/2","forced":false}"#, + r#"{"v":1,"state":"Unmounting","startedMs":1,"path":"/w/2","forced":false}"#, ); let r2 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/2") .unwrap() @@ -141,7 +141,7 @@ fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { // 3. Completed forced unmount that remained Retained (forced=true, code=Failed) write_rec( "03.unmount", - r#"{"state":"Retained","startedMs":1,"path":"/w/3","forced":true,"code":"Failed","message":"m","detail":null}"#, + r#"{"v":1,"state":"Retained","startedMs":1,"path":"/w/3","forced":true,"code":"Failed","message":"m","detail":null}"#, ); let r3 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/3") .unwrap() @@ -152,7 +152,7 @@ fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { // (forced=false, code=Failed, stage=Detach) -> forceable! write_rec( "04.unmount", - r#"{"state":"Retained","startedMs":1,"path":"/w/4","forced":false,"stage":"Detach","code":"Failed","message":"m","detail":null}"#, + r#"{"v":1,"state":"Retained","startedMs":1,"path":"/w/4","forced":false,"stage":"Detach","code":"Failed","message":"m","detail":null}"#, ); let r4 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/4") .unwrap() @@ -164,13 +164,33 @@ fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { // not replace it. write_rec( "05.unmount", - r#"{"state":"Retained","startedMs":1,"path":"/w/5","forced":false,"code":"Failed","message":"m","detail":null}"#, + r#"{"v":1,"state":"Retained","startedMs":1,"path":"/w/5","forced":false,"code":"Failed","message":"m","detail":null}"#, ); let r5 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/5") .unwrap() .unwrap(); assert!(!layerfs_sandbox::unmount_guard::forceable(&r5)); + // 6. A version-0 record (no `v`): answered, never forceable. + write_rec( + "06.unmount", + r#"{"state":"Retained","startedMs":1,"path":"/w/6","code":"Failed","message":"m","detail":null}"#, + ); + let r6 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/6") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r6)); + + // 7. A record from a newer version: answered, never forceable. + write_rec( + "07.unmount", + r#"{"v":2,"state":"Retained","startedMs":1,"path":"/w/7","forced":false,"stage":"Detach","code":"Failed","message":"m","detail":null}"#, + ); + let r7 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/7") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r7)); + std::fs::remove_dir_all(dir).unwrap(); let _ = send.send(()); }); diff --git a/control/crates/layerfs-sandbox/tests/unmount_record.rs b/control/crates/layerfs-sandbox/tests/unmount_record.rs index 8699a41707..47e42de631 100644 --- a/control/crates/layerfs-sandbox/tests/unmount_record.rs +++ b/control/crates/layerfs-sandbox/tests/unmount_record.rs @@ -4,7 +4,7 @@ use layerfs_bridge::control::{ Request, SdkRefusal, SdkValue, TeardownCustody, TeardownStage, UnmountControl, WorkspaceToken, }; use layerfs_history::WorkspaceId; -use layerfs_sandbox::unmount_guard::{begin_at, cancel, finish, forceable, repeat}; +use layerfs_sandbox::unmount_guard::{begin_at, cancel, find_in, finish, forceable, repeat}; use layerfs_sandbox::{lifecycle, OperationCause, OperationFailure}; use layerfs_store::project::discovery; use std::{ @@ -32,6 +32,7 @@ fn record_file(dir: &Path) -> PathBuf { /// with its export still held at `Detach`. fn completed() -> SdkValue { SdkValue::map([ + ("v", SdkValue::Int(1)), ("state", SdkValue::text("Retained")), ("startedMs", SdkValue::Int(7)), ("path", SdkValue::text("/w")), @@ -48,6 +49,7 @@ fn completed() -> SdkValue { fn completed_at(stage: Option<&str>) -> SdkValue { let mut fields = Vec::new(); for (name, value) in [ + ("v", SdkValue::Int(1)), ("state", SdkValue::text("Retained")), ("startedMs", SdkValue::Int(7)), ("path", SdkValue::text("/w")), @@ -216,3 +218,90 @@ fn refusal_at(stage: &str) -> SdkRefusal { detail: SdkValue::map([("stage", SdkValue::text(stage))]), } } + +/// A version-0 record (the base of #644): no `v`, no `forced`, no `stage`. +fn version_zero() -> SdkValue { + SdkValue::map([ + ("state", SdkValue::text("Retained")), + ("startedMs", SdkValue::Int(7)), + ("path", SdkValue::text("/w")), + ("code", SdkValue::text("Failed")), + ("message", SdkValue::text("client unmount: exit 1")), + ("detail", SdkValue::Null), + ]) +} + +/// A record written before the forced fields existed is still read and +/// answered exactly as it was, and is never forceable. +#[test] +fn a_version_zero_record_is_answered_and_never_forced() { + let dir = fixture("version-zero"); + let file = record_file(&dir); + discovery::save(&file, version_zero()).unwrap(); + let value = find_in(&dir, "/w").unwrap().expect("version-0 record"); + assert_eq!(value.get_text("state"), Some("Retained")); + assert_eq!(value.get_text("code"), Some("Failed")); + assert!(!forceable(&value), "a pre-forced record is not forceable"); + let error = repeat(&value).expect_err("a retained record answers Retained"); + assert_eq!(error.code, "Failed"); + assert_eq!( + error.message, "client unmount: exit 1", + "the version-0 message is answered unchanged" + ); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// A version-1 record answers every field the version-0 reader used, with the +/// values it wrote: the new fields are additive, so an older reader reads the +/// record it expects and ignores the rest. +#[test] +fn a_current_record_answers_every_version_zero_field() { + let dir = fixture("version-one"); + let file = record_file(&dir); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + finish(&attempt, "/w", &Err(refusal_at("Detach"))).unwrap(); + let value = discovery::load(&file).unwrap(); + assert_eq!(value.get_int("v"), Some(1)); + + // The version-0 field set, and only it, is what the older reader sees. + let older = SdkValue::Map( + ["state", "startedMs", "path", "code", "message", "detail"] + .into_iter() + .filter_map(|name| Some((name.to_owned(), value.get(name)?.clone()))) + .collect(), + ); + assert_eq!(older.get_text("state"), Some("Retained")); + assert!(older.get_int("startedMs").is_some()); + assert_eq!(older.get_text("path"), Some("/w")); + assert_eq!(older.get_text("code"), Some("Failed")); + assert_eq!(older.get_text("message"), Some("retained")); + assert!(older.get("detail").is_some()); + // The same answer either way: the newer fields never change the reading. + let current = repeat(&value).expect_err("retained"); + let before = repeat(&older).expect_err("retained"); + assert_eq!( + (current.code, current.message), + (before.code, before.message) + ); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// A record from a newer version is answered as it stands and is never +/// forced: this build cannot know what the newer fields mean. +#[test] +fn a_newer_record_is_answered_and_not_forced() { + let dir = fixture("version-two"); + let file = record_file(&dir); + let mut fields = match completed() { + SdkValue::Map(fields) => fields, + _ => unreachable!(), + }; + fields[0].1 = SdkValue::Int(2); + discovery::save(&file, SdkValue::Map(fields)).unwrap(); + let value = find_in(&dir, "/w").unwrap().expect("version-2 record"); + assert!(!forceable(&value), "a newer record is not forceable"); + let error = repeat(&value).expect_err("retained"); + assert_eq!(error.code, "Failed"); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/docs/architecture/api/sdk/workspace.md b/docs/architecture/api/sdk/workspace.md index dadecb8000..92185e0587 100644 --- a/docs/architecture/api/sdk/workspace.md +++ b/docs/architecture/api/sdk/workspace.md @@ -225,8 +225,12 @@ with a forced attempt (the daemon's guard remains the real gate, and `Busy`, `Dirty`, or an unsent call restores the earlier record). A `Failed` answer without a stage — a send or connect failure, or a refusal before any effect — and an in-progress, `Unknown`, already-forced, other-stage or pre-`forced` -record keeps its answer and is not forceable. A Workspace the daemon no longer -knows answers `Unmounted` +record keeps its answer and is not forceable. The record is versioned (`v`): +version 1 adds `forced` and the daemon's `stage` to version 0's `state`, +`startedMs`, `path`, `code`, `message` and `detail`. A version-0 record has +neither field and is read and answered as before; a record from a newer +version is answered as it stands and is never forceable. A Workspace the daemon +no longer knows answers `Unmounted` ([`unmount_guard.rs`](../../../../control/crates/layerfs-sandbox/src/unmount_guard.rs)). | Error | When | From f148f2070c7136d3c1d142c5c117eefe5151ea3d Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:37:27 +0000 Subject: [PATCH 13/14] fix(daemon,bridge): state the retained reaper's real bound and keep the child on a failed wait Review item 4 (smaller items): - Retained children (`process/retained.rs`): the queue is bounded in threads and recorded exits, not in time. The docs now say so: one background thread polls every 20 ms for as long as custody lasts, because a deadline would drop the only handle that can reap the child. A failed `try_wait` no longer drops the child unreaped: it stays queued so its exit is still observed, and after 25 consecutive failed waits (~0.5 s) the entry is given up with one recorded `exit unobservable after ...` line instead of polling a wait that cannot work. Export helper and control-calls wording updated to match. - The extra `/sbin/mount` per native Attach is stated in the report: it runs on macOS only (Linux reads `/proc/self/mountinfo`), once after the mount helper returns, plus once per failed/timed-out unmount-helper settlement and per ForceUnmount; Attach adds exactly one process run. - Custody an interrupted Attach left (`ExportKept` with `ready: None`, its export `tracked: false`) is documented as never forceable and never unmounted: no receipt was published, so there is no mount to force off by name; it stays retained until the daemon exits and the next start's `reclaim` forces the leftover host mount off. Pre-existing behaviour, now written down on `ExportKept::forceable`. - `export.rs` was over the 999-line ceiling (1002) after the review fixes. The kernel-half custody types (`Exported`, `ExportLeaving`, `ExportKept`, their `forceable`/`custody` and `Debug` impls) move to the new `control/export_custody.rs`; `export.rs` re-exports them under their old paths and is 938 lines. No logic change; `mod.rs` declares the module. - Docs: control-calls.md helper rule names the real bound and links `export_custody.rs`; nfs.md links `Exported::tracked` there. Checks: `cargo +1.85.1 clippy --locked -p layerfs-daemon -p layerfs-bridge -p layerfs-nfs -p layerfs-sandbox --all-targets -- -D warnings` clean; `cargo +1.85.1 test --locked -p layerfs-nfs --test helper_deadline` 8 passed; `-p layerfs-bridge --test process_identity` 2 passed. No new test here: the reaper's exit path is covered by `wait_reaped`/`assert_not_zombie` in helper_deadline, and the `try_wait` error path cannot be provoked without fault injection; the error branch is exercised only by inspection. Durable: NOT_RUN - disabled by owner until explicit reauthorization. Production LOC: 163793 -> 163827 (delta +34); parent 3468655de. core 39787, engine 75092 -> 75126, control 42842, api 6077. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root . --- .../engine/daemon/control-calls.md | 4 +- docs/architecture/engine/kernel-routes/nfs.md | 2 +- .../layerfs-bridge/src/process/retained.rs | 56 +++++++++-- .../layerfs-daemon/src/control/export.rs | 88 +++--------------- .../src/control/export_custody.rs | 92 +++++++++++++++++++ .../src/control/export_helper.rs | 7 +- .../crates/layerfs-daemon/src/control/mod.rs | 1 + 7 files changed, 159 insertions(+), 91 deletions(-) create mode 100644 engine/crates/layerfs-daemon/src/control/export_custody.rs diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index 0588a4e625..d0451ebeda 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -199,9 +199,9 @@ reply ForceUnmounted { token, ForcedOutcome { facts, work, cleanup } } | --- | --- | | The abort write uses the connection's own control, bound from the daemon's `fusectl` tree at Attach; without it Force is refused before any effect | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`application/filesystem.rs`](../../../../engine/crates/layerfs-daemon/src/application/filesystem.rs) | | A normal unmount stalled at `Join` with its abort unused is forced from its retained session; its plain detach counts as the one detach | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`serving.rs`](../../../../engine/crates/layerfs-daemon/src/control/serving.rs) | -| An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts. On Linux that helper detaches lazily, so its failure never counts as a detach even when the table no longer lists the mount | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) `ExportKept::forceable` | +| An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts. On Linux that helper detaches lazily, so its failure never counts as a detach even when the table no longer lists the mount | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) `ExportKept::forceable` | | Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), when the table still lists it or cannot be read (`detached_export`), or when the helper detaches lazily (`-l`/`--lazy` in its own words: the mount leaves the table before the kernel gives its superblock back, so the table proves nothing); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | -| A helper past `KILL` and `REAP_WAIT` enters the bounded retained queue, whose one background thread reaps and records its exit (shared with the macOS driver's `umount -f`). For tests only, `ExportServingConfig` adds `mount_table`, `reap_wait` and `helper_wait`, and `ExportServing::reaped_helpers` reads observed exits | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | +| A helper past `KILL` and `REAP_WAIT` enters the retained queue, whose one background thread reaps and records its exit (shared with the macOS driver's `umount -f`). Its bound is one thread and 64 recorded exits, not time: a child whose exit never comes keeps that thread polling for as long as the custody lasts, because a deadline would drop the only handle that can reap it. For tests only, `ExportServingConfig` adds `mount_table`, `reap_wait` and `helper_wait`, and `ExportServing::reaped_helpers` reads observed exits | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | `relinquish_unknown` permits local disposal of stopped unknown Commit custody; the receipt keeps it `Unknown` | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | | On the NFS route the force helper is the abort | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Outside Linux without NFS the guard runs and the call is refused (`kernel serving requires Linux`) | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 4eccecd3d5..d3abe33939 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -198,7 +198,7 @@ reaches the Workspace or a Commit; everything is dropped with the export. | Mount-table and helper rule | Source | | --- | --- | | Every step above is one attempt; the mount table is the only witness for a failed or timed-out unmount helper. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` in [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) | | The mount point itself is never resolved (a lookup through a wedged mount can block or fail); `listed` compares the configured path and the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | | A failed or timed-out unmount helper completes teardown only when Attach saw it listed and the table no longer does; a still-listed mount, an unreadable table, or a helper that detaches lazily (`-l`) keeps `Retained(Detach)` and the export serving. | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | A helper past `KILL` and `REAP_WAIT` is reaped in the background by the shared retained queue, with nothing waiting for it or signalling twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | diff --git a/engine/crates/layerfs-bridge/src/process/retained.rs b/engine/crates/layerfs-bridge/src/process/retained.rs index e9f063d759..e451579d23 100644 --- a/engine/crates/layerfs-bridge/src/process/retained.rs +++ b/engine/crates/layerfs-bridge/src/process/retained.rs @@ -2,8 +2,13 @@ //! helper with a deadline: the daemon's NFS mount and unmount helpers and the //! macOS driver's `umount -f` (layerfs-project#136). A child that survived its //! KILL and the post-KILL grace window is never waited on synchronously and is -//! never signalled twice. One bounded background thread polls it instead and -//! records the exit it observes. +//! never signalled twice. One background thread polls it instead and records +//! the exit it observes. +//! +//! The bound is one reaper thread and [`RECORDED`] observed exits, not time: a +//! child whose exit never arrives keeps the queue non-empty and the one thread +//! polling every [`PAUSE`] for as long as its custody lasts. That is the point +//! of keeping it; a deadline would drop the only handle that can reap it. use std::{ fmt, process::Child, @@ -18,6 +23,10 @@ use std::{ const PAUSE: Duration = Duration::from_millis(20); /// Observed exits kept for diagnostics; the oldest go first. const RECORDED: usize = 64; +/// Consecutive failed waits after which a child is given up on (~0.5 s at +/// [`PAUSE`]). A wait that keeps failing cannot reap the child, so holding its +/// handle only polls a broken one; the failure is recorded instead. +const ERROR_TRIES: u8 = 25; /// Children a caller could not observe exiting, and the exits seen since. /// `retain`, `reap`, `exits` and `record` are the whole surface: nothing here @@ -27,11 +36,19 @@ pub struct Retained(Arc); #[derive(Debug, Default)] struct Inner { - children: Mutex>, + children: Mutex>, exits: Mutex>, reaping: AtomicBool, } +/// One retained child and, when its last wait failed, that failure. +#[derive(Debug)] +struct Kept { + child: Child, + error: Option, + failures: u8, +} + impl Retained { /// Keeps `child` and makes sure one background reaper is polling for it. pub fn retain(&self, child: Child) { @@ -40,7 +57,11 @@ impl Retained { .children .lock() .unwrap_or_else(|error| error.into_inner()) - .push(child); + .push(Kept { + child, + error: None, + failures: 0, + }); if self.0.reaping.swap(true, Ordering::AcqRel) { return; } @@ -97,14 +118,33 @@ impl Inner { .lock() .unwrap_or_else(|error| error.into_inner()); let mut exits = self.exits.lock().unwrap_or_else(|error| error.into_inner()); - children.retain_mut(|child| match child.try_wait() { + children.retain_mut(|kept| match kept.child.try_wait() { Ok(Some(status)) => { - exits.push((child.id(), status.to_string())); + exits.push((kept.child.id(), status.to_string())); false } - Ok(None) => true, + Ok(None) => { + kept.error = None; + kept.failures = 0; + true + } + // A failed wait is not an exit. Keep the child so its exit is + // still observed, record the failure once (changed message or the + // final give-up), and stop polling it after `ERROR_TRIES`. Err(error) => { - exits.push((child.id(), format!("wait error: {error}"))); + let message = error.to_string(); + if kept.error.as_deref() != Some(message.as_str()) { + kept.error = Some(message.clone()); + kept.failures = 0; + } + kept.failures = kept.failures.saturating_add(1); + if kept.failures < ERROR_TRIES { + return true; + } + exits.push(( + kept.child.id(), + format!("exit unobservable after {ERROR_TRIES} failed waits: {message}"), + )); false } }); diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 2c55498aca..06fcad3ad7 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -26,10 +26,12 @@ //! (`export_helper::listed`, `Helper::lazy`). //! - ForceUnmount runs the forced helper once, then the same teardown. It is //! also how an export kept at Detach for a mount still listed is finished. -//! A helper that outlives its KILL goes to the bounded retained queue. +//! A helper that outlives its KILL goes to the retained queue, whose one +//! background thread polls it (no call waits for it). +pub(super) use super::export_custody::{ExportKept, ExportLeaving, Exported}; use super::{ export_helper::{listed, mounted_under, Helpers, Ran}, - kernel::{bounded, Kernel}, + kernel::Kernel, registry::{bound_mut, idle}, Failure, Service, Success, }; @@ -37,12 +39,11 @@ use crate::store::BoundWorkspace; use layerfs_bridge::control::{ AbortDisposition, Activity, ControlCode, DetachDisposition, ForceUnmounted, ForcedCleanup, ForcedFacts, ForcedOutcome, MountPhase, MountReceipt, MountStatus, MountWork, ReadyMount, - Reply, TeardownCustody, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, + Reply, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, }; use layerfs_nfs::{ExportConfig, Released, Server}; use layerfs_overlay::{CleanupState, KernelMount}; use std::{ - fmt, net::{IpAddr, Ipv4Addr, TcpListener}, path::PathBuf, sync::Arc, @@ -154,8 +155,8 @@ pub struct ExportServingConfig { pub helper_wait: std::time::Duration, /// The post-KILL window in which the helper's own exit is still observed /// synchronously; past it the child goes to the retained queue. Zero - /// skips the window, which bounded tests use to reach the retained path - /// at once. Production value [`REAP_WAIT`]. + /// skips the window, so a test reaches the retained path at once with a + /// bounded wait. Production value [`REAP_WAIT`]. pub reap_wait: std::time::Duration, /// Test hook, `None` in production: a file read line by line in place of /// the host's mount table, so a test states which mounts exist without @@ -193,8 +194,9 @@ impl ExportServing { pub fn config(&self) -> &ExportServingConfig { &self.config } - /// Test and diagnostic accessor: the helper exits the bounded reaper has - /// observed since the daemon started (newest last, at most 64 kept). + /// Test and diagnostic accessor: the helper exits the queue's one + /// background thread has observed since the daemon started (newest last, + /// at most 64 kept). /// Production code reports a retained child inside its own refusal text /// instead; nothing in the daemon reads this. pub fn reaped_helpers(&self) -> Vec<(u32, String)> { @@ -236,73 +238,6 @@ impl ExportServing { .collect() } } -/// A serving export and the mount the host client holds of it. -pub(super) struct Exported { - pub server: Arc, - pub mount: KernelMount, - pub ready: ReadyMount, - pub tracked: bool, -} -/// A terminal operation owns the export; status still observes it. -pub(super) struct ExportLeaving { - pub ready: ReadyMount, - pub phase: MountPhase, - pub work: MountWork, -} -/// Export custody a terminal or attach step could not finish disposing: -/// kept exactly, never retried or settled by a later observation. -pub(super) struct ExportKept { - pub ready: Option, - pub stage: TeardownStage, - pub detail: String, - pub work: Option, - pub exported: Option>, - pub evidence: Box, - pub forced: Option, -} -impl ExportKept { - pub(super) fn forceable(&self) -> bool { - self.exported.is_some() - && self.stage == TeardownStage::Detach - && self.forced.is_none() - && self.ready.is_some() - } - pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { - TeardownCustody { - token, - stage: self.stage, - detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), - work: self.work, - detail: bounded(&self.detail), - forced: self.forced.clone(), - } - } -} -impl fmt::Debug for ExportKept { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let evidence: &dyn fmt::Debug = self - .exported - .as_ref() - .map_or(&*self.evidence as &dyn fmt::Debug, |e| e); - f.debug_struct("ExportKept") - .field("ready", &self.ready) - .field("stage", &self.stage) - .field("detail", &self.detail) - .field("work", &self.work) - .field("evidence", &evidence) - .field("forced", &self.forced) - .finish() - } -} -impl fmt::Debug for Exported { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Exported") - .field("address", &self.server.local_addr()) - .field("mount", &self.mount) - .field("held", &self.server.held()) - .finish() - } -} /// The maintained counters of a serving export. pub(super) fn work(server: &Server) -> MountWork { let held = server.held(); @@ -707,7 +642,8 @@ impl Service { format!("client reported the mount busy; no terminal effect: {detail}"), )); } - Ran::Failed(_) | Ran::Interrupted(_) if self.detached_export(&exported, helper.as_ref()) => {} + Ran::Failed(_) | Ran::Interrupted(_) + if self.detached_export(&exported, helper.as_ref()) => {} Ran::Failed(detail) | Ran::Interrupted(detail) => { return Err(self.keep_export( token, diff --git a/engine/crates/layerfs-daemon/src/control/export_custody.rs b/engine/crates/layerfs-daemon/src/control/export_custody.rs new file mode 100644 index 0000000000..3457c34702 --- /dev/null +++ b/engine/crates/layerfs-daemon/src/control/export_custody.rs @@ -0,0 +1,92 @@ +//! The NFS route's kernel-half custody: a serving export, one a terminal +//! operation is tearing down, and one kept exactly as a terminal or attach +//! step left it. +//! +//! [`export`](super::export) owns the mount entry and the `Service` steps; the +//! types here are what an entry holds while an export lives, so status, +//! teardown and Force name one ownership model instead of two. +use super::kernel::bounded; +use layerfs_bridge::control::{ + ForcedFacts, MountPhase, MountWork, ReadyMount, TeardownCustody, TeardownStage, WorkspaceToken, +}; +use layerfs_nfs::Server; +use layerfs_overlay::KernelMount; +use std::{fmt, sync::Arc}; + +/// A serving export and the mount the host client holds of it. +pub(super) struct Exported { + pub server: Arc, + pub mount: KernelMount, + pub ready: ReadyMount, + pub tracked: bool, +} +/// A terminal operation owns the export; status still observes it. +pub(super) struct ExportLeaving { + pub ready: ReadyMount, + pub phase: MountPhase, + pub work: MountWork, +} +/// Export custody a terminal or attach step could not finish disposing: +/// kept exactly, never retried or settled by a later observation. +pub(super) struct ExportKept { + pub ready: Option, + pub stage: TeardownStage, + pub detail: String, + pub work: Option, + pub exported: Option>, + pub evidence: Box, + pub forced: Option, +} +impl ExportKept { + /// Whether one forced client unmount may still finish this custody: the + /// export and its receipt are held, the first boundary not established is + /// `Detach`, and nothing was forced yet. + /// + /// Custody a failed Attach left (`ready: None`, its inner export + /// `tracked: false`, because the mount helper was interrupted) is never + /// forceable: no receipt was published, so there is no mount to force off + /// by name, and a failed force would leave the same custody. It stays + /// retained until the daemon exits; the next start's `reclaim` forces the + /// leftover host mount off. This predates #644. + pub(super) fn forceable(&self) -> bool { + self.exported.is_some() + && self.stage == TeardownStage::Detach + && self.forced.is_none() + && self.ready.is_some() + } + pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { + TeardownCustody { + token, + stage: self.stage, + detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), + work: self.work, + detail: bounded(&self.detail), + forced: self.forced.clone(), + } + } +} +impl fmt::Debug for ExportKept { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let evidence: &dyn fmt::Debug = self + .exported + .as_ref() + .map_or(&*self.evidence as &dyn fmt::Debug, |e| e); + f.debug_struct("ExportKept") + .field("ready", &self.ready) + .field("stage", &self.stage) + .field("detail", &self.detail) + .field("work", &self.work) + .field("evidence", &evidence) + .field("forced", &self.forced) + .finish() + } +} +impl fmt::Debug for Exported { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Exported") + .field("address", &self.server.local_addr()) + .field("mount", &self.mount) + .field("held", &self.server.held()) + .finish() + } +} diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index 7219e048db..6e6e750e24 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -31,9 +31,7 @@ impl Helper { pub fn lazy(&self) -> bool { self.0.iter().any(|word| { word == "--lazy" - || (word.starts_with('-') - && !word.starts_with("--") - && word.contains('l')) + || (word.starts_with('-') && !word.starts_with("--") && word.contains('l')) }) } fn command(&self, port: u16, directory: &str) -> Option { @@ -56,7 +54,8 @@ pub(super) enum Ran { } /// Retained children are never waited on synchronously or signalled twice. -/// A bounded background reaper reaps their exits and records their status. +/// The retained queue's one background thread reaps their exits and records +/// their status; its bound is that thread, not time. #[derive(Debug, Default)] pub(super) struct Helpers { retained: Retained, diff --git a/engine/crates/layerfs-daemon/src/control/mod.rs b/engine/crates/layerfs-daemon/src/control/mod.rs index 1fb83c0879..b57c756ddc 100644 --- a/engine/crates/layerfs-daemon/src/control/mod.rs +++ b/engine/crates/layerfs-daemon/src/control/mod.rs @@ -7,6 +7,7 @@ mod detach; mod diagnostic; mod dirty; mod export; +mod export_custody; mod export_helper; mod failure; mod force; From 89116b1774e8033a7d25c6b1ae3997567adba24f Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:59:07 +0000 Subject: [PATCH 14/14] docs(control): a Retained terminal step carries its stage, not a phase The sandbox daemon-channel table still folded a stopped terminal step (`Reply::Retained`) into the transport-failure row, which says `Failed` carries a phase and `attempted`. Since the unmount record keeps the custody's stage, a `Retained` reply is now its own row: code `Failed` with `detail.stage` and `detail.detached`, the stage the machine persists, and only `Detach` forceable. Found by the doc grep for stale wording after the review fixes; no code or behaviour change. Checks: docs only; `git diff` reviewed. Durable: NOT_RUN - disabled by owner until explicit reauthorization. Production LOC: 163827 -> 163827 (delta 0); parent f148f2070. core 39787, engine 75126, control 42842, api 6077. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 tools/production_loc.py --root . --- docs/architecture/control/sandboxes/daemon-channel-and-exec.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/architecture/control/sandboxes/daemon-channel-and-exec.md b/docs/architecture/control/sandboxes/daemon-channel-and-exec.md index 0fe217ae48..b27a7d5eb0 100644 --- a/docs/architecture/control/sandboxes/daemon-channel-and-exec.md +++ b/docs/architecture/control/sandboxes/daemon-channel-and-exec.md @@ -63,7 +63,8 @@ one before it. | --- | --- | --- | --- | | Daemon refusal (`Reply::Refused`) | its control code (`Busy`, `HeadMoved`, `Missing`, `Capacity`, `Invalid`, `Failed`, `Unknown`), `detail.phase` | yes, except `Unknown` | `refusal` in [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | Sent, connection ended before the reply (phase `Receive`) | `Unavailable`, `outcome: "unknown"`, `retryable: false` | no | same | -| Other transport or protocol failure, a stopped terminal step (`Retained`), a reply of the wrong kind | `Failed`, with phase and `attempted` in the message | no once `attempted` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation`, [`daemon/failure.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/failure.rs) | +| Other transport or protocol failure, or a reply of the wrong kind | `Failed`, with phase and `attempted` in the message | no once `attempted` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation`, [`daemon/failure.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/failure.rs) | +| A stopped terminal step (`Retained`) | `Failed`, with the custody's `stage` (`Detach`, `Join`, ...) and `detached` in `detail`; the machine persists the stage in `workspace/.unmount`, and only `Detach` is forceable | no | `refusal` in [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), [`unmount_guard.rs`](../../../../control/crates/layerfs-sandbox/src/unmount_guard.rs) | | SDK method over one control record, before sending | `TooLarge` | yes: not sent | [`daemon/call.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/call.rs) | | Open refused: capacity notice, closed before admission, reset | `Capacity`, `Unknown` or `Unavailable`, each `retryable: true`; nothing attempted | yes | `daemon/pool.rs` | | No free connection in 60 s | `Busy`, "nothing was attempted" | yes | `daemon/pool.rs` |