From 05aa187f5a579d6b7e6c440743dfddfa4fd09466 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 19:31:00 +0800 Subject: [PATCH] fix(daemon,sandbox): follow-ups to the native NFS forced teardown (#644) Review follow-ups after merging #644 (layerfs-project#136): - A failed or timed-out unmount helper proves the client mount gone only once its exit has been observed (`Ran::ended`). A plain Linux `umount` drops the mountinfo entry before the superblock's shutdown, which flushes the client's writes through the export; a killed helper still running keeps `Retained(Detach)`, and Force finishes it. - The sandbox undoes an unmount attempt's record when the exchange failed before sending was attempted, as for a call that never connected, so a forceable `Retained/Detach` record is not replaced by an unforceable one. - The macOS mount table is read by one bounded listing, moved from the macOS driver to `layerfs_bridge::process::mount_table` (500 ms, then killed and read as an unreadable table); the daemon's Attach tracking, failed-unmount checks and reclaim now use it instead of an unbounded `/sbin/mount`. - `Helpers` no longer derives `Default` (a zero helper deadline) and keeps its waits private. Tests: new `a_helper_still_running_never_proves_a_vanished_mount_gone` (fails under the previous rule); `timed_out_helper_when_mount_already_gone_ completes_teardown` now observes the killed helper's exit. No tests deleted. Production LOC: 172,703 -> 172,717 (delta +14) core 39,795 -> 39,795 (0) engine 77,204 -> 77,277 (+73) layerfs-bridge +79 (mount_table moved in), layerfs-daemon -6 control 49,287 -> 49,228 (-59) layerfs-sandbox +6, layerfs-sandbox-backend -65 (mount_table moved out) api 6,417 -> 6,417 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: python3 tools/production_loc.py --json on the parent (39a25138ea) and the staged tree, exported with git archive. Co-Authored-By: Claude Opus 5.5 --- .../src/local/os/macos/darwin.rs | 83 +--------------- .../layerfs-sandbox/src/workspace/serve.rs | 12 ++- docs/architecture/api/sdk/workspace.md | 5 +- .../engine/daemon/control-calls.md | 4 +- docs/architecture/engine/kernel-routes/nfs.md | 4 +- .../crates/layerfs-bridge/src/process/mod.rs | 7 +- .../layerfs-bridge/src/process/mount_table.rs | 93 ++++++++++++++++++ .../layerfs-daemon/src/control/export.rs | 30 +++--- .../src/control/export_helper.rs | 56 ++++++----- .../layerfs-nfs/tests/helper_deadline.rs | 98 ++++++++++++++++++- 10 files changed, 263 insertions(+), 129 deletions(-) create mode 100644 engine/crates/layerfs-bridge/src/process/mount_table.rs diff --git a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs index 83e211ddfa..aa381f503a 100644 --- a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs +++ b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs @@ -20,14 +20,8 @@ const UMOUNT: &str = "/sbin/umount"; const UNMOUNT_WAIT: Duration = Duration::from_secs(60); const REAP_WAIT: Duration = Duration::from_secs(5); const OUTPUT: usize = 2048; -/// How long an inspection read waits for `mount` to print the table. -const TABLE_WAIT: Duration = Duration::from_millis(500); -/// How long it then waits for a `mount` it killed to end. -const TABLE_REAP: Duration = Duration::from_millis(500); -/// The most of the table that is read. -const TABLE: usize = 1 << 20; -/// Children a `umount` KILL or `mount` timeout did not release. Their exits +/// Children a `umount` KILL did not release. Their exits /// are reaped by the shared retained queue /// ([`layerfs_bridge::process::Retained`]), never by a second wait here. static RETAINED: LazyLock = LazyLock::new(Retained::default); @@ -69,77 +63,10 @@ pub(crate) fn mount_line(line: &str) -> Option<(PathBuf, String)> { Some((PathBuf::from(point), fs.to_owned())) } -/// What `mount` prints, for an inspection read: one run with a fixed path, -/// no environment but the C locale and no input, given [`TABLE_WAIT`] to -/// end. A run that takes longer is killed and reaped, and is an error, as -/// is one that fails; the caller then reads without a table. `mount` -/// lists the kernel's table as it is (`getfsstat` with `MNT_NOWAIT`) and -/// asks no mounted server, so it is not expected to wait; the deadline is -/// for a host where it does. -pub(crate) fn mount_table() -> io::Result { - RETAINED.reap(); - let mut child = Command::new(MOUNT) - .env_clear() - .env("LC_ALL", "C") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn()?; - // a pipe that cannot be read without waiting is not read - let mut pipe = child.stdout.take().and_then(|pipe| { - fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)) - .ok() - .map(|_| pipe) - }); - let mut text = Vec::new(); - let mut read = |text: &mut Vec| { - let mut bytes = [0; 4096]; - while let Some(Ok(count @ 1..)) = pipe.as_mut().map(|pipe| pipe.read(&mut bytes)) { - text.extend_from_slice(&bytes[..count.min(TABLE.saturating_sub(text.len()))]); - } - }; - let deadline = Instant::now() + TABLE_WAIT; - let ended = loop { - read(&mut text); - match child.try_wait() { - Ok(Some(status)) => break Ok(status), - Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(2)), - Ok(None) => { - break Err(io::Error::new( - io::ErrorKind::TimedOut, - "mount took too long", - )) - } - Err(error) => break Err(error), - } - }; - match ended { - Ok(status) if status.success() => { - read(&mut text); - Ok(String::from_utf8_lossy(&text).into_owned()) - } - Ok(status) => Err(io::Error::other(format!("mount listing failed: {status}"))), - Err(error) => { - let pid = child.id(); - let _ = child.kill(); - let deadline = Instant::now() + TABLE_REAP; - let reaped = loop { - match child.try_wait() { - Ok(Some(status)) => break Some(status), - Ok(None) if Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(2)); - } - Ok(None) | Err(_) => break None, - } - }; - match reaped { - Some(status) => RETAINED.record(pid, status), - None => RETAINED.retain(child), - } - Err(error) - } - } -} +/// What `mount` prints, for an inspection read: the shared bounded listing +/// ([`layerfs_bridge::process::mount_table`]). An error means the caller +/// reads without a table. +pub(crate) use layerfs_bridge::process::mount_table; /// Forces the mount at `point` off once; a point no longer mounted is done. pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { diff --git a/control/crates/layerfs-sandbox/src/workspace/serve.rs b/control/crates/layerfs-sandbox/src/workspace/serve.rs index 7cb83fb3cb..292007b4bd 100644 --- a/control/crates/layerfs-sandbox/src/workspace/serve.rs +++ b/control/crates/layerfs-sandbox/src/workspace/serve.rs @@ -281,17 +281,25 @@ pub fn unmount(host: &Host, path: Option<&str>, discard: bool, force: bool) -> O let mut control = crate::daemon::trace::control(&state.controls)?; sent = true; let mut api = workspace(&mut control); + // A request whose sending was never attempted reached no daemon: + // the attempt's record is undone like a call that never connected. + let mut answered = |failure: Box| { + if matches!(&failure.cause, OperationCause::Exchange(exchange) if !exchange.attempted) { + sent = false; + } + operation(failure) + }; // Forced: one abort write ends the connection, also after a normal // unmount stalled at Join; uncommitted changes are dropped. if force { - match api.unmount(token, true).map_err(operation)? { + match api.unmount(token, true).map_err(&mut answered)? { Some(outcome) => Ok(values::force_unmounted(&outcome)), None => values::unmounted(&Unmounted::Closed), } } else { match api.unmount_checked(token, discard) { Ok(answer) => values::unmounted(&answer), - Err(error) => Err(operation(error)), + Err(error) => Err(answered(error)), } } })(); diff --git a/docs/architecture/api/sdk/workspace.md b/docs/architecture/api/sdk/workspace.md index fb2ac5f038..7fa659b717 100644 --- a/docs/architecture/api/sdk/workspace.md +++ b/docs/architecture/api/sdk/workspace.md @@ -225,8 +225,9 @@ established without dialling again. The one exception is a record the daemon itself answered `Retained` at stage `Detach`: its export is still held and one forced client unmount is what finishes it, so `force` may replace that record with a forced attempt (the daemon's guard remains the real gate, and `Busy`, -`Dirty`, or an unsent call restores the earlier record). A `Failed` answer -without a stage — a send or connect failure, or a refusal before any effect — +`Dirty`, or a call whose request was never sent restores the earlier record: +a connect failure, or an exchange that failed before sending was attempted). +A `Failed` answer without a stage — a send that was attempted, or a refusal — and an in-progress, `Unknown`, already-forced, other-stage or pre-`forced` record keeps its answer and is not forceable. The record is versioned (`v`): version 1 adds `forced` and the daemon's `stage` to version 0's `state`, diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index d0451ebeda..68c9f8a1ed 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -98,7 +98,7 @@ differs. `Exported`, `ExportLeaving` and `ExportKept` mirror `Ready`, | --- | --- | --- | | Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount, and the host's mount table is read once to record whether the mount is listed. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Commit | Flush first (flush helper, or the export volume's own flush: `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere; a failed flush refuses the Commit as `Failed` at `commit:flush`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | -| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close. A failed or timed-out helper is settled by the mount table: a mount listed at Attach and gone now completes teardown, while a still-listed one or an unreadable table keeps `Retained(Detach)` | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close. A failed or timed-out helper is settled by the mount table: a mount listed at Attach and gone now completes teardown once the helper has ended, while a still-listed one, an unreadable table or a helper still running keeps `Retained(Detach)` | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | ForceUnmount | Force helper once, then the same teardown; also finishes an export kept at `Detach` whose client mount stayed | same | | Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported; a child still unreaped past `REAP_WAIT` 5 s enters the retained queue and is polled by its one background reaper | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | @@ -200,7 +200,7 @@ reply ForceUnmounted { token, ForcedOutcome { facts, work, cleanup } } | The abort write uses the connection's own control, bound from the daemon's `fusectl` tree at Attach; without it Force is refused before any effect | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`application/filesystem.rs`](../../../../engine/crates/layerfs-daemon/src/application/filesystem.rs) | | A normal unmount stalled at `Join` with its abort unused is forced from its retained session; its plain detach counts as the one detach | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`serving.rs`](../../../../engine/crates/layerfs-daemon/src/control/serving.rs) | | An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts. On Linux that helper detaches lazily, so its failure never counts as a detach even when the table no longer lists the mount | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) `ExportKept::forceable` | -| Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), when the table still lists it or cannot be read (`detached_export`), or when the helper detaches lazily (`-l`/`--lazy` in its own words: the mount leaves the table before the kernel gives its superblock back, so the table proves nothing); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), when the table still lists it or cannot be read (`detached_export`), when the killed helper's exit was not observed (`Ran::ended`: a plain Linux `umount` leaves the table before the superblock's shutdown flushes through the export), or when the helper detaches lazily (`-l`/`--lazy` in its own words: the mount leaves the table before the kernel gives its superblock back, so the table proves nothing); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | | A helper past `KILL` and `REAP_WAIT` enters the retained queue, whose one background thread reaps and records its exit (shared with the macOS driver's `umount -f`). Its bound is one thread and 64 recorded exits, not time: a child whose exit never comes keeps that thread polling for as long as the custody lasts, because a deadline would drop the only handle that can reap it. For tests only, `ExportServingConfig` adds `mount_table`, `reap_wait` and `helper_wait`, and `ExportServing::reaped_helpers` reads observed exits | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | `relinquish_unknown` permits local disposal of stopped unknown Commit custody; the receipt keeps it `Unknown` | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | | On the NFS route the force helper is the abort | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 568fc1e120..ca370bb37d 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -346,9 +346,9 @@ or container, where the environment is passed differently. | Mount-table and helper rule | Source | | --- | --- | | Every step above is one attempt; the mount table is the only witness for a failed or timed-out unmount helper. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` in [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) | +| Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. The macOS run is bounded (500 ms, then killed and read as an unreadable table) by the listing the driver's inspection reads share, [`process::mount_table`](../../../../engine/crates/layerfs-bridge/src/process/mount_table.rs). | `Exported::tracked` in [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) | | The mount point itself is never resolved (a lookup through a wedged mount can block or fail); `listed` compares the configured path and the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | -| A failed or timed-out unmount helper completes teardown only when Attach saw it listed and the table no longer does; a still-listed mount, an unreadable table, or a helper that detaches lazily (`-l`) keeps `Retained(Detach)` and the export serving. | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| A failed or timed-out unmount helper completes teardown only when Attach saw it listed, the helper has ended, and the table no longer lists it; a still-listed mount, an unreadable table, a helper that detaches lazily (`-l`), or a killed helper whose exit was not observed keeps `Retained(Detach)` and the export serving. A plain Linux `umount` drops the table entry before the superblock's shutdown, which flushes the client's writes through this export, so only an ended helper lets the table witness the end (`Ran::ended`). | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | A helper past `KILL` and `REAP_WAIT` is reaped in the background by the shared retained queue, with nothing waiting for it or signalling twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | MOUNT procedure | Answer | diff --git a/engine/crates/layerfs-bridge/src/process/mod.rs b/engine/crates/layerfs-bridge/src/process/mod.rs index 4390fefb02..bccf102e6e 100644 --- a/engine/crates/layerfs-bridge/src/process/mod.rs +++ b/engine/crates/layerfs-bridge/src/process/mod.rs @@ -3,7 +3,8 @@ //! from `/proc` on Linux (`linux.rs`) and `proc_pidinfo`/`sysctl` on macOS //! (`darwin.rs`, this crate's one audited `unsafe` file), and the retained //! child custody both the daemon's NFS helpers and the macOS driver's -//! `umount -f` use after a KILL (`retained.rs`). It sits below the +//! `umount -f` use after a KILL (`retained.rs`), and the macOS mount table +//! under a deadline (`mount_table.rs`). It sits below the //! control layer so the home and the drivers share it without an edge //! between them. #[cfg(target_os = "macos")] @@ -11,10 +12,14 @@ mod darwin; mod identity; #[cfg(target_os = "linux")] mod linux; +#[cfg(target_os = "macos")] +mod mount_table; mod retained; #[cfg(target_os = "macos")] pub use darwin::{boot, processes, stat, sync_volume_wait}; pub use identity::{pid, LocalProcess, Stat}; #[cfg(target_os = "linux")] pub use linux::{boot, processes, stat}; +#[cfg(target_os = "macos")] +pub use mount_table::mount_table; pub use retained::Retained; diff --git a/engine/crates/layerfs-bridge/src/process/mount_table.rs b/engine/crates/layerfs-bridge/src/process/mount_table.rs new file mode 100644 index 0000000000..4f2168e94e --- /dev/null +++ b/engine/crates/layerfs-bridge/src/process/mount_table.rs @@ -0,0 +1,93 @@ +//! The macOS mount table under a deadline: one `/sbin/mount` run with a fixed +//! path, the C locale and no input. The daemon's NFS route (Attach tracking, +//! failed-unmount checks, reclaim) and the macOS driver's inspection reads +//! share it (layerfs-project#178 security rule 8, layerfs-project#136). +//! `mount` lists the kernel's table as it is (`getfsstat` with `MNT_NOWAIT`) +//! and asks no mounted server, so it is not expected to wait; the deadline is +//! for a host where it does. +use super::Retained; +use nix::fcntl::{fcntl, FcntlArg, OFlag}; +use std::{ + io::{self, Read}, + process::{Command, Stdio}, + sync::LazyLock, + time::{Duration, Instant}, +}; + +const MOUNT: &str = "/sbin/mount"; +/// How long one listing waits for `mount` to print the table. +const TABLE_WAIT: Duration = Duration::from_millis(500); +/// How long it then waits for a `mount` it killed to end. +const TABLE_REAP: Duration = Duration::from_millis(500); +/// The most of the table that is read. +const TABLE: usize = 1 << 20; +const PAUSE: Duration = Duration::from_millis(2); + +/// Listings a KILL did not release; the shared retained queue reaps them. +static RETAINED: LazyLock = LazyLock::new(Retained::default); + +/// What `mount` prints. A run that takes longer than [`TABLE_WAIT`] is +/// killed and is an error, as is one that fails; a killed run whose exit is +/// not seen within [`TABLE_REAP`] goes to the retained queue. +pub fn mount_table() -> io::Result { + RETAINED.reap(); + let mut child = Command::new(MOUNT) + .env_clear() + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + // a pipe that cannot be read without waiting is not read + let mut pipe = child.stdout.take().and_then(|pipe| { + fcntl(&pipe, FcntlArg::F_SETFL(OFlag::O_NONBLOCK)) + .ok() + .map(|_| pipe) + }); + let mut text = Vec::new(); + let mut read = |text: &mut Vec| { + let mut bytes = [0; 4096]; + while let Some(Ok(count @ 1..)) = pipe.as_mut().map(|pipe| pipe.read(&mut bytes)) { + text.extend_from_slice(&bytes[..count.min(TABLE.saturating_sub(text.len()))]); + } + }; + let deadline = Instant::now() + TABLE_WAIT; + let ended = loop { + read(&mut text); + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) => { + break Err(io::Error::new( + io::ErrorKind::TimedOut, + "mount took too long", + )) + } + Err(error) => break Err(error), + } + }; + match ended { + Ok(status) if status.success() => { + read(&mut text); + Ok(String::from_utf8_lossy(&text).into_owned()) + } + Ok(status) => Err(io::Error::other(format!("mount listing failed: {status}"))), + Err(error) => { + let pid = child.id(); + let _ = child.kill(); + let deadline = Instant::now() + TABLE_REAP; + let reaped = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) | Err(_) => break None, + } + }; + match reaped { + Some(status) => RETAINED.record(pid, status), + None => RETAINED.retain(child), + } + Err(error) + } + } +} diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index ea503da692..05086f5f96 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -247,7 +247,7 @@ impl ExportServing { let text = point.display().to_string(); match self.helpers.run(&self.config.mounter.force, 0, &text) { Ran::Done => format!("forced off {text}"), - Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail, _) => { format!("left {text}: {detail}") } } @@ -477,7 +477,7 @@ impl Service { let port = server.local_addr().port(); match serving.helpers.run(&config.mounter.mount, port, &text) { Ran::Done => {} - Ran::Interrupted(detail) => { + Ran::Interrupted(detail, _) => { // The mount helper may have reached the kernel before its // deadline. Keep the export alive and do not revoke/delete // through a possibly mounted directory. @@ -567,7 +567,7 @@ impl Service { .run(helper, server.local_addr().port(), &directory) { Ran::Done => {} - Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail, _) => { return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)) } } @@ -658,9 +658,9 @@ impl Service { format!("client reported the mount busy; no terminal effect: {detail}"), )); } - Ran::Failed(_) | Ran::Interrupted(_) - if self.detached_export(&exported, helper.as_ref()) => {} - Ran::Failed(detail) | Ran::Interrupted(detail) => { + Ran::Failed(_) | Ran::Interrupted(..) + if self.detached_export(&exported, helper.as_ref(), &ran) => {} + Ran::Failed(detail) | Ran::Interrupted(detail, _) => { return Err(self.keep_export( token, TeardownStage::Detach, @@ -675,11 +675,15 @@ impl Service { } /// Whether a failed or timed-out helper proves the client mount is gone: /// Attach saw the mount listed (`tracked`), the helper cannot detach - /// lazily, and the host's table does not list it now. A lazy detach - /// (`umount -l`, the Linux force helper) removes the table entry while the - /// kernel still holds the superblock, so it is never proof. - fn detached_export(&self, exported: &Exported, helper: Option<&Helper>) -> bool { + /// lazily, the helper has ended, and the host's table does not list it + /// now. A lazy detach (`umount -l`, the Linux force helper) removes the + /// table entry while the kernel still holds the superblock, so it is + /// never proof. Neither is a plain Linux `umount` still running: it drops + /// the table entry before the superblock's shutdown, which flushes the + /// client's writes through this export ([`Ran::ended`]). + fn detached_export(&self, exported: &Exported, helper: Option<&Helper>, ran: &Ran) -> bool { !helper.is_some_and(Helper::lazy) + && ran.ended() && exported.tracked && (self.export.as_ref()) .is_some_and(|s| matches!(s.is_mounted(&exported.ready.directory), Ok(false))) @@ -853,9 +857,9 @@ impl Service { }; match ran { Ran::Done => {} - Ran::Busy(_) | Ran::Failed(_) | Ran::Interrupted(_) - if self.detached_export(&exported, helper.as_ref()) => {} - Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { + Ran::Busy(_) | Ran::Failed(_) | Ran::Interrupted(..) + if self.detached_export(&exported, helper.as_ref(), &ran) => {} + Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail, _) => { facts.detach = DetachDisposition::Failed; return Err(self.keep_export( token, diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index 6e6e750e24..501c1fbf58 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -4,7 +4,7 @@ use nix::fcntl::{fcntl, FcntlArg, OFlag}; use std::{ io::{self, Read}, path::{Path, PathBuf}, - process::{Child, Command, Stdio}, + process::{Command, Stdio}, sync::Arc, time::{Duration, Instant}, }; @@ -50,24 +50,36 @@ pub(super) enum Ran { Busy(String), Failed(String), /// Timeout or observation failure after spawn: effects are not known. - Interrupted(String), + /// The flag says whether the killed helper's exit was then observed: a + /// helper still running may still be inside the kernel's unmount. + Interrupted(String, bool), +} +impl Ran { + /// Whether the helper is known to have ended, so no unmount of its own + /// is still in progress. On Linux a plain `umount` takes the mount out of + /// the table before the superblock shuts down (that shutdown flushes + /// through the export), so only an ended helper lets the table witness + /// the mount's end. + pub fn ended(&self) -> bool { + !matches!(self, Ran::Interrupted(_, false)) + } } /// Retained children are never waited on synchronously or signalled twice. /// The retained queue's one background thread reaps their exits and records /// their status; its bound is that thread, not time. -#[derive(Debug, Default)] +#[derive(Debug)] pub(super) struct Helpers { retained: Retained, - pub helper_wait: Duration, - pub reap_wait: Duration, + helper_wait: Duration, + reap_wait: Duration, } impl Helpers { pub fn new(helper_wait: Duration, reap_wait: Duration) -> Arc { Arc::new(Self { + retained: Retained::default(), helper_wait, reap_wait, - ..Self::default() }) } pub fn reap(&self) -> bool { @@ -76,9 +88,6 @@ impl Helpers { pub fn reaped(&self) -> Vec<(u32, String)> { self.retained.exits() } - fn retain(self: &Arc, child: Child) { - self.retained.retain(child) - } pub fn run(self: &Arc, helper: &Helper, port: u16, directory: &str) -> Ran { self.reap(); let Some(mut command) = helper.command(port, directory) else { @@ -152,11 +161,14 @@ impl Helpers { format!("exit reaped ({status})") } None => { - self.retain(child); + self.retained.retain(child); format!("exit not observed after {:?}; helper may remain in kernel wait; child custody retained", self.reap_wait) } }; - Err(format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}")) + Err(( + format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}"), + reaped.is_some(), + )) } }; drain(&mut pipes, &mut output); @@ -171,7 +183,7 @@ impl Helpers { Ran::Failed(detail) } } - Err(detail) => Ran::Interrupted(format!("{detail}: {}", output.trim())), + Err((detail, ended)) => Ran::Interrupted(format!("{detail}: {}", output.trim()), ended), } } } @@ -206,22 +218,12 @@ pub(super) fn mounted_under(home: &Path, table: Option<&Path>) -> io::Result panic!("{other:?}"), }; - // Helper timed out, but mount table confirms the mount is gone: teardown completes. + // Helper timed out and ended, and the mount table confirms the mount + // is gone: teardown completes. let reply = service .execute_control(&Request::Unmount(UnmountControl { token, @@ -424,6 +427,97 @@ fn timed_out_helper_when_mount_already_gone_completes_teardown() { worker.join().unwrap(); } +#[test] +fn a_helper_still_running_never_proves_a_vanished_mount_gone() { + // A plain Linux `umount` drops the table entry before the superblock's + // shutdown, which flushes the client's writes through the export. While + // the killed helper's exit is not observed, the vanished entry is no + // proof: the export is kept at Detach, and Force finishes it. + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("helper-running"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("rm -f {}; sleep 5", table.display()), + ]), + force: Helper::words("true").unwrap(), + flush: None, + }, + ); + config.helper_wait = Duration::from_millis(100); + config.reap_wait = Duration::ZERO; + config.mount_table = Some(table); + let service = rig.service(Arc::new(ExportServing::start(config))); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([39; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + service.execute_control(&Request::Attach(token)).unwrap(); + match service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err() + { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + } + other => panic!("expected Retained, got {other:?}"), + } + match service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => { + assert_eq!(done.outcome.facts.detach, DetachDisposition::Detached); + } + other => panic!("expected ForceUnmounted, got {other:?}"), + } + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + #[test] fn a_lazily_detached_force_helper_never_completes_teardown() { // `umount -f -l` detaches lazily: the mount leaves the host's table while