diff --git a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs index e402a697f5..83e211ddfa 100644 --- a/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs +++ b/control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs @@ -5,12 +5,13 @@ //! deadline (layerfs-project#178 security rule 8). //! The process facts (`proc_pidinfo`, `sysctl`) are `layerfs_bridge::process` //! (layerfs-project#112), below the drivers and the home. +use layerfs_bridge::process::Retained; use nix::fcntl::{fcntl, FcntlArg, OFlag}; use std::{ io::{self, Read}, path::{Path, PathBuf}, - process::{Child, Command, Stdio}, - sync::Mutex, + process::{Command, Stdio}, + sync::LazyLock, time::{Duration, Instant}, }; @@ -26,22 +27,10 @@ const TABLE_REAP: Duration = Duration::from_millis(500); /// The most of the table that is read. const TABLE: usize = 1 << 20; -/// Children that were killed and whose exit was not seen in time: kept, so -/// a later call can reap them. -static RETAINED: Mutex> = Mutex::new(Vec::new()); - -/// Reaps the retained children that have ended since. -fn reap_retained() { - let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner()); - held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); -} - -fn retain(child: Child) { - RETAINED - .lock() - .unwrap_or_else(|error| error.into_inner()) - .push(child); -} +/// Children a `umount` KILL or `mount` timeout did not release. Their exits +/// are reaped by the shared retained queue +/// ([`layerfs_bridge::process::Retained`]), never by a second wait here. +static RETAINED: LazyLock = LazyLock::new(Retained::default); /// One tool's standard output, or its failure with what it said. fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { @@ -88,7 +77,7 @@ pub(crate) fn mount_line(line: &str) -> Option<(PathBuf, String)> { /// asks no mounted server, so it is not expected to wait; the deadline is /// for a host where it does. pub(crate) fn mount_table() -> io::Result { - reap_retained(); + RETAINED.reap(); let mut child = Command::new(MOUNT) .env_clear() .env("LC_ALL", "C") @@ -131,14 +120,21 @@ pub(crate) fn mount_table() -> io::Result { } Ok(status) => Err(io::Error::other(format!("mount listing failed: {status}"))), Err(error) => { + let pid = child.id(); let _ = child.kill(); let deadline = Instant::now() + TABLE_REAP; - while !matches!(child.try_wait(), Ok(Some(_))) { - if Instant::now() >= deadline { - retain(child); - break; + let reaped = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(2)); + } + Ok(None) | Err(_) => break None, } - std::thread::sleep(Duration::from_millis(2)); + }; + match reaped { + Some(status) => RETAINED.record(pid, status), + None => RETAINED.retain(child), } Err(error) } @@ -147,7 +143,7 @@ pub(crate) fn mount_table() -> io::Result { /// Forces the mount at `point` off once; a point no longer mounted is done. pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { - reap_retained(); + RETAINED.reap(); let mut child = Command::new(UMOUNT) .arg("-f") .arg(point) @@ -191,18 +187,23 @@ pub(crate) fn force_unmount(point: &Path) -> io::Result<()> { let reaped = loop { unmount_output(&mut stderr, &mut output); match child.try_wait() { - Ok(Some(_)) => break true, + Ok(Some(status)) => break Some(status), Ok(None) if Instant::now() < deadline => { std::thread::sleep(Duration::from_millis(5)); } - Ok(None) | Err(_) => break false, + Ok(None) | Err(_) => break None, } }; - let disposition = if reaped { - "exit reaped" - } else { - retain(child); - "exit not observed after 5s; child custody retained (possible kernel wait)" + let disposition = match reaped { + Some(status) => { + RETAINED.record(pid, status); + format!("exit reaped ({status})") + } + None => { + RETAINED.retain(child); + "exit not observed after 5s; child custody retained (possible kernel wait)" + .into() + } }; return Err(io::Error::new( error.kind(), diff --git a/control/crates/layerfs-sandbox/src/daemon/values.rs b/control/crates/layerfs-sandbox/src/daemon/values.rs index e0dad01add..389e912513 100644 --- a/control/crates/layerfs-sandbox/src/daemon/values.rs +++ b/control/crates/layerfs-sandbox/src/daemon/values.rs @@ -6,7 +6,9 @@ //! refusals with stable codes. use super::workspace::{CommitReply, MountedWorkspace, Unmounted}; use crate::{OperationCause, OperationFailure}; -use layerfs_bridge::control::{ForcedOutcome, SdkRefusal, SdkValue, WorkspaceStatus}; +use layerfs_bridge::control::{ + ForcedOutcome, SdkRefusal, SdkValue, TeardownStage, WorkspaceStatus, +}; use layerfs_bridge::sdk_values::{ commit_value, conflicts, conflicts_truncated, hex, lww, lww_truncated, }; @@ -24,6 +26,22 @@ pub fn refusal(failure: &OperationFailure) -> Option { message: refused.detail.clone(), detail: SdkValue::map([("phase", SdkValue::text(refused.phase.as_str()))]), }), + // A terminal step the daemon stopped after effects. Its stage is the + // one fact a later `force` is judged by, so it travels in the detail + // the machine writes into `workspace/.unmount`; a failure without + // such an answer (a send or connect failure, or a refusal before any + // effect) has no stage. + OperationCause::Retained(custody) => Some(SdkRefusal { + code: "Failed".to_owned(), + message: format!( + "teardown custody retained at {}; do not resend", + stage_text(custody.stage) + ), + detail: SdkValue::map([ + ("stage", SdkValue::text(stage_text(custody.stage))), + ("detached", SdkValue::Bool(custody.detached)), + ]), + }), // The request was sent but no reply arrived (the daemon reclaimed or // expired the worker, or it died mid-operation): its effect is // unknown, so the caller observes `Unavailable { outcome: unknown }` @@ -45,6 +63,21 @@ pub fn refusal(failure: &OperationFailure) -> Option { } } +/// A retained teardown stage as the machine's unmount record writes it. +fn stage_text(stage: TeardownStage) -> &'static str { + match stage { + TeardownStage::Detach => "Detach", + TeardownStage::Join => "Join", + TeardownStage::Owner => "Owner", + TeardownStage::Requests => "Requests", + TeardownStage::Lane => "Lane", + TeardownStage::Revoke => "Revoke", + TeardownStage::Close => "Close", + TeardownStage::Registry => "Registry", + TeardownStage::Abort => "Abort", + } +} + /// A mounted Workspace: its path and binding. pub fn mounted(workspace: &MountedWorkspace, branch: &str) -> SdkValue { let binding = &workspace.bound.binding; diff --git a/control/crates/layerfs-sandbox/src/registry.rs b/control/crates/layerfs-sandbox/src/registry.rs index 679a261eab..2143093eee 100644 --- a/control/crates/layerfs-sandbox/src/registry.rs +++ b/control/crates/layerfs-sandbox/src/registry.rs @@ -131,11 +131,18 @@ impl SandboxState { /// The token of the Workspace mounted at `path`. pub fn token(&self, path: &str) -> Result { + crate::unmount_guard::check(self, path)?; + self.mounted(path, |mount| mount.workspace.bound.token) + } + + /// The token of the Workspace at `path` after unmount admission checked it. + pub fn unmount_token(&self, path: &str) -> Result { self.mounted(path, |mount| mount.workspace.bound.token) } /// The kernel directory of the Workspace mounted at `path`. pub fn directory(&self, path: &str) -> Result { + crate::unmount_guard::check(self, path)?; self.mounted(path, |mount| mount.workspace.ready.directory.clone()) } @@ -145,7 +152,6 @@ impl SandboxState { /// once, for whatever is mounted there now; the full refresh replaces /// the table, so a stale entry goes with it. fn mounted(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result { - crate::unmount_guard::check(self, path)?; let known = lock(&self.mounts)? .get(path) .map(|mount| (mount.workspace.bound.token, pick(mount))); diff --git a/control/crates/layerfs-sandbox/src/unmount_guard.rs b/control/crates/layerfs-sandbox/src/unmount_guard.rs index 66f25d91d3..5c26b354ec 100644 --- a/control/crates/layerfs-sandbox/src/unmount_guard.rs +++ b/control/crates/layerfs-sandbox/src/unmount_guard.rs @@ -3,6 +3,21 @@ //! admission and shared-record cleanup, never during the daemon's helper wait. //! Mount paths are unique within one daemon lifetime; Sandbox removal clears //! the terminal records before that directory can be reused. +//! +//! The record is JSON in the version its `v` field names. +//! +//! - Version 0 (no `v`) has `state`, `startedMs`, `path`, `code`, `message` +//! and `detail`, and nothing else. It predates any forced attempt. +//! - Version 1 ([`VERSION`]) adds `forced`, whether this attempt was started +//! with `force`, and — in a published result — `stage`, the daemon's own +//! retained teardown stage (`Null` when the failure never got a daemon +//! answer). Both are extra fields: a version-0 reader reads the version-0 +//! fields unchanged and ignores them, which is what +//! `tests/unmount_record.rs` checks. +//! - A record whose `v` is above [`VERSION`] is answered as it stands and is +//! never forceable: the only step that could destroy unknown custody is the +//! one this version does not take. Every reader keeps answering version-0 +//! records exactly as before. use crate::files as discovery; use crate::registry::SandboxState; use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken}; @@ -14,6 +29,9 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +/// The record version this build writes and the newest one it forces. +const VERSION: i64 = 1; + fn millis() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -51,6 +69,10 @@ pub struct Attempt { _lease: Lease, path: PathBuf, started: i64, + forced: bool, + /// The record this attempt replaced, put back when the attempt has no + /// effect of its own (see [`restore`]). + prior: Option, } fn result(value: &SdkValue) -> Outcome { if value.get_text("state") == Some("Unmounted") { @@ -123,12 +145,20 @@ fn inspect(path: &std::path::Path) -> Result, SdkRefusal> { Some(_) => { let error = progress(&value, false); Ok(Some(SdkValue::map([ + ("v", value.get("v").cloned().unwrap_or_default()), ("state", SdkValue::text("Retained")), ( "startedMs", value.get("startedMs").cloned().unwrap_or_default(), ), ("path", value.get("path").cloned().unwrap_or_default()), + ( + "forced", + value + .get("forced") + .cloned() + .unwrap_or(SdkValue::Bool(false)), + ), ("code", SdkValue::text(error.code)), ("message", SdkValue::text(error.message)), ("detail", error.detail), @@ -183,7 +213,7 @@ pub fn check_in(directory: &std::path::Path, path: &str) -> Result<(), SdkRefusa Ok(()) } /// [`find`] by scanning Sandbox folder `directory`'s unmount records. -fn find_in(directory: &std::path::Path, path: &str) -> Result, SdkRefusal> { +pub fn find_in(directory: &std::path::Path, path: &str) -> Result, SdkRefusal> { for entry in unmounts_in(directory)? { let entry = entry.map_err(failed)?; if entry.path().extension().is_some_and(|ext| ext == "unmount") { @@ -220,37 +250,77 @@ pub fn check_all(state: &SandboxState) -> Result, SdkRefusal> { } Ok(recoverable) } +pub fn forceable(value: &SdkValue) -> bool { + // Only a record the daemon itself answered `Retained` at `Detach` may be + // replaced by `force`: it holds an export whose client unmount was not + // established, and the one forced client unmount is what finishes it. The + // daemon's own guard remains the real gate, and a refusal answers with + // the record it holds. A record that is still `Unmounting` (its caller + // ended without publishing), one the machine wrote after a send or + // connect failure or a refusal before any effect (code `Failed` without + // the daemon's `stage`), one kept at another stage, one already forced, + // and a record written before this field existed keep their answer. + value.get_text("state") == Some("Retained") + && value.get_int("v") == Some(VERSION) + && value.get("forced") == Some(&SdkValue::Bool(false)) + && value.get_text("code") == Some("Failed") + && value.get_text("stage") == Some(FORCEABLE_STAGE) +} +/// The daemon's retained stage one forced client unmount can still finish: +/// an NFS export kept because its client unmount was not established +/// (`ExportKept::forceable` in the daemon). Any other stage keeps its answer. +const FORCEABLE_STAGE: &str = "Detach"; + /// Called under the Sandbox guard, after all before-effect checks. pub fn begin( state: &SandboxState, token: WorkspaceToken, path: &str, + forced: bool, ) -> Result { - let path_record = record(state, token); + begin_at(record(state, token), path, forced) +} +/// [`begin`] at the record file `path_record` (a Workspace's +/// `workspace/.unmount`), holding its lease and replacing the record it +/// found with this attempt's `Unmounting` record. +pub fn begin_at(path_record: PathBuf, path: &str, forced: bool) -> Result { let file = lease(&path_record.with_extension("unmount-lock"), true)? .ok_or_else(|| refused("Busy", "Workspace is Unmounting"))?; let attempt = Attempt { _lease: file, + prior: observed(&path_record)?, path: path_record, started: millis(), + forced, }; layerfs_store::project::discovery::save( &attempt.path, SdkValue::map([ + ("v", SdkValue::Int(VERSION)), ("state", SdkValue::text("Unmounting")), ("startedMs", SdkValue::Int(attempt.started)), ("path", SdkValue::text(path)), + ("forced", SdkValue::Bool(attempt.forced)), ]), )?; Ok(attempt) } +/// Undo this attempt's own record: what it replaced comes back, or the file +/// goes when there was none. A forced attempt admitted over an earlier record +/// that answers `Busy` or `Dirty` changed nothing, so the earlier record — +/// possibly one that says "do not resend" — must not be lost with it. A call +/// that sent nothing is the same case. +fn restore(attempt: &Attempt) -> Result<(), SdkRefusal> { + match &attempt.prior { + Some(prior) => layerfs_store::project::discovery::save(&attempt.path, prior.clone()), + None => std::fs::remove_file(&attempt.path).map_err(failed), + } +} /// Publish while the Workspace lease prevents another terminal request. /// No Sandbox guard is needed to replace this attempt's own result file. pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), SdkRefusal> { match outcome { - Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => { - std::fs::remove_file(&attempt.path).map_err(failed)?; - } + Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => restore(attempt)?, _ => { let (state, code, message, detail) = match outcome { Ok(_) => ("Unmounted", "", "", SdkValue::Null), @@ -261,12 +331,21 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd error.detail.clone(), ), }; + // The daemon's own retained stage, when it answered one; `Null` + // for a failure that never got that answer (see [`forceable`]). + let stage = match outcome { + Ok(_) => SdkValue::Null, + Err(error) => error.detail.get("stage").cloned().unwrap_or(SdkValue::Null), + }; layerfs_store::project::discovery::save( &attempt.path, SdkValue::map([ + ("v", SdkValue::Int(VERSION)), ("state", SdkValue::text(state)), ("startedMs", SdkValue::Int(attempt.started)), ("path", SdkValue::text(path)), + ("forced", SdkValue::Bool(attempt.forced)), + ("stage", stage), ("code", SdkValue::text(code)), ("message", SdkValue::text(message)), ("detail", detail), @@ -277,7 +356,7 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd Ok(()) } pub fn cancel(attempt: &Attempt) -> Result<(), SdkRefusal> { - std::fs::remove_file(&attempt.path).map_err(failed) + restore(attempt) } pub fn cleanup_failure(mut outcome: Outcome, error: SdkRefusal) -> Outcome { // Do not replace a known daemon success with a cleanup Busy/Failed. diff --git a/control/crates/layerfs-sandbox/src/workspace/serve.rs b/control/crates/layerfs-sandbox/src/workspace/serve.rs index 457cfb1356..7cb83fb3cb 100644 --- a/control/crates/layerfs-sandbox/src/workspace/serve.rs +++ b/control/crates/layerfs-sandbox/src/workspace/serve.rs @@ -250,15 +250,19 @@ pub fn unmount(host: &Host, path: Option<&str>, discard: bool, force: bool) -> O let state = sandbox(host)?; let path = required(path, "path")?; if let Some(value) = crate::unmount_guard::find(&state, path)? { - return crate::unmount_guard::repeat(&value); + if !(force && crate::unmount_guard::forceable(&value)) { + return crate::unmount_guard::repeat(&value); + } } let guard = layerfs_store::project::discovery::guard(&state.directory)?; // Another process may have registered while this call waited for the // short admission section. Check again before sending anything. if let Some(value) = crate::unmount_guard::find(&state, path)? { - return crate::unmount_guard::repeat(&value); + if !(force && crate::unmount_guard::forceable(&value)) { + return crate::unmount_guard::repeat(&value); + } } - let token = state.token(path)?; + let token = state.unmount_token(path)?; // its running commands (D-10): `Busy`, or with `force` cancelled first let wsid = layerfs_bridge::sdk_values::hex(&token.workspace.to_bytes()); let unreached = &mut crate::exec::cancel::unreached; @@ -268,7 +272,7 @@ pub fn unmount(host: &Host, path: Option<&str>, discard: bool, force: bool) -> O // ordinary 60 s control wait running (#540). crate::files::refresh_within(&state, UNMOUNT_STATUS_WAIT)?; // the daemon's kernel probe answers Busy for a process still holding it - let attempt = crate::unmount_guard::begin(&state, token, path)?; + let attempt = crate::unmount_guard::begin(&state, token, path, force)?; drop(guard); // Only this Workspace's lease remains held. Other Workspaces and // processes can take Sandbox ownership while the daemon helper waits. diff --git a/control/crates/layerfs-sandbox/tests/record_codec.rs b/control/crates/layerfs-sandbox/tests/record_codec.rs index 1e4355b8e9..95b6c884e1 100644 --- a/control/crates/layerfs-sandbox/tests/record_codec.rs +++ b/control/crates/layerfs-sandbox/tests/record_codec.rs @@ -88,3 +88,114 @@ fn a_workspace_record_is_written_in_its_order() { assert!(!std::fs::read_to_string(&file).unwrap().contains("\"name\"")); std::fs::remove_dir_all(dir).unwrap(); } + +#[test] +fn unmount_guard_forceable_rejects_crashed_and_already_forced_records() { + use std::os::unix::fs::PermissionsExt; + use std::sync::mpsc; + use std::time::Duration; + + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let dir = std::env::temp_dir().join(format!("lfs-unmount-guard-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let ws = dir.join("workspace"); + std::fs::create_dir_all(&ws).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&ws, std::fs::Permissions::from_mode(0o700)).unwrap(); + + let write_rec = |name: &str, text: &str| { + let p = ws.join(name); + let bytes = layerfs_bridge::control::Answer { + id: 1, + reply: layerfs_bridge::control::Reply::Sdk(value(text)), + } + .encode() + .unwrap(); + std::fs::write(&p, bytes).unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o600)).unwrap(); + }; + // 1. Crashed force unmount (state=Unmounting, forced=true, no active lock -> code=Unknown) + write_rec( + "01.unmount", + r#"{"v":1,"state":"Unmounting","startedMs":1,"path":"/w/1","forced":true}"#, + ); + let r1 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/1") + .unwrap() + .unwrap(); + assert_eq!(r1.get_text("code"), Some("Unknown")); + assert_eq!(r1.get("forced"), Some(&SdkValue::Bool(true))); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r1)); + + // 2. Crashed normal unmount (state=Unmounting, forced=false, no active lock -> code=Unknown) + write_rec( + "02.unmount", + r#"{"v":1,"state":"Unmounting","startedMs":1,"path":"/w/2","forced":false}"#, + ); + let r2 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/2") + .unwrap() + .unwrap(); + assert_eq!(r2.get_text("code"), Some("Unknown")); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r2)); + + // 3. Completed forced unmount that remained Retained (forced=true, code=Failed) + write_rec( + "03.unmount", + r#"{"v":1,"state":"Retained","startedMs":1,"path":"/w/3","forced":true,"code":"Failed","message":"m","detail":null}"#, + ); + let r3 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/3") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r3)); + + // 4. Completed normal unmount the daemon answered Retained at Detach + // (forced=false, code=Failed, stage=Detach) -> forceable! + write_rec( + "04.unmount", + r#"{"v":1,"state":"Retained","startedMs":1,"path":"/w/4","forced":false,"stage":"Detach","code":"Failed","message":"m","detail":null}"#, + ); + let r4 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/4") + .unwrap() + .unwrap(); + assert!(layerfs_sandbox::unmount_guard::forceable(&r4)); + + // 5. The same completed `Retained` answer without the daemon's stage: + // the machine wrote it after a send/connect failure, so force must + // not replace it. + write_rec( + "05.unmount", + r#"{"v":1,"state":"Retained","startedMs":1,"path":"/w/5","forced":false,"code":"Failed","message":"m","detail":null}"#, + ); + let r5 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/5") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r5)); + + // 6. A version-0 record (no `v`): answered, never forceable. + write_rec( + "06.unmount", + r#"{"state":"Retained","startedMs":1,"path":"/w/6","code":"Failed","message":"m","detail":null}"#, + ); + let r6 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/6") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r6)); + + // 7. A record from a newer version: answered, never forceable. + write_rec( + "07.unmount", + r#"{"v":2,"state":"Retained","startedMs":1,"path":"/w/7","forced":false,"stage":"Detach","code":"Failed","message":"m","detail":null}"#, + ); + let r7 = layerfs_sandbox::unmount_guard::find_in(&dir, "/w/7") + .unwrap() + .unwrap(); + assert!(!layerfs_sandbox::unmount_guard::forceable(&r7)); + + std::fs::remove_dir_all(dir).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(5)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} diff --git a/control/crates/layerfs-sandbox/tests/unmount_record.rs b/control/crates/layerfs-sandbox/tests/unmount_record.rs new file mode 100644 index 0000000000..47e42de631 --- /dev/null +++ b/control/crates/layerfs-sandbox/tests/unmount_record.rs @@ -0,0 +1,307 @@ +//! The unmount guard's record file across a forced attempt (layerfs-project#136): +//! an attempt that changed nothing must leave the earlier record as it was. +use layerfs_bridge::control::{ + Request, SdkRefusal, SdkValue, TeardownCustody, TeardownStage, UnmountControl, WorkspaceToken, +}; +use layerfs_history::WorkspaceId; +use layerfs_sandbox::unmount_guard::{begin_at, cancel, find_in, finish, forceable, repeat}; +use layerfs_sandbox::{lifecycle, OperationCause, OperationFailure}; +use layerfs_store::project::discovery; +use std::{ + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +/// A Sandbox folder with its `workspace/` subfolder, both private. +fn fixture(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("lfs-unmount-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let workspace = dir.join("workspace"); + std::fs::create_dir_all(&workspace).unwrap(); + for path in [&dir, &workspace] { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + dir +} + +fn record_file(dir: &Path) -> PathBuf { + dir.join("workspace").join("01.unmount") +} + +/// A completed record a force may replace: the daemon answered `Retained` +/// with its export still held at `Detach`. +fn completed() -> SdkValue { + SdkValue::map([ + ("v", SdkValue::Int(1)), + ("state", SdkValue::text("Retained")), + ("startedMs", SdkValue::Int(7)), + ("path", SdkValue::text("/w")), + ("forced", SdkValue::Bool(false)), + ("stage", SdkValue::text("Detach")), + ("code", SdkValue::text("Failed")), + ("message", SdkValue::text("detach not established")), + ("detail", SdkValue::Null), + ]) +} + +/// The same shape with another daemon stage, or with no stage at all (a send +/// or connect failure, or a refusal before any effect). +fn completed_at(stage: Option<&str>) -> SdkValue { + let mut fields = Vec::new(); + for (name, value) in [ + ("v", SdkValue::Int(1)), + ("state", SdkValue::text("Retained")), + ("startedMs", SdkValue::Int(7)), + ("path", SdkValue::text("/w")), + ("forced", SdkValue::Bool(false)), + ] { + fields.push((name.to_owned(), value)); + } + if let Some(stage) = stage { + fields.push(("stage".to_owned(), SdkValue::text(stage))); + } + for (name, value) in [ + ("code", SdkValue::text("Failed")), + ("message", SdkValue::text("detach not established")), + ("detail", SdkValue::Null), + ] { + fields.push((name.to_owned(), value)); + } + SdkValue::Map(fields) +} + +fn refused(code: &str) -> SdkRefusal { + SdkRefusal { + code: code.into(), + message: code.into(), + detail: SdkValue::Null, + } +} + +/// The earlier record comes back exactly, and still answers as it did. +fn assert_restored(file: &Path, prior: &SdkValue) { + let kept = discovery::load(file).unwrap(); + assert_eq!(kept.get_text("state"), prior.get_text("state")); + assert_eq!(kept.get_text("code"), prior.get_text("code")); + assert_eq!(kept.get("forced"), prior.get("forced")); + assert_eq!(kept.get_int("startedMs"), prior.get_int("startedMs")); + assert!(forceable(&kept), "the earlier record answers as it did"); +} + +#[test] +fn a_busy_force_keeps_the_record_it_replaced() { + let dir = fixture("busy"); + let file = record_file(&dir); + let prior = completed(); + discovery::save(&file, prior.clone()).unwrap(); + let attempt = begin_at(file.clone(), "/w", true).unwrap(); + assert_eq!( + discovery::load(&file).unwrap().get_text("state"), + Some("Unmounting") + ); + finish(&attempt, "/w", &Err(refused("Busy"))).unwrap(); + assert_restored(&file, &prior); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_dirty_force_keeps_the_record_it_replaced() { + let dir = fixture("dirty"); + let file = record_file(&dir); + let prior = completed(); + discovery::save(&file, prior.clone()).unwrap(); + let attempt = begin_at(file.clone(), "/w", true).unwrap(); + finish(&attempt, "/w", &Err(refused("Dirty"))).unwrap(); + assert_restored(&file, &prior); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_call_that_sent_nothing_keeps_the_record_it_replaced() { + let dir = fixture("cancel"); + let file = record_file(&dir); + let prior = completed(); + discovery::save(&file, prior.clone()).unwrap(); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + cancel(&attempt).unwrap(); + assert_restored(&file, &prior); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn a_first_attempt_that_does_not_run_leaves_no_record() { + let dir = fixture("first"); + let file = record_file(&dir); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + finish(&attempt, "/w", &Err(refused("Busy"))).unwrap(); + assert!(!file.exists()); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// Only the daemon's own `Retained(Detach)` answer is replaceable. A stage +/// another step stopped at, or a failure that never got a stage, is answered +/// as it stands: the daemon would refuse it, and `force` must not overwrite +/// the earlier answer with an attempt of its own. +#[test] +fn a_record_force_cannot_replace_keeps_its_answer() { + for (label, stage) in [ + ("join", Some("Join")), + ("revoke", Some("Revoke")), + ("close", Some("Close")), + ("abort", Some("Abort")), + ("no-stage", None), + ] { + let value = completed_at(stage); + assert!(!forceable(&value), "{label}: {value:?}"); + let answer = repeat(&value); + let error = answer.expect_err("a retained record answers Retained"); + assert_eq!(error.code, "Failed"); + assert_eq!(error.detail.get_text("state"), Some("Retained")); + } +} + +/// The daemon's own answer carries the stage the record keeps: a `Retained` +/// reply becomes `Failed` with `detail.stage`, while a refusal or a lost +/// exchange has no stage at all. +#[test] +fn a_retained_answer_carries_its_stage_into_the_record() { + let token = WorkspaceToken { + workspace: WorkspaceId::from_authority([9; 32]).unwrap(), + namespace: 3, + }; + let failure = |stage: TeardownStage| { + Box::new(OperationFailure { + request: Request::Unmount(UnmountControl { + token, + discard: true, + }), + cause: OperationCause::Retained(Box::new(TeardownCustody { + token, + stage, + detached: false, + work: None, + detail: "client unmount failed".into(), + forced: None, + })), + }) + }; + let refusal = lifecycle::operation(failure(TeardownStage::Detach)); + assert_eq!(refusal.code, "Failed"); + assert_eq!(refusal.detail.get_text("stage"), Some("Detach")); + assert!(!refusal.detail.get_bool("detached")); + + // The same record shape the attempt publishes: the stage it keeps is the + // daemon's, so this one is forceable and a `Join` one is not. + for (stage, forceable_stage) in [("Detach", true), ("Join", false)] { + let dir = fixture(&format!("stage-{stage}")); + let file = record_file(&dir); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + finish(&attempt, "/w", &Err(refusal_at(stage))).unwrap(); + let kept = discovery::load(&file).unwrap(); + assert_eq!(kept.get_text("stage"), Some(stage)); + assert_eq!(forceable(&kept), forceable_stage, "{stage}: {kept:?}"); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); + } +} + +/// The refusal shape `lifecycle::operation` builds from a daemon `Retained` +/// answer at `stage`. +fn refusal_at(stage: &str) -> SdkRefusal { + SdkRefusal { + code: "Failed".into(), + message: "retained".into(), + detail: SdkValue::map([("stage", SdkValue::text(stage))]), + } +} + +/// A version-0 record (the base of #644): no `v`, no `forced`, no `stage`. +fn version_zero() -> SdkValue { + SdkValue::map([ + ("state", SdkValue::text("Retained")), + ("startedMs", SdkValue::Int(7)), + ("path", SdkValue::text("/w")), + ("code", SdkValue::text("Failed")), + ("message", SdkValue::text("client unmount: exit 1")), + ("detail", SdkValue::Null), + ]) +} + +/// A record written before the forced fields existed is still read and +/// answered exactly as it was, and is never forceable. +#[test] +fn a_version_zero_record_is_answered_and_never_forced() { + let dir = fixture("version-zero"); + let file = record_file(&dir); + discovery::save(&file, version_zero()).unwrap(); + let value = find_in(&dir, "/w").unwrap().expect("version-0 record"); + assert_eq!(value.get_text("state"), Some("Retained")); + assert_eq!(value.get_text("code"), Some("Failed")); + assert!(!forceable(&value), "a pre-forced record is not forceable"); + let error = repeat(&value).expect_err("a retained record answers Retained"); + assert_eq!(error.code, "Failed"); + assert_eq!( + error.message, "client unmount: exit 1", + "the version-0 message is answered unchanged" + ); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// A version-1 record answers every field the version-0 reader used, with the +/// values it wrote: the new fields are additive, so an older reader reads the +/// record it expects and ignores the rest. +#[test] +fn a_current_record_answers_every_version_zero_field() { + let dir = fixture("version-one"); + let file = record_file(&dir); + let attempt = begin_at(file.clone(), "/w", false).unwrap(); + finish(&attempt, "/w", &Err(refusal_at("Detach"))).unwrap(); + let value = discovery::load(&file).unwrap(); + assert_eq!(value.get_int("v"), Some(1)); + + // The version-0 field set, and only it, is what the older reader sees. + let older = SdkValue::Map( + ["state", "startedMs", "path", "code", "message", "detail"] + .into_iter() + .filter_map(|name| Some((name.to_owned(), value.get(name)?.clone()))) + .collect(), + ); + assert_eq!(older.get_text("state"), Some("Retained")); + assert!(older.get_int("startedMs").is_some()); + assert_eq!(older.get_text("path"), Some("/w")); + assert_eq!(older.get_text("code"), Some("Failed")); + assert_eq!(older.get_text("message"), Some("retained")); + assert!(older.get("detail").is_some()); + // The same answer either way: the newer fields never change the reading. + let current = repeat(&value).expect_err("retained"); + let before = repeat(&older).expect_err("retained"); + assert_eq!( + (current.code, current.message), + (before.code, before.message) + ); + drop(attempt); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// A record from a newer version is answered as it stands and is never +/// forced: this build cannot know what the newer fields mean. +#[test] +fn a_newer_record_is_answered_and_not_forced() { + let dir = fixture("version-two"); + let file = record_file(&dir); + let mut fields = match completed() { + SdkValue::Map(fields) => fields, + _ => unreachable!(), + }; + fields[0].1 = SdkValue::Int(2); + discovery::save(&file, SdkValue::Map(fields)).unwrap(); + let value = find_in(&dir, "/w").unwrap().expect("version-2 record"); + assert!(!forceable(&value), "a newer record is not forceable"); + let error = repeat(&value).expect_err("retained"); + assert_eq!(error.code, "Failed"); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/docs/architecture/api/sdk/workspace.md b/docs/architecture/api/sdk/workspace.md index d0d357a916..fb2ac5f038 100644 --- a/docs/architecture/api/sdk/workspace.md +++ b/docs/architecture/api/sdk/workspace.md @@ -211,7 +211,7 @@ CLI layerfs workspace unmount [--discard] [--force] | Option | Effect | | --- | --- | | `discard` | drop uncommitted changes instead of refusing `Dirty` | -| `force` | cancel each running command first (5000 ms grace), then one abort write ends the connection, also after a normal unmount stalled; uncommitted changes are dropped | +| `force` | cancel each running command first (5000 ms grace), then one abort write ends the connection (also after a normal unmount stalled); on the native NFS route one forced client unmount takes its place and also finishes an entry kept at `Detach` when a normal unmount's helper failed or timed out while the client mount stayed; uncommitted changes are dropped | ```text normal: { kind: "Unmounted" } @@ -221,8 +221,19 @@ either may add cleanup_error (daemon succeeded; a local record stayed) The machine writes `workspace/.unmount` before it calls the daemon and records the outcome after, so a repeated unmount answers what the first one -established without dialling again; a Workspace the daemon no longer knows -answers `Unmounted` +established without dialling again. The one exception is a record the daemon +itself answered `Retained` at stage `Detach`: its export is still held and one +forced client unmount is what finishes it, so `force` may replace that record +with a forced attempt (the daemon's guard remains the real gate, and `Busy`, +`Dirty`, or an unsent call restores the earlier record). A `Failed` answer +without a stage — a send or connect failure, or a refusal before any effect — +and an in-progress, `Unknown`, already-forced, other-stage or pre-`forced` +record keeps its answer and is not forceable. The record is versioned (`v`): +version 1 adds `forced` and the daemon's `stage` to version 0's `state`, +`startedMs`, `path`, `code`, `message` and `detail`. A version-0 record has +neither field and is read and answered as before; a record from a newer +version is answered as it stands and is never forceable. A Workspace the daemon +no longer knows answers `Unmounted` ([`unmount_guard.rs`](../../../../control/crates/layerfs-sandbox/src/unmount_guard.rs)). | Error | When | diff --git a/docs/architecture/control/sandboxes/daemon-channel-and-exec.md b/docs/architecture/control/sandboxes/daemon-channel-and-exec.md index af14cd40dd..5b0af0880b 100644 --- a/docs/architecture/control/sandboxes/daemon-channel-and-exec.md +++ b/docs/architecture/control/sandboxes/daemon-channel-and-exec.md @@ -64,7 +64,8 @@ one before it. | --- | --- | --- | --- | | Daemon refusal (`Reply::Refused`) | its control code (`Busy`, `HeadMoved`, `Missing`, `Capacity`, `Invalid`, `Failed`, `Unknown`), `detail.phase` | yes, except `Unknown` | `refusal` in [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs) | | Sent, connection ended before the reply (phase `Receive`) | `Unavailable`, `outcome: "unknown"`, `retryable: false` | no | same | -| Other transport or protocol failure, a stopped terminal step (`Retained`), a reply of the wrong kind | `Failed`, with phase and `attempted` in the message | no once `attempted` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation`, [`daemon/failure.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/failure.rs) | +| Other transport or protocol failure, or a reply of the wrong kind | `Failed`, with phase and `attempted` in the message | no once `attempted` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation`, [`daemon/failure.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/failure.rs) | +| A stopped terminal step (`Retained`) | `Failed`, with the custody's `stage` (`Detach`, `Join`, ...) and `detached` in `detail`; the machine persists the stage in `workspace/.unmount`, and only `Detach` is forceable | no | `refusal` in [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), [`unmount_guard.rs`](../../../../control/crates/layerfs-sandbox/src/unmount_guard.rs) | | SDK method over one control record, before sending | `TooLarge` | yes: not sent | [`daemon/call.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/call.rs) | | Open refused: capacity notice, closed before admission, reset | `Capacity`, `Unknown` or `Unavailable`, each `retryable: true`; nothing attempted | yes | `daemon/pool.rs` | | No free connection in 60 s | `Busy`, "nothing was attempted" | yes | `daemon/pool.rs` | diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index b814f8aa7b..d0451ebeda 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -96,11 +96,11 @@ differs. `Exported`, `ExportLeaving` and `ExportKept` mirror `Ready`, | Step | NFS route | Source | | --- | --- | --- | -| Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount, and the host's mount table is read once to record whether the mount is listed. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Commit | Flush first (flush helper, or the export volume's own flush: `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere; a failed flush refuses the Commit as `Failed` at `commit:flush`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | -| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| ForceUnmount | Force helper once, then the same teardown | same | -| Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close. A failed or timed-out helper is settled by the mount table: a mount listed at Attach and gone now completes teardown, while a still-listed one or an unreadable table keeps `Retained(Detach)` | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| ForceUnmount | Force helper once, then the same teardown; also finishes an export kept at `Detach` whose client mount stayed | same | +| Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported; a child still unreaped past `REAP_WAIT` 5 s enters the retained queue and is polled by its one background reaper | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | ## Unmount @@ -163,7 +163,8 @@ each step is made once. ```text under the registry lock (a refusal changes nothing): 1 ExportKept / Retained -> Retained (a normal unmount stalled at - Join is admitted instead) + Join, or a forceable export + kept at Detach, is admitted) 2 Committing | Attaching | Closing -> Busy force:admission 3 no serving connection -> Invalid / Unknown / Busy 4 Commit knowledge: Idle -> Absent; LocalFailure + published -> @@ -198,6 +199,9 @@ reply ForceUnmounted { token, ForcedOutcome { facts, work, cleanup } } | --- | --- | | The abort write uses the connection's own control, bound from the daemon's `fusectl` tree at Attach; without it Force is refused before any effect | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`application/filesystem.rs`](../../../../engine/crates/layerfs-daemon/src/application/filesystem.rs) | | A normal unmount stalled at `Join` with its abort unused is forced from its retained session; its plain detach counts as the one detach | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`serving.rs`](../../../../engine/crates/layerfs-daemon/src/control/serving.rs) | +| An export kept at `Detach` whose export and receipt are still held and whose forced facts are unset is forced by the one forced client unmount (`umount -f` on macOS, `umount -f -l` on Linux); a still-listed mount keeps `Retained(Detach)` with forced facts. On Linux that helper detaches lazily, so its failure never counts as a detach even when the table no longer lists the mount | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs), [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) `ExportKept::forceable` | +| Normal NFS `Unmount` keeps `Retained(Detach)` when Attach never saw the mount listed (`Exported::tracked` false), when the table still lists it or cannot be read (`detached_export`), or when the helper detaches lazily (`-l`/`--lazy` in its own words: the mount leaves the table before the kernel gives its superblock back, so the table proves nothing); the mount point itself is never resolved (`listed` compares the configured path and the name below canonical mount home) | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | +| A helper past `KILL` and `REAP_WAIT` enters the retained queue, whose one background thread reaps and records its exit (shared with the macOS driver's `umount -f`). Its bound is one thread and 64 recorded exits, not time: a child whose exit never comes keeps that thread polling for as long as the custody lasts, because a deadline would drop the only handle that can reap it. For tests only, `ExportServingConfig` adds `mount_table`, `reap_wait` and `helper_wait`, and `ExportServing::reaped_helpers` reads observed exits | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | | `relinquish_unknown` permits local disposal of stopped unknown Commit custody; the receipt keeps it `Unknown` | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | | On the NFS route the force helper is the abort | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Outside Linux without NFS the guard runs and the call is refused (`kernel serving requires Linux`) | [`force.rs`](../../../../engine/crates/layerfs-daemon/src/control/force.rs) | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 48032f82a8..568fc1e120 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -331,15 +331,26 @@ or container, where the environment is passed differently. ## Mount (`mount.rs`, daemon `export.rs`) ```text - Attach: bind 127.0.0.1:0 -> Server::start -> port P - helper mount_nfs -o vers=3,tcp,port=P,mountport=P,... - client MOUNT MNT "/" -> root handle, AUTH_UNIX - Commit: flush helper (else the export volume's own flush: sync_volume_np - on macOS, sync(2) elsewhere), close_files, Commit - Unmount: unmount helper (busy is reversible), Server::stop, - revoke the engine mount, Close + Attach: bind 127.0.0.1:0 -> Server::start -> port P + helper mount_nfs -o vers=3,tcp,port=P,mountport=P,... + client MOUNT MNT "/" -> root handle, AUTH_UNIX + read host mount table once -> tracked: is listed? + Commit: flush helper (else the export volume's own flush: sync_volume_np + on macOS, sync(2) elsewhere), close_files, Commit + Unmount: unmount helper (busy is reversible); failure or deadline + settled by the mount table; Server::stop, revoke, Close + ForceUnmount: one forced client unmount (umount -f, or umount -f -l on + Linux), then the same teardown ``` +| Mount-table and helper rule | Source | +| --- | --- | +| Every step above is one attempt; the mount table is the only witness for a failed or timed-out unmount helper. | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| Attach reads the host's mount table once after the mount helper returns: one `/sbin/mount` run (macOS) or `/proc/self/mountinfo` read (Linux) per Attach; unmount timing is unaffected. | `Exported::tracked` in [`export_custody.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_custody.rs) | +| The mount point itself is never resolved (a lookup through a wedged mount can block or fail); `listed` compares the configured path and the same name below the canonical mount home. | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) `listed` | +| A failed or timed-out unmount helper completes teardown only when Attach saw it listed and the table no longer does; a still-listed mount, an unreadable table, or a helper that detaches lazily (`-l`) keeps `Retained(Detach)` and the export serving. | `detached_export` in [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | +| A helper past `KILL` and `REAP_WAIT` is reaped in the background by the shared retained queue, with nothing waiting for it or signalling twice. | [`process/retained.rs`](../../../../engine/crates/layerfs-bridge/src/process/retained.rs) | + | MOUNT procedure | Answer | | --- | --- | | 0 `NULL`, 4 `UMNTALL` | Empty success | @@ -352,6 +363,8 @@ Client options are the daemon's `Mounter::host`. macOS: `vers=3,proto=tcp,port={port},mountport={port},mountproto=tcp,nolock,actimeo=60` ([`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs)). An NFS ready receipt has `abi_major == 0` and its `mount_id` is the port. +Forced teardown on this route is in +[Daemon: Control calls](../daemon/control-calls.md#force). ## Application SQLite databases on the NFS route diff --git a/engine/crates/layerfs-bridge/src/process/mod.rs b/engine/crates/layerfs-bridge/src/process/mod.rs index 2d8f629d6c..4390fefb02 100644 --- a/engine/crates/layerfs-bridge/src/process/mod.rs +++ b/engine/crates/layerfs-bridge/src/process/mod.rs @@ -1,7 +1,9 @@ //! A local process as records and Sandboxes name it (moved from the Sandbox //! driver, layerfs-project#112): its id, kernel start time and boot, read //! from `/proc` on Linux (`linux.rs`) and `proc_pidinfo`/`sysctl` on macOS -//! (`darwin.rs`, this crate's one audited `unsafe` file). It sits below the +//! (`darwin.rs`, this crate's one audited `unsafe` file), and the retained +//! child custody both the daemon's NFS helpers and the macOS driver's +//! `umount -f` use after a KILL (`retained.rs`). It sits below the //! control layer so the home and the drivers share it without an edge //! between them. #[cfg(target_os = "macos")] @@ -9,8 +11,10 @@ mod darwin; mod identity; #[cfg(target_os = "linux")] mod linux; +mod retained; #[cfg(target_os = "macos")] pub use darwin::{boot, processes, stat, sync_volume_wait}; pub use identity::{pid, LocalProcess, Stat}; #[cfg(target_os = "linux")] pub use linux::{boot, processes, stat}; +pub use retained::Retained; diff --git a/engine/crates/layerfs-bridge/src/process/retained.rs b/engine/crates/layerfs-bridge/src/process/retained.rs new file mode 100644 index 0000000000..e451579d23 --- /dev/null +++ b/engine/crates/layerfs-bridge/src/process/retained.rs @@ -0,0 +1,155 @@ +//! Retained child custody, one implementation for every caller that stops a +//! helper with a deadline: the daemon's NFS mount and unmount helpers and the +//! macOS driver's `umount -f` (layerfs-project#136). A child that survived its +//! KILL and the post-KILL grace window is never waited on synchronously and is +//! never signalled twice. One background thread polls it instead and records +//! the exit it observes. +//! +//! The bound is one reaper thread and [`RECORDED`] observed exits, not time: a +//! child whose exit never arrives keeps the queue non-empty and the one thread +//! polling every [`PAUSE`] for as long as its custody lasts. That is the point +//! of keeping it; a deadline would drop the only handle that can reap it. +use std::{ + fmt, + process::Child, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +/// Between two looks at the retained queue. +const PAUSE: Duration = Duration::from_millis(20); +/// Observed exits kept for diagnostics; the oldest go first. +const RECORDED: usize = 64; +/// Consecutive failed waits after which a child is given up on (~0.5 s at +/// [`PAUSE`]). A wait that keeps failing cannot reap the child, so holding its +/// handle only polls a broken one; the failure is recorded instead. +const ERROR_TRIES: u8 = 25; + +/// Children a caller could not observe exiting, and the exits seen since. +/// `retain`, `reap`, `exits` and `record` are the whole surface: nothing here +/// signals a child, and no call waits for one. +#[derive(Clone, Debug, Default)] +pub struct Retained(Arc); + +#[derive(Debug, Default)] +struct Inner { + children: Mutex>, + exits: Mutex>, + reaping: AtomicBool, +} + +/// One retained child and, when its last wait failed, that failure. +#[derive(Debug)] +struct Kept { + child: Child, + error: Option, + failures: u8, +} + +impl Retained { + /// Keeps `child` and makes sure one background reaper is polling for it. + pub fn retain(&self, child: Child) { + self.reap(); + self.0 + .children + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(Kept { + child, + error: None, + failures: 0, + }); + if self.0.reaping.swap(true, Ordering::AcqRel) { + return; + } + let queue = self.0.clone(); + if std::thread::Builder::new() + .name("layerfs-child-reap".into()) + .spawn(move || loop { + if queue.reap() { + queue.reaping.store(false, Ordering::Release); + if queue.reap() || queue.reaping.swap(true, Ordering::AcqRel) { + break; + } + } + std::thread::sleep(PAUSE); + }) + .is_err() + { + self.0.reaping.store(false, Ordering::Release); + } + } + + /// Reaps every child that has exited; true when none remain. + pub fn reap(&self) -> bool { + self.0.reap() + } + + /// The exits observed so far, oldest first. + pub fn exits(&self) -> Vec<(u32, String)> { + self.reap(); + self.0 + .exits + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + } + + /// Records an exit the caller observed itself with its own `try_wait`. + pub fn record(&self, child: u32, status: impl fmt::Display) { + let mut exits = self + .0 + .exits + .lock() + .unwrap_or_else(|error| error.into_inner()); + exits.push((child, status.to_string())); + let excess = exits.len().saturating_sub(RECORDED); + exits.drain(..excess); + } +} + +impl Inner { + fn reap(&self) -> bool { + let mut children = self + .children + .lock() + .unwrap_or_else(|error| error.into_inner()); + let mut exits = self.exits.lock().unwrap_or_else(|error| error.into_inner()); + children.retain_mut(|kept| match kept.child.try_wait() { + Ok(Some(status)) => { + exits.push((kept.child.id(), status.to_string())); + false + } + Ok(None) => { + kept.error = None; + kept.failures = 0; + true + } + // A failed wait is not an exit. Keep the child so its exit is + // still observed, record the failure once (changed message or the + // final give-up), and stop polling it after `ERROR_TRIES`. + Err(error) => { + let message = error.to_string(); + if kept.error.as_deref() != Some(message.as_str()) { + kept.error = Some(message.clone()); + kept.failures = 0; + } + kept.failures = kept.failures.saturating_add(1); + if kept.failures < ERROR_TRIES { + return true; + } + exits.push(( + kept.child.id(), + format!("exit unobservable after {ERROR_TRIES} failed waits: {message}"), + )); + false + } + }); + let excess = exits.len().saturating_sub(RECORDED); + exits.drain(..excess); + children.is_empty() + } +} diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 2128e08893..ea503da692 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -20,10 +20,18 @@ //! every reference and descriptor given back), revokes the engine mount //! and runs the one logical Close. A step that is not established keeps //! the export and its engine mount in the entry, as FUSE custody is kept. -//! - ForceUnmount runs the forced helper once, then the same teardown. +//! A helper that failed or ran out of time is judged by the host's mount +//! table, never by the mount itself: only a mount Attach saw listed there, +//! and only an ordinary (not lazy) helper, can be known gone +//! (`export_helper::listed`, `Helper::lazy`). +//! - ForceUnmount runs the forced helper once, then the same teardown. It is +//! also how an export kept at Detach for a mount still listed is finished. +//! A helper that outlives its KILL goes to the retained queue, whose one +//! background thread polls it (no call waits for it). +pub(super) use super::export_custody::{ExportKept, ExportLeaving, Exported}; use super::{ - export_helper::{Helpers, Ran}, - kernel::{bounded, Kernel}, + export_helper::{listed, mounted_under, Helpers, Ran}, + kernel::Kernel, registry::{bound_mut, idle}, Failure, Service, Success, }; @@ -31,14 +39,11 @@ use crate::store::BoundWorkspace; use layerfs_bridge::control::{ AbortDisposition, Activity, ControlCode, DetachDisposition, ForceUnmounted, ForcedCleanup, ForcedFacts, ForcedOutcome, MountPhase, MountReceipt, MountStatus, MountWork, ReadyMount, - Reply, TeardownCustody, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, + Reply, TeardownStage, WorkspaceToken, MOUNT_DIRECTORY_LIMIT, }; use layerfs_nfs::{ExportConfig, Released, Server}; use layerfs_overlay::{CleanupState, KernelMount}; -#[cfg(target_os = "macos")] -use std::process::{Command, Stdio}; use std::{ - fmt, net::{IpAddr, Ipv4Addr, TcpListener}, path::PathBuf, sync::Arc, @@ -49,7 +54,7 @@ use std::{ /// `rsize`/`wsize` it asks the host client to mount with. One source: the /// receipt, the mount options and FSINFO cannot drift apart. const WINDOW: u32 = layerfs_nfs::TRANSFER_WINDOW; -pub use super::export_helper::{Helper, HELPER_WAIT}; +pub use super::export_helper::{Helper, HELPER_WAIT, REAP_WAIT}; /// Stale inodes one install refreshes through the mount point at most. pub const REFRESHED: usize = 4096; @@ -145,6 +150,18 @@ pub struct ExportServingConfig { pub refuse_apple_double: bool, /// Lookup references one export keeps before collecting. pub lookups_held: usize, + /// How long one client helper may run before it is killed and reported + /// as unknown. Production value [`HELPER_WAIT`]. + pub helper_wait: std::time::Duration, + /// The post-KILL window in which the helper's own exit is still observed + /// synchronously; past it the child goes to the retained queue. Zero + /// skips the window, so a test reaches the retained path at once with a + /// bounded wait. Production value [`REAP_WAIT`]. + pub reap_wait: std::time::Duration, + /// Test hook, `None` in production: a file read line by line in place of + /// the host's mount table, so a test states which mounts exist without + /// mounting anything. + pub mount_table: Option, } impl ExportServingConfig { pub fn new(mounts: PathBuf, command_uid: u32, command_gid: u32, mounter: Mounter) -> Self { @@ -156,6 +173,9 @@ impl ExportServingConfig { mounter, refuse_apple_double: true, lookups_held: layerfs_nfs::kernel::LOOKUPS_HELD, + helper_wait: HELPER_WAIT, + reap_wait: REAP_WAIT, + mount_table: None, } } } @@ -164,21 +184,37 @@ impl ExportServingConfig { #[derive(Debug)] pub struct ExportServing { config: ExportServingConfig, - helpers: Helpers, + helpers: Arc, } impl ExportServing { pub fn start(config: ExportServingConfig) -> Self { - Self { - config, - helpers: Helpers::default(), - } + let helpers = Helpers::new(config.helper_wait, config.reap_wait); + Self { config, helpers } } pub fn config(&self) -> &ExportServingConfig { &self.config } + /// Test and diagnostic accessor: the helper exits the queue's one + /// background thread has observed since the daemon started (newest last, + /// at most 64 kept). + /// Production code reports a retained child inside its own refusal text + /// instead; nothing in the daemon reads this. + pub fn reaped_helpers(&self) -> Vec<(u32, String)> { + self.helpers.reaped() + } fn directory(&self, token: WorkspaceToken) -> PathBuf { self.config.mounts.join(token.namespace.to_string()) } + /// Whether the host's table still lists this mount. The table comes from + /// the home the daemon owns, never from the mount itself: see [`listed`]. + fn is_mounted(&self, directory: &str) -> std::io::Result { + let points = mounted_under(&self.config.mounts, self.config.mount_table.as_deref())?; + Ok(listed( + &points, + &self.config.mounts, + std::path::Path::new(directory), + )) + } fn companion_spill_dir(&self, token: WorkspaceToken) -> PathBuf { self.spill_root().join(token.namespace.to_string()) } @@ -200,7 +236,7 @@ impl ExportServing { if std::fs::remove_dir_all(&spill).is_ok() { lines.push(format!("removed leftovers under {}", spill.display())); } - let points = match mounted_under(&self.config.mounts) { + let points = match mounted_under(&self.config.mounts, self.config.mount_table.as_deref()) { Ok(points) => points, Err(error) => { lines.push(format!("mount table unreadable: {error}")); @@ -219,91 +255,6 @@ impl ExportServing { lines } } -/// Mount points at or under `home`, deepest first, from the host's table: -/// `mount`'s ` on ()` lines on macOS, -/// `/proc/self/mountinfo` (field 5, octal escapes) on Linux. -fn mounted_under(home: &std::path::Path) -> std::io::Result> { - #[cfg(target_os = "macos")] - let points: Vec = { - let out = Command::new("/sbin/mount") - .env("LC_ALL", "C") - .stdin(Stdio::null()) - .output()?; - (String::from_utf8_lossy(&out.stdout).lines()) - .filter_map(|line| { - Some(PathBuf::from( - line.split_once(" on ")?.1.rsplit_once(" (")?.0, - )) - }) - .collect() - }; - #[cfg(not(target_os = "macos"))] - let points: Vec = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) - .filter_map(|line| line.split(' ').nth(4)) - .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) - .collect(); - let mut found: Vec = (points.into_iter()) - .filter(|point| point.starts_with(home)) - .collect(); - found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); - found.dedup(); - Ok(found) -} -/// A serving export and the mount the host client holds of it. -pub(super) struct Exported { - pub server: Arc, - pub mount: KernelMount, - pub ready: ReadyMount, -} -/// A terminal operation owns the export; status still observes it. -pub(super) struct ExportLeaving { - pub ready: ReadyMount, - pub phase: MountPhase, - pub work: MountWork, -} -/// Export custody a terminal or attach step could not finish disposing: -/// kept exactly, never retried or settled by a later observation. -pub(super) struct ExportKept { - pub ready: Option, - pub stage: TeardownStage, - pub detail: String, - pub work: Option, - pub evidence: Box, - pub forced: Option, -} -impl ExportKept { - pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { - TeardownCustody { - token, - stage: self.stage, - detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), - work: self.work, - detail: bounded(&self.detail), - forced: self.forced.clone(), - } - } -} -impl fmt::Debug for ExportKept { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("ExportKept") - .field("ready", &self.ready) - .field("stage", &self.stage) - .field("detail", &self.detail) - .field("work", &self.work) - .field("evidence", &self.evidence) - .field("forced", &self.forced) - .finish() - } -} -impl fmt::Debug for Exported { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Exported") - .field("address", &self.server.local_addr()) - .field("mount", &self.mount) - .field("held", &self.server.held()) - .finish() - } -} /// The maintained counters of a serving export. pub(super) fn work(server: &Server) -> MountWork { let held = server.held(); @@ -311,8 +262,8 @@ pub(super) fn work(server: &Server) -> MountWork { MountWork { loops_configured: 1, loops_entered: 1, - loops_exited: 0, - loops_joined: 0, + loops_exited: u16::from(server.exited()), + loops_joined: u16::from(server.joined()), active: 0, retained: u32::try_from(held.retained).unwrap_or(u32::MAX), completed: served, @@ -535,11 +486,13 @@ impl Service { stage: TeardownStage::Detach, detail, work: Some(work(&server)), - evidence: Box::new(Exported { + exported: Some(Box::new(Exported { server: Arc::new(server), mount, ready: receipt(token, text, mount, port), - }), + tracked: false, + })), + evidence: Box::new(()), forced: None, }))); } @@ -558,7 +511,8 @@ impl Service { Ok(Exported { server: Arc::new(server), mount, - ready: receipt(token, text, mount, port), + ready: receipt(token, text.clone(), mount, port), + tracked: serving.is_mounted(&text).unwrap_or(false), }) } /// No client mount and no export of this attempt remains: revoke its @@ -579,6 +533,7 @@ impl Service { stage: TeardownStage::Revoke, detail: revoke.wire().detail, work: None, + exported: None, evidence: Box::new((failure, mount, released, revoke)), forced: None, })), @@ -703,6 +658,8 @@ impl Service { format!("client reported the mount busy; no terminal effect: {detail}"), )); } + Ran::Failed(_) | Ran::Interrupted(_) + if self.detached_export(&exported, helper.as_ref()) => {} Ran::Failed(detail) | Ran::Interrupted(detail) => { return Err(self.keep_export( token, @@ -716,6 +673,17 @@ impl Service { self.teardown_export(token, &workspace, exported, None)?; self.close(token, &workspace, None) } + /// Whether a failed or timed-out helper proves the client mount is gone: + /// Attach saw the mount listed (`tracked`), the helper cannot detach + /// lazily, and the host's table does not list it now. A lazy detach + /// (`umount -l`, the Linux force helper) removes the table entry while the + /// kernel still holds the superblock, so it is never proof. + fn detached_export(&self, exported: &Exported, helper: Option<&Helper>) -> bool { + !helper.is_some_and(Helper::lazy) + && exported.tracked + && (self.export.as_ref()) + .is_some_and(|s| matches!(s.is_mounted(&exported.ready.directory), Ok(false))) + } /// After the client mount is gone: stop the export, revoke the engine /// mount, remove the directory and leave the entry Unattached for Close. fn teardown_export( @@ -724,15 +692,16 @@ impl Service { workspace: &BoundWorkspace, exported: Exported, forced: Option, - ) -> Result<(), Failure> { + ) -> Result { self.export_phase(token, MountPhase::Draining, Some(work(&exported.server))); let Exported { server, mount, ready, + tracked, } = exported; - let released = match Arc::try_unwrap(server) { - Ok(server) => server.stop(), + let (released, stopped) = match Arc::try_unwrap(server) { + Ok(mut server) => (server.shutdown(), work(&server)), Err(server) => { return Err(self.keep_export( token, @@ -742,6 +711,7 @@ impl Service { server, mount, ready, + tracked, }, forced, )) @@ -755,6 +725,7 @@ impl Service { stage: TeardownStage::Revoke, detail: failure.wire().detail, work: None, + exported: None, evidence: Box::new((failure, mount, released)), forced, }; @@ -779,7 +750,7 @@ impl Service { .lock() .map(|mut last| *last = Some((token, released))) .ok(); - Ok(()) + Ok(stopped) } fn keep_export( &self, @@ -794,7 +765,8 @@ impl Service { stage, detail, work: Some(work(&exported.server)), - evidence: Box::new(exported), + exported: Some(Box::new(exported)), + evidence: Box::new(()), forced, }; let custody = kept.custody(token); @@ -819,6 +791,9 @@ impl Service { } /// Describes retained export custody for an operator. Observation only. pub fn retained_export(&self, token: WorkspaceToken) -> Result, Failure> { + if let Some(serving) = &self.export { + serving.helpers.reap(); + } let mut entries = self.entries.lock().map_err(|_| Failure::Poisoned)?; Ok(match &bound_mut(&mut entries, token)?.kernel { Kernel::ExportKept(kept) => Some(format!("{kept:?}")), @@ -877,7 +852,9 @@ impl Service { None => Ran::Failed("export serving not assembled".into()), }; match ran { - Ran::Done => facts.detach = DetachDisposition::Detached, + Ran::Done => {} + Ran::Busy(_) | Ran::Failed(_) | Ran::Interrupted(_) + if self.detached_export(&exported, helper.as_ref()) => {} Ran::Busy(detail) | Ran::Failed(detail) | Ran::Interrupted(detail) => { facts.detach = DetachDisposition::Failed; return Err(self.keep_export( @@ -889,8 +866,8 @@ impl Service { )); } } - let work = work(&exported.server); - self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; + facts.detach = DetachDisposition::Detached; + let work = self.teardown_export(token, &workspace, exported, Some(facts.clone()))?; let kernel = self.closed(token, &workspace, None, Some(facts.clone()))?; let (cleanup, observation_failure) = match self .owner @@ -930,8 +907,17 @@ impl Service { /// Terminal admission over an export-owning entry; the entry observes the /// export as leaving until the operation settles it. pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result { - match kernel { - Kernel::Exported(_) => {} + let exported = match kernel { + Kernel::Exported(_) => match std::mem::replace(kernel, Kernel::Unattached) { + Kernel::Exported(exported) => *exported, + _ => unreachable!("checked export"), + }, + Kernel::ExportKept(kept) if phase == MountPhase::Stopping && kept.forceable() => { + match std::mem::replace(kernel, Kernel::Unattached) { + Kernel::ExportKept(mut kept) => *kept.exported.take().expect("forceable export"), + _ => unreachable!("checked retained export"), + } + } Kernel::ExportLeaving(_) => { return Err(kernel_failure( ControlCode::Busy, @@ -946,16 +932,13 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result, + pub mount: KernelMount, + pub ready: ReadyMount, + pub tracked: bool, +} +/// A terminal operation owns the export; status still observes it. +pub(super) struct ExportLeaving { + pub ready: ReadyMount, + pub phase: MountPhase, + pub work: MountWork, +} +/// Export custody a terminal or attach step could not finish disposing: +/// kept exactly, never retried or settled by a later observation. +pub(super) struct ExportKept { + pub ready: Option, + pub stage: TeardownStage, + pub detail: String, + pub work: Option, + pub exported: Option>, + pub evidence: Box, + pub forced: Option, +} +impl ExportKept { + /// Whether one forced client unmount may still finish this custody: the + /// export and its receipt are held, the first boundary not established is + /// `Detach`, and nothing was forced yet. + /// + /// Custody a failed Attach left (`ready: None`, its inner export + /// `tracked: false`, because the mount helper was interrupted) is never + /// forceable: no receipt was published, so there is no mount to force off + /// by name, and a failed force would leave the same custody. It stays + /// retained until the daemon exits; the next start's `reclaim` forces the + /// leftover host mount off. This predates #644. + pub(super) fn forceable(&self) -> bool { + self.exported.is_some() + && self.stage == TeardownStage::Detach + && self.forced.is_none() + && self.ready.is_some() + } + pub(super) fn custody(&self, token: WorkspaceToken) -> TeardownCustody { + TeardownCustody { + token, + stage: self.stage, + detached: matches!(self.stage, TeardownStage::Revoke | TeardownStage::Close), + work: self.work, + detail: bounded(&self.detail), + forced: self.forced.clone(), + } + } +} +impl fmt::Debug for ExportKept { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let evidence: &dyn fmt::Debug = self + .exported + .as_ref() + .map_or(&*self.evidence as &dyn fmt::Debug, |e| e); + f.debug_struct("ExportKept") + .field("ready", &self.ready) + .field("stage", &self.stage) + .field("detail", &self.detail) + .field("work", &self.work) + .field("evidence", &evidence) + .field("forced", &self.forced) + .finish() + } +} +impl fmt::Debug for Exported { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Exported") + .field("address", &self.server.local_addr()) + .field("mount", &self.mount) + .field("held", &self.server.held()) + .finish() + } +} diff --git a/engine/crates/layerfs-daemon/src/control/export_helper.rs b/engine/crates/layerfs-daemon/src/control/export_helper.rs index a4796ab725..6e6e750e24 100644 --- a/engine/crates/layerfs-daemon/src/control/export_helper.rs +++ b/engine/crates/layerfs-daemon/src/control/export_helper.rs @@ -1,15 +1,17 @@ //! One bounded NFS client helper attempt, including post-KILL observation. +use layerfs_bridge::process::Retained; use nix::fcntl::{fcntl, FcntlArg, OFlag}; use std::{ io::{self, Read}, + path::{Path, PathBuf}, process::{Child, Command, Stdio}, - sync::Mutex, + sync::Arc, time::{Duration, Instant}, }; /// How long one helper run may take before it is killed and reported. pub const HELPER_WAIT: Duration = Duration::from_secs(60); -const REAP_WAIT: Duration = Duration::from_secs(5); +pub const REAP_WAIT: Duration = Duration::from_secs(5); const PAUSE: Duration = Duration::from_millis(5); const OUTPUT: usize = 2048; @@ -22,6 +24,16 @@ impl Helper { let words: Vec = line.split_whitespace().map(str::to_owned).collect(); (!words.is_empty()).then_some(Self(words)) } + /// Whether this helper can detach lazily: `-l`/`--lazy` (also inside a + /// combined word such as `-fl`) asks the kernel to drop the mount from the + /// host's table before its superblock goes. The table then cannot witness + /// that the client's mount ended, so a vanished entry proves nothing. + pub fn lazy(&self) -> bool { + self.0.iter().any(|word| { + word == "--lazy" + || (word.starts_with('-') && !word.starts_with("--") && word.contains('l')) + }) + } fn command(&self, port: u16, directory: &str) -> Option { let mut words = self.0.iter().map(|word| { word.replace("{port}", &port.to_string()) @@ -42,15 +54,33 @@ pub(super) enum Ran { } /// Retained children are never waited on synchronously or signalled twice. -/// A later helper attempt may only reap their observed exits, not replay them. +/// The retained queue's one background thread reaps their exits and records +/// their status; its bound is that thread, not time. #[derive(Debug, Default)] -pub(super) struct Helpers(Mutex>); +pub(super) struct Helpers { + retained: Retained, + pub helper_wait: Duration, + pub reap_wait: Duration, +} impl Helpers { - pub fn run(&self, helper: &Helper, port: u16, directory: &str) -> Ran { - { - let mut held = self.0.lock().unwrap_or_else(|error| error.into_inner()); - held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_)))); - } + pub fn new(helper_wait: Duration, reap_wait: Duration) -> Arc { + Arc::new(Self { + helper_wait, + reap_wait, + ..Self::default() + }) + } + pub fn reap(&self) -> bool { + self.retained.reap() + } + pub fn reaped(&self) -> Vec<(u32, String)> { + self.retained.exits() + } + fn retain(self: &Arc, child: Child) { + self.retained.retain(child) + } + pub fn run(self: &Arc, helper: &Helper, port: u16, directory: &str) -> Ran { + self.reap(); let Some(mut command) = helper.command(port, directory) else { return Ran::Failed("empty helper command".into()); }; @@ -86,7 +116,7 @@ impl Helpers { } } let mut output = Vec::new(); - let deadline = Instant::now() + HELPER_WAIT; + let deadline = Instant::now() + self.helper_wait; let status = loop { drain(&mut pipes, &mut output); if let Some(error) = pipe_error.take() { @@ -95,7 +125,7 @@ impl Helpers { match child.try_wait() { Ok(Some(status)) => break Ok(status), Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), - Ok(None) => break Err(format!("helper did not finish in {HELPER_WAIT:?}")), + Ok(None) => break Err(format!("helper did not finish in {:?}", self.helper_wait)), Err(error) => break Err(format!("helper wait: {error}")), } }; @@ -103,23 +133,28 @@ impl Helpers { Ok(status) => Ok(status), Err(detail) => { let kill = child.kill(); - let deadline = Instant::now() + REAP_WAIT; - let reaped = loop { - match child.try_wait() { - Ok(Some(_)) => break true, - Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), - Ok(None) | Err(_) => break false, + let deadline = Instant::now() + self.reap_wait; + let reaped = if self.reap_wait.is_zero() { + None + } else { + loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(PAUSE), + Ok(None) | Err(_) => break None, + } } }; let pid = child.id(); - let disposition = if reaped { - "exit reaped".to_owned() - } else { - self.0 - .lock() - .unwrap_or_else(|error| error.into_inner()) - .push(child); - "exit not observed after 5s; helper may remain in kernel wait; child custody retained".into() + let disposition = match reaped { + Some(status) => { + self.retained.record(pid, status); + format!("exit reaped ({status})") + } + None => { + self.retain(child); + format!("exit not observed after {:?}; helper may remain in kernel wait; child custody retained", self.reap_wait) + } }; Err(format!("{detail}; pid={pid}; KILL={kill:?}; {disposition}")) } @@ -141,6 +176,68 @@ impl Helpers { } } +/// Whether the host's table still lists `directory`: the path as configured, +/// or the same name below the canonical mount home (`/var` and `/tmp` are +/// `/private/var` and `/private/tmp` on macOS, which is what the table +/// prints). Only the home is resolved. Resolving the mount directory itself +/// would look the path up through the mount: against a wedged NFS client that +/// lookup can block with no bound, and it fails (EIO, ETIMEDOUT) exactly while +/// the mount is in trouble, which would read as "not mounted". +pub(super) fn listed(points: &[PathBuf], home: &Path, directory: &Path) -> bool { + if points.iter().any(|point| point == directory) { + return true; + } + let (Some(name), Ok(real)) = (directory.file_name(), home.canonicalize()) else { + return false; + }; + points.iter().any(|point| point == &real.join(name)) +} + +/// Mount points at or under `home`, deepest first, from `table` or the host's +/// mount table (`mount` on macOS, `/proc/self/mountinfo` on Linux). +pub(super) fn mounted_under(home: &Path, table: Option<&Path>) -> io::Result> { + let points: Vec = if let Some(table) = table { + let text = match std::fs::read_to_string(table) { + Ok(text) => text, + Err(error) if error.kind() == io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(error), + }; + (text.lines().map(str::trim).filter(|l| !l.is_empty())) + .map(PathBuf::from) + .collect() + } else { + #[cfg(target_os = "macos")] + let points = { + let out = Command::new("/sbin/mount") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .output()?; + if !out.status.success() { + return Err(io::Error::other(format!( + "mount listing failed: {}", + out.status + ))); + } + (String::from_utf8_lossy(&out.stdout).lines()) + .filter_map(|l| Some(PathBuf::from(l.split_once(" on ")?.1.rsplit_once(" (")?.0))) + .collect() + }; + #[cfg(not(target_os = "macos"))] + let points = (std::fs::read_to_string("/proc/self/mountinfo")?.lines()) + .filter_map(|line| line.split(' ').nth(4)) + .map(|point| PathBuf::from(point.replace("\\040", " ").replace("\\011", "\t"))) + .collect(); + points + }; + let real = home.canonicalize().ok(); + let mut found: Vec = (points.into_iter()) + .filter(|p| p.starts_with(home) || real.as_deref().is_some_and(|r| p.starts_with(r))) + .collect(); + found.sort_by_key(|point| std::cmp::Reverse(point.components().count())); + found.dedup(); + Ok(found) +} + fn drain(pipes: &mut [Box], output: &mut Vec) { for pipe in pipes { let mut bytes = [0; OUTPUT]; diff --git a/engine/crates/layerfs-daemon/src/control/force.rs b/engine/crates/layerfs-daemon/src/control/force.rs index 1699cdc211..2ad6b6254a 100644 --- a/engine/crates/layerfs-daemon/src/control/force.rs +++ b/engine/crates/layerfs-daemon/src/control/force.rs @@ -1,7 +1,8 @@ //! Forced terminal unmount: a before-effect guard, one abort write, the local //! drain with its one plain detach, then Revoke and Close. Nothing is repeated. //! A normal unmount stalled at Join is forced the same way, its plain detach -//! counting as the one detach. +//! counting as the one detach; on the NFS route the one forced client unmount +//! is the abort. #[cfg(target_os = "linux")] use super::serving::{stage, work, Attached, Evidence, Leaving}; use super::{ @@ -51,9 +52,16 @@ pub(super) fn guard( relinquish_unknown: bool, ) -> Result { // Stored custody answers first: no second abort or detach ever follows - // it. The one exception is a normal unmount stalled at Join, admitted - // idle with no Commit custody, whose abort control is still unused. + // it. Two shapes are the exception, both admitted with nothing yet + // aborted or detached: an export whose client unmount is not established + // (`ExportKept` at `Detach`, export still held, see + // `ExportKept::forceable`), where the one forced client unmount is the + // abort; and a normal unmount stalled at Join, admitted idle with no + // Commit custody, whose abort control is still unused. if let Kernel::ExportKept(kept) = &value.kernel { + if kept.forceable() { + return Ok(CommitKnowledge::Absent); + } return Err(Failure::Retained(Box::new(kept.custody(token)))); } #[cfg(target_os = "linux")] diff --git a/engine/crates/layerfs-daemon/src/control/mod.rs b/engine/crates/layerfs-daemon/src/control/mod.rs index 1fb83c0879..b57c756ddc 100644 --- a/engine/crates/layerfs-daemon/src/control/mod.rs +++ b/engine/crates/layerfs-daemon/src/control/mod.rs @@ -7,6 +7,7 @@ mod detach; mod diagnostic; mod dirty; mod export; +mod export_custody; mod export_helper; mod failure; mod force; diff --git a/engine/crates/layerfs-nfs/src/server.rs b/engine/crates/layerfs-nfs/src/server.rs index 8db26faf1f..ab57eef018 100644 --- a/engine/crates/layerfs-nfs/src/server.rs +++ b/engine/crates/layerfs-nfs/src/server.rs @@ -134,6 +134,8 @@ struct Shared { config: ExportConfig, counters: Counters, stopping: AtomicBool, + exited: AtomicBool, + joined: AtomicBool, connections: Mutex)>>, } /// A running export. @@ -165,6 +167,8 @@ impl Server { config, counters: Counters::default(), stopping: AtomicBool::new(false), + exited: AtomicBool::new(false), + joined: AtomicBool::new(false), connections: Mutex::default(), }); let accepting = shared.clone(); @@ -220,13 +224,19 @@ impl Server { pub fn verifier(&self) -> [u8; 8] { self.shared.verifier } + pub fn exited(&self) -> bool { + self.shared.exited.load(Ordering::Acquire) + } + pub fn joined(&self) -> bool { + self.shared.joined.load(Ordering::Acquire) + } /// Stops accepting, closes every connection, joins every thread, then /// releases the synthetic kernel's descriptors and references. The engine /// mount itself stays with the caller, who revokes it afterwards. pub fn stop(mut self) -> Released { self.shutdown() } - fn shutdown(&mut self) -> Released { + pub fn shutdown(&mut self) -> Released { let Some(accept) = self.accept.take() else { return Released::default(); }; @@ -234,6 +244,7 @@ impl Server { // Wake the blocking accept; it sees the flag and returns. drop(TcpStream::connect(self.address)); drop(accept.join()); + self.shared.exited.store(true, Ordering::Release); let connections = std::mem::take( &mut *self .shared @@ -247,6 +258,7 @@ impl Server { for (_, thread) in connections { drop(thread.join()); } + self.shared.joined.store(true, Ordering::Release); self.shared.kernel.release() } } @@ -258,6 +270,7 @@ impl Drop for Server { fn accept_loop(listener: &TcpListener, shared: &Arc) { for stream in listener.incoming() { if shared.stopping.load(Ordering::Acquire) { + shared.exited.store(true, Ordering::Release); return; } let Ok(stream) = stream else { continue }; diff --git a/engine/crates/layerfs-nfs/tests/helper_deadline.rs b/engine/crates/layerfs-nfs/tests/helper_deadline.rs index 781e82710a..490a0e1a3e 100644 --- a/engine/crates/layerfs-nfs/tests/helper_deadline.rs +++ b/engine/crates/layerfs-nfs/tests/helper_deadline.rs @@ -2,15 +2,20 @@ #[allow(dead_code)] #[path = "support/rig.rs"] mod rig; -use layerfs_bridge::control::{MountPhase, Reply, Request, UnmountControl}; -use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter, HELPER_WAIT}; +use layerfs_bridge::control::{ + CleanupObservation, DetachDisposition, ForcedCleanup, MountPhase, Reply, Request, + TeardownStage, UnmountControl, +}; +use layerfs_daemon::control::{ + ExportServing, ExportServingConfig, Failure, Helper, Mounter, HELPER_WAIT, +}; use layerfs_history::WorkspaceId; use std::{ fs, - path::PathBuf, + path::{Path, PathBuf}, process::Command, sync::{mpsc, Arc}, - time::Duration, + time::{Duration, Instant}, }; fn base(label: &str) -> PathBuf { @@ -140,3 +145,613 @@ fn verbose_helper_preserves_busy_refusal_without_pipe_deadlock() { assert!(reply.to_ascii_lowercase().contains("busy"), "{reply}"); assert_eq!(phase, MountPhase::Ready); } + +fn wait_reaped(serving: &ExportServing, pid: u32, limit: Duration) -> String { + let deadline = Instant::now() + limit; + loop { + if let Some((_, status)) = serving + .reaped_helpers() + .into_iter() + .find(|(p, _)| *p == pid) + { + return status; + } + assert!( + Instant::now() < deadline, + "timed-out helper {pid} was not reaped within {limit:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn assert_not_zombie(pid: u32) { + let out = Command::new("ps") + .args(["-o", "state=", "-p", &pid.to_string()]) + .output() + .unwrap(); + let state = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + assert!( + !state.starts_with('Z'), + "helper pid {pid} remained a zombie: state={state:?}" + ); +} + +#[test] +fn timed_out_helper_when_mount_remains_stays_retained_reaps_zombie_and_forces_cleanly() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("mount-remains"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let pidfile = base.join("unmount.pid"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo $$ > {}; sleep 5", pidfile.display()), + ]), + force: Helper(vec![ + "sh".into(), + "-c".into(), + format!("rm -f {}", table.display()), + ]), + flush: None, + }, + ); + config.helper_wait = Duration::from_millis(100); + config.reap_wait = Duration::ZERO; + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export.clone()); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([34; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // Mount remains in table when unmount helper times out: keep Retained(Detach). + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + let work = custody.work.expect("mount work"); + assert_eq!(work.loops_entered, 1); + assert_eq!(work.loops_exited, 0); + assert_eq!(work.loops_joined, 0); + } + other => panic!("expected Retained, got {other:?}"), + } + + // With no new Helpers::run call, background reaper reaps the child and records exit. + let pid: u32 = fs::read_to_string(&pidfile) + .unwrap() + .trim() + .parse() + .unwrap(); + let status = wait_reaped(&export, pid, Duration::from_secs(2)); + assert!( + !status.is_empty(), + "recorded exit status should not be empty" + ); + assert_not_zombie(pid); + + // Explicit ForceUnmount on ExportKept(Detach) finishes teardown and closes Workspace. + match service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => { + assert_eq!(done.outcome.facts.detach, DetachDisposition::Detached); + assert_eq!(done.outcome.work.loops_entered, 1); + assert_eq!(done.outcome.work.loops_exited, 1); + assert_eq!(done.outcome.work.loops_joined, 1); + assert!(matches!( + done.outcome.cleanup, + ForcedCleanup::Queued | ForcedCleanup::Gone + )); + } + other => panic!("expected ForceUnmounted, got {other:?}"), + } + let released = service.export_released(token).expect("export released"); + assert_eq!(released.retained, 0); + assert!(!Path::new(&ready.directory).exists()); + assert!(service.execute_control(&Request::Status(token)).is_err()); + match service + .execute_control(&Request::Cleanup(token)) + .unwrap() + .reply + { + Reply::Cleanup { state, .. } => assert_eq!(state, CleanupObservation::Gone), + other => panic!("expected Cleanup, got {other:?}"), + } + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + +#[test] +fn timed_out_helper_when_mount_already_gone_completes_teardown() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("mount-gone"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let pidfile = base.join("unmount.pid"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec![ + "sh".into(), + "-c".into(), + format!( + "rm -f {}; echo $$ > {}; sleep 5", + table.display(), + pidfile.display() + ), + ]), + force: Helper::words("true").unwrap(), + flush: None, + }, + ); + config.helper_wait = Duration::from_millis(100); + config.reap_wait = Duration::ZERO; + config.mount_table = Some(table); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export.clone()); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([35; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // Helper timed out, but mount table confirms the mount is gone: teardown completes. + let reply = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .expect("unmount should complete teardown when mount is gone") + .reply; + assert_eq!(reply, Reply::Unmounted(token)); + let released = service.export_released(token).expect("export released"); + assert_eq!(released.retained, 0); + assert!(!Path::new(&ready.directory).exists()); + assert!(service.execute_control(&Request::Status(token)).is_err()); + match service + .execute_control(&Request::Cleanup(token)) + .unwrap() + .reply + { + Reply::Cleanup { state, .. } => assert_eq!(state, CleanupObservation::Gone), + other => panic!("expected Cleanup, got {other:?}"), + } + + // The timed-out helper is still reaped in the background without becoming a zombie. + let pid: u32 = fs::read_to_string(&pidfile) + .unwrap() + .trim() + .parse() + .unwrap(); + let status = wait_reaped(&export, pid, Duration::from_secs(2)); + assert!(!status.is_empty()); + assert_not_zombie(pid); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + +#[test] +fn a_lazily_detached_force_helper_never_completes_teardown() { + // `umount -f -l` detaches lazily: the mount leaves the host's table while + // the kernel still holds its superblock. A vanished entry is therefore no + // proof that the client's mount ended, and Force must keep the custody. + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("lazy-force"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("mounts")).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + base.join("mounts"), + 501, + 20, + Mounter { + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("echo '{{dir}}' > {}", table.display()), + ]), + unmount: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + // `-l` is what the Linux force helper (`umount -f -l`) passes: + // the helper fails, yet the entry is gone from the table. + force: Helper(vec![ + "sh".into(), + "-c".into(), + format!("rm -f {}; exit 1", table.display()), + "-l".into(), + ]), + flush: None, + }, + ); + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([38; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // The plain helper fails while the table still lists the mount. + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + } + other => panic!("expected Retained, got {other:?}"), + } + + // The forced helper fails too, and the entry is gone from the table: + // a lazy detach, so the mount is not known gone. + let err = service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + assert_eq!( + custody.forced.as_ref().map(|facts| facts.detach), + Some(DetachDisposition::Failed) + ); + } + other => panic!("expected Retained, got {other:?}"), + } + assert!(Path::new(&ready.directory).exists()); + match service.execute_control(&Request::Status(token)) { + Ok(done) => match done.reply { + Reply::Status(status) => assert_eq!( + status.kernel.map(|kernel| kernel.phase), + Some(MountPhase::Retained) + ), + other => panic!("{other:?}"), + }, + Err(error) => panic!("{error:?}"), + } + assert!( + service.retained_export(token).unwrap().is_some(), + "the export stays in custody" + ); + assert!(service.export_released(token).is_none()); + assert!(!table.exists(), "the lazy helper removed the table entry"); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + +#[test] +fn canonical_mount_paths_and_unreadable_table_preserve_detach_safety() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("canon-table"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("real_mounts")).unwrap(); + let link_mounts = base.join("link_mounts"); + std::os::unix::fs::symlink(base.join("real_mounts"), &link_mounts).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + link_mounts, + 501, + 20, + Mounter { + // Record the canonicalized path (like macOS /sbin/mount printing /private/...). + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("cd '{{dir}}' && pwd -P > {}", table.display()), + ]), + unmount: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + force: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + flush: None, + }, + ); + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([36; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + + // M1: Replace mount.table with a directory so reading fails with a non-NotFound error. + // Unmount must NOT treat the read error as an empty mount table; it must keep Retained. + fs::remove_file(&table).unwrap(); + fs::create_dir(&table).unwrap(); + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + assert!( + matches!(err, Failure::Retained(_)), + "expected Retained, got {err:?}" + ); + assert!(Path::new(&ready.directory).exists()); + + // M2: Once mount.table is removed (NotFound -> mount gone), even if force helper exits 1 + // ("not currently mounted"), canonicalized tracked path recognizes the mount is gone. + fs::remove_dir(&table).unwrap(); + match service + .execute_control(&Request::ForceUnmount { + token, + relinquish_unknown: false, + }) + .unwrap() + .reply + { + Reply::ForceUnmounted(done) => { + assert_eq!(done.outcome.facts.detach, DetachDisposition::Detached); + assert_eq!(done.outcome.work.loops_joined, 1); + } + other => panic!("expected ForceUnmounted, got {other:?}"), + } + assert!(!Path::new(&ready.directory).exists()); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +} + +/// The mount home is a symbolic link (macOS `/var`, `/tmp` are +/// `/private/...`), so the host's table prints the resolved point. A still +/// mounted Workspace must stay `Retained(Detach)` even when the mount +/// directory itself cannot be resolved: that lookup would go through the +/// mount, which is exactly what a wedged client makes fail. +#[test] +fn a_listed_canonical_mount_stays_retained_when_the_mount_point_does_not_resolve() { + let (send, receive) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let base = base("canon-listed"); + drop(fs::remove_dir_all(&base)); + fs::create_dir_all(base.join("src")).unwrap(); + fs::create_dir_all(base.join("real_mounts")).unwrap(); + let link_mounts = base.join("link_mounts"); + std::os::unix::fs::symlink(base.join("real_mounts"), &link_mounts).unwrap(); + fs::write(base.join("src/file"), b"fixture").unwrap(); + let table = base.join("mount.table"); + let rig = rig::Rig::new( + &base.join("src"), + &base.join("work"), + Duration::from_secs(5), + ); + let mut config = ExportServingConfig::new( + link_mounts, + 501, + 20, + Mounter { + // The host prints the resolved path, as `mount` does on macOS. + mount: Helper(vec![ + "sh".into(), + "-c".into(), + format!("cd '{{dir}}' && pwd -P > {}", table.display()), + ]), + unmount: Helper(vec!["sh".into(), "-c".into(), "exit 1".into()]), + force: Helper::words("true").unwrap(), + flush: None, + }, + ); + config.mount_table = Some(table.clone()); + let export = Arc::new(ExportServing::start(config)); + let service = rig.service(export); + let token = match service + .execute_control(&Request::Mount { + workspace: WorkspaceId::from_authority([37; 32]).unwrap(), + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + let name = ready.directory.rsplit('/').next().unwrap().to_owned(); + let canonical = base.canonicalize().unwrap().join("real_mounts").join(&name); + let listed = || fs::read_to_string(&table).unwrap().trim().to_owned(); + assert_eq!( + listed(), + canonical.display().to_string(), + "the table lists the resolved point, never the configured home" + ); + + // The mount point stops resolving (a wedged mount answers EIO here). + fs::remove_dir(&ready.directory).unwrap(); + std::os::unix::fs::symlink(base.join("gone"), &ready.directory).unwrap(); + + // The table still lists it, so the mount is not gone: keep custody. + let err = service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap_err(); + match &err { + Failure::Retained(custody) => { + assert_eq!(custody.stage, TeardownStage::Detach); + assert!(!custody.detached); + } + other => panic!("expected Retained, got {other:?}"), + } + assert_eq!( + listed(), + canonical.display().to_string(), + "an unresolved mount point must not be read as a removed mount" + ); + + drop(service); + drop(rig); + fs::remove_dir_all(base).unwrap(); + let _ = send.send(()); + }); + receive + .recv_timeout(Duration::from_secs(10)) + .expect("bounded test timed out"); + worker.join().unwrap(); +}