Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
38af9c0
fix(daemon): reap timed-out unmount helpers and complete detached tea…
jiangkoumo Nov 10, 2026
7e501eb
fix(daemon,sandbox): harden mount table checks and unmount guard forc…
jiangkoumo Nov 10, 2026
09f9e4b
Merge upstream/layerfs_v2_0_0 into pr644-r1
jiangkoumo Nov 13, 2026
03204bb
refactor(bridge): share one retained-child reaper
jiangkoumo Nov 13, 2026
20b4faf
fix(daemon): never resolve the mount point to read the mount table
jiangkoumo Nov 13, 2026
523462a
fix(sandbox): keep the earlier unmount record a forced attempt replaced
jiangkoumo Nov 13, 2026
a7ca6c9
docs(daemon,sandbox): document the native NFS forced teardown
jiangkoumo Nov 13, 2026
6dc8a81
docs(daemon,nfs,api): tighten the forced teardown and mount-table wor…
jiangkoumo Nov 13, 2026
9a9012e
Merge upstream/layerfs_v2_0_0 into pr644-r1
jiangkoumo Nov 13, 2026
bbd15c3
docs(daemon,nfs,api): port native NFS forced teardown docs to v2.0 la…
jiangkoumo Nov 13, 2026
b49690d
fix(sandbox): align forceable comment and record macOS synchronous reap
jiangkoumo Nov 13, 2026
8bc995f
Merge upstream/layerfs_v2_0_0 (8cfb6e273) into fix/136-native-unmount…
jiangkoumo Nov 14, 2026
e710662
fix(daemon): never trust a lazy detach as proof the NFS mount is gone
jiangkoumo Nov 14, 2026
6f3cf77
fix(sandbox): force only a record the daemon answered Retained at Detach
jiangkoumo Nov 14, 2026
3468655
fix(sandbox): version the unmount record and prove the old record sti…
jiangkoumo Nov 14, 2026
f148f20
fix(daemon,bridge): state the retained reaper's real bound and keep t…
jiangkoumo Nov 14, 2026
89116b1
docs(control): a Retained terminal step carries its stage, not a phase
jiangkoumo Nov 14, 2026
254229c
Merge remote-tracking branch 'upstream/layerfs_v2_0_0' into pr644-merge
jiangkoumo Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 33 additions & 32 deletions control/crates/layerfs-sandbox-backend/src/local/os/macos/darwin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
//! deadline (layerfs-project#178 security rule 8).
//! The process facts (`proc_pidinfo`, `sysctl`) are `layerfs_bridge::process`
//! (layerfs-project#112), below the drivers and the home.
use layerfs_bridge::process::Retained;
use nix::fcntl::{fcntl, FcntlArg, OFlag};
use std::{
io::{self, Read},
path::{Path, PathBuf},
process::{Child, Command, Stdio},
sync::Mutex,
process::{Command, Stdio},
sync::LazyLock,
time::{Duration, Instant},
};

Expand All @@ -26,22 +27,10 @@ const TABLE_REAP: Duration = Duration::from_millis(500);
/// The most of the table that is read.
const TABLE: usize = 1 << 20;

/// Children that were killed and whose exit was not seen in time: kept, so
/// a later call can reap them.
static RETAINED: Mutex<Vec<Child>> = Mutex::new(Vec::new());

/// Reaps the retained children that have ended since.
fn reap_retained() {
let mut held = RETAINED.lock().unwrap_or_else(|error| error.into_inner());
held.retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_))));
}

fn retain(child: Child) {
RETAINED
.lock()
.unwrap_or_else(|error| error.into_inner())
.push(child);
}
/// Children a `umount` KILL or `mount` timeout did not release. Their exits
/// are reaped by the shared retained queue
/// ([`layerfs_bridge::process::Retained`]), never by a second wait here.
static RETAINED: LazyLock<Retained> = LazyLock::new(Retained::default);

/// One tool's standard output, or its failure with what it said.
fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> {
Expand Down Expand Up @@ -88,7 +77,7 @@ pub(crate) fn mount_line(line: &str) -> Option<(PathBuf, String)> {
/// asks no mounted server, so it is not expected to wait; the deadline is
/// for a host where it does.
pub(crate) fn mount_table() -> io::Result<String> {
reap_retained();
RETAINED.reap();
let mut child = Command::new(MOUNT)
.env_clear()
.env("LC_ALL", "C")
Expand Down Expand Up @@ -131,14 +120,21 @@ pub(crate) fn mount_table() -> io::Result<String> {
}
Ok(status) => Err(io::Error::other(format!("mount listing failed: {status}"))),
Err(error) => {
let pid = child.id();
let _ = child.kill();
let deadline = Instant::now() + TABLE_REAP;
while !matches!(child.try_wait(), Ok(Some(_))) {
if Instant::now() >= deadline {
retain(child);
break;
let reaped = loop {
match child.try_wait() {
Ok(Some(status)) => break Some(status),
Ok(None) if Instant::now() < deadline => {
std::thread::sleep(Duration::from_millis(2));
}
Ok(None) | Err(_) => break None,
}
std::thread::sleep(Duration::from_millis(2));
};
match reaped {
Some(status) => RETAINED.record(pid, status),
None => RETAINED.retain(child),
}
Err(error)
}
Expand All @@ -147,7 +143,7 @@ pub(crate) fn mount_table() -> io::Result<String> {

/// Forces the mount at `point` off once; a point no longer mounted is done.
pub(crate) fn force_unmount(point: &Path) -> io::Result<()> {
reap_retained();
RETAINED.reap();
let mut child = Command::new(UMOUNT)
.arg("-f")
.arg(point)
Expand Down Expand Up @@ -191,18 +187,23 @@ pub(crate) fn force_unmount(point: &Path) -> io::Result<()> {
let reaped = loop {
unmount_output(&mut stderr, &mut output);
match child.try_wait() {
Ok(Some(_)) => break true,
Ok(Some(status)) => break Some(status),
Ok(None) if Instant::now() < deadline => {
std::thread::sleep(Duration::from_millis(5));
}
Ok(None) | Err(_) => break false,
Ok(None) | Err(_) => break None,
}
};
let disposition = if reaped {
"exit reaped"
} else {
retain(child);
"exit not observed after 5s; child custody retained (possible kernel wait)"
let disposition = match reaped {
Some(status) => {
RETAINED.record(pid, status);
format!("exit reaped ({status})")
}
None => {
RETAINED.retain(child);
"exit not observed after 5s; child custody retained (possible kernel wait)"
.into()
}
};
return Err(io::Error::new(
error.kind(),
Expand Down
35 changes: 34 additions & 1 deletion control/crates/layerfs-sandbox/src/daemon/values.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
//! refusals with stable codes.
use super::workspace::{CommitReply, MountedWorkspace, Unmounted};
use crate::{OperationCause, OperationFailure};
use layerfs_bridge::control::{ForcedOutcome, SdkRefusal, SdkValue, WorkspaceStatus};
use layerfs_bridge::control::{
ForcedOutcome, SdkRefusal, SdkValue, TeardownStage, WorkspaceStatus,
};
use layerfs_bridge::sdk_values::{
commit_value, conflicts, conflicts_truncated, hex, lww, lww_truncated,
};
Expand All @@ -24,6 +26,22 @@ pub fn refusal(failure: &OperationFailure) -> Option<SdkRefusal> {
message: refused.detail.clone(),
detail: SdkValue::map([("phase", SdkValue::text(refused.phase.as_str()))]),
}),
// A terminal step the daemon stopped after effects. Its stage is the
// one fact a later `force` is judged by, so it travels in the detail
// the machine writes into `workspace/<id>.unmount`; a failure without
// such an answer (a send or connect failure, or a refusal before any
// effect) has no stage.
OperationCause::Retained(custody) => Some(SdkRefusal {
code: "Failed".to_owned(),
message: format!(
"teardown custody retained at {}; do not resend",
stage_text(custody.stage)
),
detail: SdkValue::map([
("stage", SdkValue::text(stage_text(custody.stage))),
("detached", SdkValue::Bool(custody.detached)),
]),
}),
// The request was sent but no reply arrived (the daemon reclaimed or
// expired the worker, or it died mid-operation): its effect is
// unknown, so the caller observes `Unavailable { outcome: unknown }`
Expand All @@ -45,6 +63,21 @@ pub fn refusal(failure: &OperationFailure) -> Option<SdkRefusal> {
}
}

/// A retained teardown stage as the machine's unmount record writes it.
fn stage_text(stage: TeardownStage) -> &'static str {
match stage {
TeardownStage::Detach => "Detach",
TeardownStage::Join => "Join",
TeardownStage::Owner => "Owner",
TeardownStage::Requests => "Requests",
TeardownStage::Lane => "Lane",
TeardownStage::Revoke => "Revoke",
TeardownStage::Close => "Close",
TeardownStage::Registry => "Registry",
TeardownStage::Abort => "Abort",
}
}

/// A mounted Workspace: its path and binding.
pub fn mounted(workspace: &MountedWorkspace, branch: &str) -> SdkValue {
let binding = &workspace.bound.binding;
Expand Down
8 changes: 7 additions & 1 deletion control/crates/layerfs-sandbox/src/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,11 +131,18 @@ impl SandboxState {

/// The token of the Workspace mounted at `path`.
pub fn token(&self, path: &str) -> Result<WorkspaceToken, SdkRefusal> {
crate::unmount_guard::check(self, path)?;
self.mounted(path, |mount| mount.workspace.bound.token)
}

/// The token of the Workspace at `path` after unmount admission checked it.
pub fn unmount_token(&self, path: &str) -> Result<WorkspaceToken, SdkRefusal> {
self.mounted(path, |mount| mount.workspace.bound.token)
}

/// The kernel directory of the Workspace mounted at `path`.
pub fn directory(&self, path: &str) -> Result<String, SdkRefusal> {
crate::unmount_guard::check(self, path)?;
self.mounted(path, |mount| mount.workspace.ready.directory.clone())
}

Expand All @@ -145,7 +152,6 @@ impl SandboxState {
/// once, for whatever is mounted there now; the full refresh replaces
/// the table, so a stale entry goes with it.
fn mounted<T>(&self, path: &str, pick: impl Fn(&Mount) -> T) -> Result<T, SdkRefusal> {
crate::unmount_guard::check(self, path)?;
let known = lock(&self.mounts)?
.get(path)
.map(|mount| (mount.workspace.bound.token, pick(mount)));
Expand Down
91 changes: 85 additions & 6 deletions control/crates/layerfs-sandbox/src/unmount_guard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,21 @@
//! admission and shared-record cleanup, never during the daemon's helper wait.
//! Mount paths are unique within one daemon lifetime; Sandbox removal clears
//! the terminal records before that directory can be reused.
//!
//! The record is JSON in the version its `v` field names.
//!
//! - Version 0 (no `v`) has `state`, `startedMs`, `path`, `code`, `message`
//! and `detail`, and nothing else. It predates any forced attempt.
//! - Version 1 ([`VERSION`]) adds `forced`, whether this attempt was started
//! with `force`, and — in a published result — `stage`, the daemon's own
//! retained teardown stage (`Null` when the failure never got a daemon
//! answer). Both are extra fields: a version-0 reader reads the version-0
//! fields unchanged and ignores them, which is what
//! `tests/unmount_record.rs` checks.
//! - A record whose `v` is above [`VERSION`] is answered as it stands and is
//! never forceable: the only step that could destroy unknown custody is the
//! one this version does not take. Every reader keeps answering version-0
//! records exactly as before.
use crate::files as discovery;
use crate::registry::SandboxState;
use layerfs_bridge::control::{SdkRefusal, SdkValue, WorkspaceToken};
Expand All @@ -14,6 +29,9 @@ use std::{
time::{SystemTime, UNIX_EPOCH},
};

/// The record version this build writes and the newest one it forces.
const VERSION: i64 = 1;

fn millis() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
Expand Down Expand Up @@ -51,6 +69,10 @@ pub struct Attempt {
_lease: Lease,
path: PathBuf,
started: i64,
forced: bool,
/// The record this attempt replaced, put back when the attempt has no
/// effect of its own (see [`restore`]).
prior: Option<SdkValue>,
}
fn result(value: &SdkValue) -> Outcome {
if value.get_text("state") == Some("Unmounted") {
Expand Down Expand Up @@ -123,12 +145,20 @@ fn inspect(path: &std::path::Path) -> Result<Option<SdkValue>, SdkRefusal> {
Some(_) => {
let error = progress(&value, false);
Ok(Some(SdkValue::map([
("v", value.get("v").cloned().unwrap_or_default()),
("state", SdkValue::text("Retained")),
(
"startedMs",
value.get("startedMs").cloned().unwrap_or_default(),
),
("path", value.get("path").cloned().unwrap_or_default()),
(
"forced",
value
.get("forced")
.cloned()
.unwrap_or(SdkValue::Bool(false)),
),
("code", SdkValue::text(error.code)),
("message", SdkValue::text(error.message)),
("detail", error.detail),
Expand Down Expand Up @@ -183,7 +213,7 @@ pub fn check_in(directory: &std::path::Path, path: &str) -> Result<(), SdkRefusa
Ok(())
}
/// [`find`] by scanning Sandbox folder `directory`'s unmount records.
fn find_in(directory: &std::path::Path, path: &str) -> Result<Option<SdkValue>, SdkRefusal> {
pub fn find_in(directory: &std::path::Path, path: &str) -> Result<Option<SdkValue>, SdkRefusal> {
for entry in unmounts_in(directory)? {
let entry = entry.map_err(failed)?;
if entry.path().extension().is_some_and(|ext| ext == "unmount") {
Expand Down Expand Up @@ -220,37 +250,77 @@ pub fn check_all(state: &SandboxState) -> Result<Vec<String>, SdkRefusal> {
}
Ok(recoverable)
}
pub fn forceable(value: &SdkValue) -> bool {
// Only a record the daemon itself answered `Retained` at `Detach` may be
// replaced by `force`: it holds an export whose client unmount was not
// established, and the one forced client unmount is what finishes it. The
// daemon's own guard remains the real gate, and a refusal answers with
// the record it holds. A record that is still `Unmounting` (its caller
// ended without publishing), one the machine wrote after a send or
// connect failure or a refusal before any effect (code `Failed` without
// the daemon's `stage`), one kept at another stage, one already forced,
// and a record written before this field existed keep their answer.
value.get_text("state") == Some("Retained")
&& value.get_int("v") == Some(VERSION)
&& value.get("forced") == Some(&SdkValue::Bool(false))
&& value.get_text("code") == Some("Failed")
&& value.get_text("stage") == Some(FORCEABLE_STAGE)
}
/// The daemon's retained stage one forced client unmount can still finish:
/// an NFS export kept because its client unmount was not established
/// (`ExportKept::forceable` in the daemon). Any other stage keeps its answer.
const FORCEABLE_STAGE: &str = "Detach";

/// Called under the Sandbox guard, after all before-effect checks.
pub fn begin(
state: &SandboxState,
token: WorkspaceToken,
path: &str,
forced: bool,
) -> Result<Attempt, SdkRefusal> {
let path_record = record(state, token);
begin_at(record(state, token), path, forced)
}
/// [`begin`] at the record file `path_record` (a Workspace's
/// `workspace/<id>.unmount`), holding its lease and replacing the record it
/// found with this attempt's `Unmounting` record.
pub fn begin_at(path_record: PathBuf, path: &str, forced: bool) -> Result<Attempt, SdkRefusal> {
let file = lease(&path_record.with_extension("unmount-lock"), true)?
.ok_or_else(|| refused("Busy", "Workspace is Unmounting"))?;
let attempt = Attempt {
_lease: file,
prior: observed(&path_record)?,
path: path_record,
started: millis(),
forced,
};
layerfs_store::project::discovery::save(
&attempt.path,
SdkValue::map([
("v", SdkValue::Int(VERSION)),
("state", SdkValue::text("Unmounting")),
("startedMs", SdkValue::Int(attempt.started)),
("path", SdkValue::text(path)),
("forced", SdkValue::Bool(attempt.forced)),
]),
)?;
Ok(attempt)
}
/// Undo this attempt's own record: what it replaced comes back, or the file
/// goes when there was none. A forced attempt admitted over an earlier record
/// that answers `Busy` or `Dirty` changed nothing, so the earlier record —
/// possibly one that says "do not resend" — must not be lost with it. A call
/// that sent nothing is the same case.
fn restore(attempt: &Attempt) -> Result<(), SdkRefusal> {
match &attempt.prior {
Some(prior) => layerfs_store::project::discovery::save(&attempt.path, prior.clone()),
None => std::fs::remove_file(&attempt.path).map_err(failed),
}
}
/// Publish while the Workspace lease prevents another terminal request.
/// No Sandbox guard is needed to replace this attempt's own result file.
pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), SdkRefusal> {
match outcome {
Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => {
std::fs::remove_file(&attempt.path).map_err(failed)?;
}
Err(error) if matches!(error.code.as_str(), "Busy" | "Dirty") => restore(attempt)?,
_ => {
let (state, code, message, detail) = match outcome {
Ok(_) => ("Unmounted", "", "", SdkValue::Null),
Expand All @@ -261,12 +331,21 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd
error.detail.clone(),
),
};
// The daemon's own retained stage, when it answered one; `Null`
// for a failure that never got that answer (see [`forceable`]).
let stage = match outcome {
Ok(_) => SdkValue::Null,
Err(error) => error.detail.get("stage").cloned().unwrap_or(SdkValue::Null),
};
layerfs_store::project::discovery::save(
&attempt.path,
SdkValue::map([
("v", SdkValue::Int(VERSION)),
("state", SdkValue::text(state)),
("startedMs", SdkValue::Int(attempt.started)),
("path", SdkValue::text(path)),
("forced", SdkValue::Bool(attempt.forced)),
("stage", stage),
("code", SdkValue::text(code)),
("message", SdkValue::text(message)),
("detail", detail),
Expand All @@ -277,7 +356,7 @@ pub fn finish(attempt: &Attempt, path: &str, outcome: &Outcome) -> Result<(), Sd
Ok(())
}
pub fn cancel(attempt: &Attempt) -> Result<(), SdkRefusal> {
std::fs::remove_file(&attempt.path).map_err(failed)
restore(attempt)
}
pub fn cleanup_failure(mut outcome: Outcome, error: SdkRefusal) -> Outcome {
// Do not replace a known daemon success with a cleanup Busy/Failed.
Expand Down
Loading