From 1c0a6fdb6d1e01572307f6141a19099c9e7f442c Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:58:52 +0800 Subject: [PATCH 01/19] fix(daemon): flush only the export volume on macOS before Commit When no LAYERFS_NFS_FLUSH helper is configured, flush_export previously called nix::unistd::sync(), forcing a host-wide sync(2) across every writable mount on macOS before each Workspace Commit. Add a narrow macOS FFI wrapper layerfs_bridge::process::sync_volume_wait calling Darwin's sync_volume_np(path, SYNC_VOLUME_WAIT) (SYNC_VOLUME_WAIT = 0x02 from macOS SDK ; checked alongside ) inside the bridge's existing audited process/darwin.rs module, and call it from flush_export on macOS. If sync_volume_np returns an error, log a warning and fall back to nix::unistd::sync() so client writeback flush is never skipped. Non-macOS targets and LAYERFS_NFS_FLUSH overrides keep their existing behavior. Tests: unit test sync_volume_wait on an existing temporary directory and a missing path in layerfs-bridge::process_identity, and test default export flush plus missing-directory fallback in layerfs-nfs::mounter. Deleted tests: none. Production LOC: 156995 -> 157027 (delta +32) Scope: first-party core/engine/control/API (core 39787, engine 73145 -> 73177, control 38513, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent bdeb9e91f374d61b55fc5ba775a6da8eedc9ec74 vs committed tree (engine/crates/layerfs-bridge/src/process/darwin.rs +18, engine/crates/layerfs-bridge/src/process/mod.rs +0, engine/crates/layerfs-daemon/src/control/export.rs +14; tests excluded). Co-authored-by: ImmuneFOMO --- .../layerfs-bridge/src/process/darwin.rs | 27 ++++ .../crates/layerfs-bridge/src/process/mod.rs | 2 +- .../layerfs-bridge/tests/process_identity.rs | 14 ++ .../layerfs-daemon/src/control/export.rs | 23 +++- engine/crates/layerfs-nfs/tests/mounter.rs | 128 +++++++++++++++++- 5 files changed, 188 insertions(+), 6 deletions(-) diff --git a/engine/crates/layerfs-bridge/src/process/darwin.rs b/engine/crates/layerfs-bridge/src/process/darwin.rs index 76f9a711e8..69b42c1ee1 100644 --- a/engine/crates/layerfs-bridge/src/process/darwin.rs +++ b/engine/crates/layerfs-bridge/src/process/darwin.rs @@ -7,12 +7,20 @@ use super::Stat; use nix::libc; use std::{ + ffi::CString, io, + os::unix::ffi::OsStrExt, + path::Path, process::{Command, Stdio}, sync::OnceLock, }; const SYSCTL: &str = "/usr/sbin/sysctl"; +/// Wait for volume writeback completion (`SYNC_VOLUME_WAIT = 0x02` in macOS +/// SDK ``; checked against ``, where `MNT_WAIT = 1` is +/// for `getfsstat(2)`/`vfs_sync` while `sync_volume_np(3)` and its flags live +/// in ``). +const SYNC_VOLUME_WAIT: libc::c_int = 0x02; /// One tool's standard output, or its failure with what it said. fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { @@ -155,3 +163,22 @@ fn boottime(text: &str) -> Option { }; Some(field("sec")? * 1_000_000 + field("usec")?) } + +/// Flush buffered writes on the volume containing `path` and wait for +/// completion (`sync_volume_np(path, SYNC_VOLUME_WAIT)`). +#[allow(unsafe_code)] +pub fn sync_volume_wait(path: &Path) -> io::Result<()> { + extern "C" { + fn sync_volume_np(path: *const libc::c_char, flags: libc::c_int) -> libc::c_int; + } + let c_path = CString::new(path.as_os_str().as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains interior NUL"))?; + // SAFETY: `c_path` is a valid NUL-terminated C string alive for the call, + // and `SYNC_VOLUME_WAIT` is the documented wait flag for `sync_volume_np`. + let rc = unsafe { sync_volume_np(c_path.as_ptr(), SYNC_VOLUME_WAIT) }; + match rc { + 0 => Ok(()), + 1.. => Err(io::Error::from_raw_os_error(rc)), + _ => Err(io::Error::last_os_error()), + } +} diff --git a/engine/crates/layerfs-bridge/src/process/mod.rs b/engine/crates/layerfs-bridge/src/process/mod.rs index f06582ef97..2d8f629d6c 100644 --- a/engine/crates/layerfs-bridge/src/process/mod.rs +++ b/engine/crates/layerfs-bridge/src/process/mod.rs @@ -10,7 +10,7 @@ mod identity; #[cfg(target_os = "linux")] mod linux; #[cfg(target_os = "macos")] -pub use darwin::{boot, processes, stat}; +pub use darwin::{boot, processes, stat, sync_volume_wait}; pub use identity::{pid, LocalProcess, Stat}; #[cfg(target_os = "linux")] pub use linux::{boot, processes, stat}; diff --git a/engine/crates/layerfs-bridge/tests/process_identity.rs b/engine/crates/layerfs-bridge/tests/process_identity.rs index b425aa88be..49f68e2ca6 100644 --- a/engine/crates/layerfs-bridge/tests/process_identity.rs +++ b/engine/crates/layerfs-bridge/tests/process_identity.rs @@ -45,3 +45,17 @@ fn the_process_table_lists_this_process_in_its_group() { assert_eq!(listed.group, stat.group); assert!(process::pid(u32::MAX).is_err()); } + +#[cfg(target_os = "macos")] +#[test] +fn sync_volume_wait_flushes_existing_directory_and_rejects_missing_path() { + let dir = std::env::temp_dir().join(format!("layerfs-bridge-sync-vol-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + assert!(process::sync_volume_wait(&dir).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + + let missing = dir.join("missing-subdir"); + let err = process::sync_volume_wait(&missing).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); +} diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index c92759b3b7..5c84dbf1f7 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -77,8 +77,9 @@ pub struct Mounter { pub mount: Helper, pub unmount: Helper, pub force: Helper, - /// Writes back what the client buffered before a Commit; `sync(2)` of - /// the daemon itself when absent. + /// Writes back what the client buffered before a Commit; `sync_volume_np` + /// of the export directory on macOS (falling back to `sync(2)`) or + /// `sync(2)` elsewhere when absent. pub flush: Option, } impl Mounter { @@ -583,7 +584,7 @@ impl Service { return Ok(()); }; match &serving.config.mounter.flush { - None => nix::unistd::sync(), + None => flush_default(std::path::Path::new(&directory)), Some(helper) => { match serving .helpers @@ -932,3 +933,19 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + let port = u16::try_from(ready.receipt.mount_id).unwrap(); + let mut nfs = Client::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, port)), 501, 20); + let root = nfs.mnt("/"); + + // 1. Existing directory: `sync_volume_wait` succeeds (`Ok(())`). + let f1 = nfs.touch(&root, b"one.txt", 0o644).unwrap(); + nfs.write_all(&f1.handle, 0, b"first\n", 6); + assert!(Path::new(&ready.directory).is_dir()); + match service + .execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("first".into()), + })) + .unwrap() + .reply + { + Reply::Committed(_) => {} + other => panic!("{other:?}"), + } + + // 2. Missing directory: `sync_volume_wait` returns `Err(NotFound)` and + // `flush_export` falls back to `sync(2)` rather than failing or skipping. + let f2 = nfs.touch(&root, b"two.txt", 0o644).unwrap(); + nfs.write_all(&f2.handle, 0, b"second\n", 7); + fs::remove_dir_all(&ready.directory).unwrap(); + assert!(!Path::new(&ready.directory).exists()); + match service + .execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("second".into()), + })) + .unwrap() + .reply + { + Reply::Committed(_) => {} + other => panic!("{other:?}"), + } + + fs::create_dir_all(&ready.directory).unwrap(); + drop(nfs); + match service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap() + .reply + { + Reply::Unmounted(unmounted) => assert_eq!(unmounted, token), + other => panic!("{other:?}"), + } + let released = service.export_released(token).unwrap(); + drop(service); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(&base).unwrap(); +} From c5d5df2fa972df56013745575ea71f1d1155ce1d Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:37:05 +0800 Subject: [PATCH 02/19] fix(persistence): admit Store writes on macOS through the directory lane Writer::acquire returned an empty lock on macOS, so every writer of the shared Store file ran with SQLite's busy_timeout=0 and met another process's write window as Busy (#534, #647). Widen the lane's cfg to macOS: acquire and release now flock the Store's parent directory there, an independent description that leaves SQLite's own main/WAL/SHM descriptors and fcntl locks untouched. Port the same admission to the two points that write outside a transaction. A writable Session takes the lane for its open: the PRAGMA and schema statements of an open write, and another opener now waits for it instead of failing. And the idle WAL checkpoint's try_acquire takes the same non-blocking branch on macOS as on Linux (LockExclusiveNonblock, EWOULDBLOCK -> Ok(None)), because the port must never turn that skip into a blocking wait while the caller holds its Session turn; #570 merged into a base with this function would have blocked there. A Store open refused for Busy now answers OpenRefusal::Busy (SDK code Busy) instead of Failed, since nothing was attempted. Tests: writer_admission's cross-process and open-time cases and idle_checkpoint's held-lane case run on macOS too (cfg widened; the child-side socket is made blocking again, as #570 does), and three_processes_open_and_reserve_concurrently is new: three children open the same Store at once and make twenty reservations each, all succeeding. Deleted tests: none. Production LOC: 157027 -> 157039 (delta +12) Scope: first-party core/engine/control/API (core 39787, engine 73177 -> 73188, control 38513 -> 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent 1c0a6fdb6d vs committed tree (engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs -5, engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +8, engine/crates/layerfs-store-service/src/store_service.rs +8, control/crates/layerfs-store/src/service.rs +1; tests excluded). --- control/crates/layerfs-store/src/service.rs | 1 + engine/crates/layerfs-persistence/Cargo.toml | 2 +- .../backend/sqlite/lane/writer_admission.rs | 35 +++--- .../src/backend/sqlite/session/connection.rs | 13 +++ .../tests/idle_checkpoint.rs | 2 +- .../tests/writer_admission.rs | 101 ++++++++++++++++-- .../src/store_service.rs | 17 ++- 7 files changed, 137 insertions(+), 34 deletions(-) diff --git a/control/crates/layerfs-store/src/service.rs b/control/crates/layerfs-store/src/service.rs index 02069bda6b..e57360725a 100644 --- a/control/crates/layerfs-store/src/service.rs +++ b/control/crates/layerfs-store/src/service.rs @@ -50,6 +50,7 @@ fn open(store: &Path, args: &OpenArgs) -> Result { .map_err(|refusal| match refusal { OpenRefusal::Unsupported(reason) => refused("Unsupported", reason.0), OpenRefusal::Missing(reason) => refused("NoStore", reason), + OpenRefusal::Busy(reason) => refused("Busy", reason), OpenRefusal::Failed(reason) => refused("Failed", reason), }) } diff --git a/engine/crates/layerfs-persistence/Cargo.toml b/engine/crates/layerfs-persistence/Cargo.toml index 2f0b22459e..090f56a2be 100644 --- a/engine/crates/layerfs-persistence/Cargo.toml +++ b/engine/crates/layerfs-persistence/Cargo.toml @@ -32,7 +32,7 @@ blake3 = { version = "=1.8.5", default-features = false } [target.'cfg(any(target_os = "macos", target_os = "linux"))'.dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs"] } -[target.'cfg(target_os = "linux")'.dev-dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dev-dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthread", "signal"] } [target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs index f9ed78e2d9..2c58af8ecb 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs @@ -1,7 +1,7 @@ -//! Linux local-volume cooperative ownership before one SQLite write attempt. +//! Cooperative local-volume ownership before one SQLite write attempt. use crate::backend::records::BackendError; use std::path::Path; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use { nix::{ errno::Errno, @@ -11,20 +11,22 @@ use { }; pub(crate) struct Writer { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] lock: Flock, } impl Writer { pub(crate) fn acquire(path: &Path) -> Result { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { // Never open another FD to SQLite's main/WAL/SHM files here: // closing any main-file FD cancels every process POSIX lock on - // that inode, including SQLite's live SHARED locks. An independent - // description of the existing parent directory coordinates short - // writer jobs without interfering with SQLite's descriptors. - // Multiple Store files in one directory share this coarse lane. + // that inode, including SQLite's live SHARED locks, and Darwin + // flock on that file conflicts with SQLite's own fcntl locks. An + // independent description of the existing parent directory + // coordinates short writer jobs without interfering with SQLite's + // descriptors. Multiple Store files in one directory share this + // coarse lane. let directory = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) @@ -47,13 +49,6 @@ impl Writer { Err(_) => Err(BackendError::Integrity), } } - #[cfg(target_os = "macos")] - { - // Darwin's flock conflicts with SQLite's open-file locks. Host Init - // is sole-owner; cross-session contention retains exact SQLite Busy. - let _ = path; - Ok(Self {}) - } #[cfg(not(any(target_os = "linux", target_os = "macos")))] { let _ = path; @@ -67,7 +62,7 @@ impl Writer { /// This is a single attempt, never a retry: no loop, sleep or busy handler /// follows it here, and a caller must not wrap it in one. pub(crate) fn try_acquire(path: &Path) -> Result, BackendError> { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { // The same independent directory description as `acquire`. let directory = path @@ -89,15 +84,15 @@ impl Writer { Err(_) => Err(BackendError::Integrity), } } - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { - // As `acquire`: no advisory lane off Linux. + // As `acquire`: no advisory lane on these platforms. Self::acquire(path).map(Some) } } pub(crate) fn release(self) -> Result<(), Self> { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { match self.lock.unlock() { Ok(file) => { @@ -107,7 +102,7 @@ impl Writer { Err((lock, _)) => Err(Self { lock }), } } - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { Ok(()) } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs index 6744ceba04..3149ee5380 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs @@ -205,6 +205,16 @@ impl Session { if !cfg!(any(target_os = "macos", target_os = "linux")) { return Err(BackendError::Integrity); } + // A writable open runs PRAGMA and schema statements that write, so it + // takes the same local-volume lane a transaction takes: another + // opener waits for it instead of meeting SQLite's Busy (#534, #647). + // A read-only open takes no lane, and the lane is released below + // before this session serves its first call. + let writer = if writable { + Some(crate::backend::sqlite::lane::writer_admission::Writer::acquire(path)?) + } else { + None + }; let flags = if writable { OpenFlags::SQLITE_OPEN_READ_WRITE } else { @@ -310,6 +320,9 @@ impl Session { if !diagnostic { profile::check(&profile)?; } + if let Some(writer) = writer { + writer.release().map_err(|_| BackendError::Unknown)?; + } super::thread_work::add(&work.borrow()); let background = (writable && selected == SqlitePersistenceProfile::Disposable) .then(|| super::background::Checkpointer::start(path)) diff --git a/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs b/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs index cdd38e853a..9f1c0a6c4f 100644 --- a/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs +++ b/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs @@ -160,7 +160,7 @@ fn a_reader_in_the_log_leaves_it_for_the_last_close() { assert_eq!(wal_bytes(&path), 0); } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn another_process_writer_lane_leaves_it_without_waiting() { use nix::fcntl::{Flock, FlockArg}; diff --git a/engine/crates/layerfs-persistence/tests/writer_admission.rs b/engine/crates/layerfs-persistence/tests/writer_admission.rs index eb9fce9fa5..2a9c6cc860 100644 --- a/engine/crates/layerfs-persistence/tests/writer_admission.rs +++ b/engine/crates/layerfs-persistence/tests/writer_admission.rs @@ -4,7 +4,7 @@ use layerfs_content::{FinalizedObject, ObjectRole}; use layerfs_history::HistoryCatalogConfig; use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; use layerfs_storage::{location::PackInfo, port::*, Storage, StoragePolicy}; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use std::{ io::{Read, Write}, os::unix::net::{UnixListener, UnixStream}, @@ -26,7 +26,7 @@ fn config(path: &Path) -> PersistenceConfig { fn create(path: &Path) -> SqliteHandles { SqliteHandles::create(config(path), StoragePolicy::frozen_default(), &authority()).unwrap() } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn open(path: &Path) -> SqliteHandles { SqliteHandles::open_writable( config(path), @@ -130,12 +130,12 @@ fn shared_session_reservation_waits_for_an_ordinary_public_read_owner() { assert_eq!(after.rollbacks, before.rollbacks); } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] struct Held { child: Child, socket: UnixStream, } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] impl Held { fn start(path: &Path, socket_path: &Path) -> Self { let listener = UnixListener::bind(socket_path).unwrap(); @@ -169,6 +169,7 @@ impl Held { } }; let mut held = Self { child, socket }; + held.socket.set_nonblocking(false).unwrap(); held.socket .set_read_timeout(Some(Duration::from_secs(3))) .unwrap(); @@ -189,11 +190,11 @@ impl Held { break; } assert!(Instant::now() < deadline, "child exit deadline"); - std::thread::yield_now(); + std::thread::yield_now() } } } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] impl Drop for Held { fn drop(&mut self) { let _ = self.child.kill(); @@ -202,7 +203,7 @@ impl Drop for Held { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn another_process_admission_precedes_the_single_sqlite_begin() { // Keep the owned AF_UNIX path below Darwin's sockaddr_un capacity. let temp = support::Temp::new("wa"); @@ -243,7 +244,7 @@ fn another_process_admission_precedes_the_single_sqlite_begin() { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn two_opened_handles_admit_eight_distinct_reservations_without_conflict() { let temp = support::Temp::new("writer-admission-two-handles"); let path = temp.join("store.sqlite"); @@ -373,7 +374,7 @@ fn queued_writers_outlive_the_old_window_and_all_succeed() { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn a_cross_process_waiter_outlives_the_old_window_and_succeeds() { let temp = support::Temp::new("wl"); let path = temp.join("store.sqlite"); @@ -497,7 +498,7 @@ fn writers_queue_behind_a_held_publication_lane_without_deadlock() { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn a_holder_process_that_exits_hands_the_lock_to_the_waiter() { let temp = support::Temp::new("wx"); let path = temp.join("store.sqlite"); @@ -606,3 +607,83 @@ fn a_thread_reentering_its_own_turn_is_refused_instead_of_deadlocking() { // The turn was released: the next call is admitted at once. assert!(handles.storage.reserve(reserve()).is_ok()); } + +/// Child half of `three_processes_open_and_reserve_concurrently`: it waits for +/// the parent's go on the inherited socket name, then opens the Store (which +/// takes the open-time lane) and makes twenty reservations there. +#[test] +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn concurrent_open_child() { + let Some(path) = std::env::var_os("LAYERFS_ADMISSION_OPEN_STORE") else { + return; + }; + let mut sock = + UnixStream::connect(std::env::var_os("LAYERFS_ADMISSION_OPEN_SOCK").unwrap()).unwrap(); + sock.write_all(b"R").unwrap(); + let mut go = [0u8; 1]; + sock.read_exact(&mut go).unwrap(); + let handles = open(Path::new(&path)); + for _ in 0..20 { + handles.storage.reserve(reserve()).unwrap(); + } +} + +/// Three processes open the same Store at once: the open-time lane and the +/// transaction lane queue them instead of meeting SQLite's Busy, so every +/// open and every reservation of all three succeeds (#534, #647). +#[test] +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn three_processes_open_and_reserve_concurrently() { + let temp = support::Temp::new("wo"); + let path = temp.join("store.sqlite"); + create(&path).seal().unwrap(); + let sock_path = temp.join("s"); + let listener = UnixListener::bind(&sock_path).unwrap(); + listener.set_nonblocking(true).unwrap(); + let exe = std::env::current_exe().unwrap(); + let children: Vec<_> = (0..3) + .map(|_| { + Command::new(&exe) + .args(["--exact", "concurrent_open_child", "--nocapture"]) + .env("LAYERFS_ADMISSION_OPEN_STORE", &path) + .env("LAYERFS_ADMISSION_OPEN_SOCK", &sock_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() + }) + .collect(); + let mut sockets = Vec::new(); + let deadline = Instant::now() + Duration::from_secs(5); + while sockets.len() < 3 { + match listener.accept() { + Ok((mut stream, _)) => { + stream.set_nonblocking(false).unwrap(); + let mut ready = [0u8; 1]; + stream.read_exact(&mut ready).unwrap(); + assert_eq!(&ready, b"R"); + sockets.push(stream); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { + std::thread::yield_now(); + } + Err(e) => panic!("barrier accept: {e}"), + } + } + for mut stream in sockets { + stream.write_all(b"G").unwrap(); + } + for child in children { + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let handles = open(&path); + let next = handles.storage.reserve(reserve()).unwrap(); + assert_eq!(next.first_pack_id, 61); + assert_eq!(next.first_ordinal, 61); +} diff --git a/engine/crates/layerfs-store-service/src/store_service.rs b/engine/crates/layerfs-store-service/src/store_service.rs index abcc147fd3..57c4861eb0 100644 --- a/engine/crates/layerfs-store-service/src/store_service.rs +++ b/engine/crates/layerfs-store-service/src/store_service.rs @@ -44,10 +44,23 @@ pub enum OpenRefusal { Unsupported(Unsupported), /// The Store file is absent: no Sandbox installed it into the volume yet. Missing(String), + /// Another writer held the Store at open, before any effect. The call is + /// a definite refusal the caller may repeat; nothing was changed. + Busy(String), /// The Store or the overlay template failed to open. Failed(String), } +/// The refusal for a Store open that failed: a Busy open is a definite +/// pre-effect refusal, every other open failure is not. +fn open_refusal(error: layerfs_storage::StorageError) -> OpenRefusal { + let reason = format!("Store open: {error}"); + match error { + layerfs_storage::StorageError::Busy => OpenRefusal::Busy(reason), + _ => OpenRefusal::Failed(reason), + } +} + /// The Store and overlay owner of this process's Store calls. pub struct StoreService { pub opened: OpenedStore, @@ -101,7 +114,7 @@ impl StoreService { CACHE_BYTES, ReservationBlocks::default(), ) - .map_err(|error| OpenRefusal::Failed(format!("Store open: {error}")))?; + .map_err(open_refusal)?; Self::owning(opened, template) } @@ -124,7 +137,7 @@ impl StoreService { CACHE_BYTES, ReservationBlocks::default(), ) - .map_err(|error| OpenRefusal::Failed(format!("Store open: {error}")))?; + .map_err(open_refusal)?; Self::owning(opened, template) } From 4f9a966f20206b6d066274c26bcef1dbd5ed7186 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:38:28 +0800 Subject: [PATCH 03/19] fix(store): keep a refused file-digest write a definite Busy Commit The commit path's file-digest port wraps a failed record_file_digests as WorkspaceError::Service(HistoryError). The original-cause classification only looked inside that carrier for an owner job and for an Arc, so a HistoryError fell through to CommitError::Workspace(Service(..)), whose uncertain() is true: a definite pre-effect refusal (SQLite's Busy at BEGIN IMMEDIATE, no statement attempted, no row written) was answered as an unknown outcome and the daemon kept that control slot for good (#540's mechanism, #647's first trigger). Move the carrier's downcast chain into store::service_cause and add the typed HistoryError: a Busy keeps its own class as CommitError::History, which the control boundary already answers as ControlCode::Busy and whose uncertain() is false, so settle_failure resolves the capture and the slot returns. Every other carrier value is unchanged. Tests: commit_service_busy.rs classifies the exact value the digest port produces (WorkspaceError::Service(HistoryError::Busy)) as CommitError::History(Busy) with uncertain() false, keeps UnknownOutcome unknown, and leaves a non-product carrier as a Workspace service failure. Deleted tests: none. Production LOC: 157039 -> 157052 (delta +13) Scope: first-party core/engine/control/API (core 39787, engine 73188 -> 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent c5d5df2fa vs committed tree (engine/crates/layerfs-store-service/src/store/captured.rs -6, engine/crates/layerfs-store-service/src/store/commit_types.rs +18, engine/crates/layerfs-store-service/src/store/mod.rs +1; tests excluded). --- .../src/store/captured.rs | 17 +++---- .../src/store/commit_types.rs | 23 +++++++++ .../layerfs-store-service/src/store/mod.rs | 1 + .../tests/commit_service_busy.rs | 47 +++++++++++++++++++ 4 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 engine/crates/layerfs-store-service/tests/commit_service_busy.rs diff --git a/engine/crates/layerfs-store-service/src/store/captured.rs b/engine/crates/layerfs-store-service/src/store/captured.rs index 9df08bbf7e..4e7ad30989 100644 --- a/engine/crates/layerfs-store-service/src/store/captured.rs +++ b/engine/crates/layerfs-store-service/src/store/captured.rs @@ -4,7 +4,7 @@ use super::pipeline::Piped; use super::{ BoundWorkspace, CapturedConstruction, CommitError, CommitFailure, CommitOptions, CommitSuccess, - PortError, ReleasedOwner, StoreOperation, + ReleasedOwner, StoreOperation, }; use crate::OwnerError; use layerfs_content::{ @@ -303,15 +303,12 @@ fn classify( (Err(label), _) => label, }; Err(match original { - Some(WorkspaceError::Service(service)) => match service.downcast::() { - Ok(owner) => CommitError::Owner(*owner), - Err(other) => match other.downcast::>() { - Ok(port) => CommitError::Content { - error: label, - provider: Some(*port), - }, - Err(other) => CommitError::Workspace(WorkspaceError::Service(other)), - }, + // The port's own typed cause: an owner job, the Store provider's + // failure, or a History failure that keeps its exact class (a + // definite pre-effect `Busy` is not an unknown outcome, #647). + Some(WorkspaceError::Service(service)) => match super::service_cause(service, label) { + Ok(error) => error, + Err(other) => CommitError::Workspace(WorkspaceError::Service(other)), }, Some(WorkspaceError::Content(error)) => CommitError::Content { error, diff --git a/engine/crates/layerfs-store-service/src/store/commit_types.rs b/engine/crates/layerfs-store-service/src/store/commit_types.rs index ccd0e87a17..014b1ded39 100644 --- a/engine/crates/layerfs-store-service/src/store/commit_types.rs +++ b/engine/crates/layerfs-store-service/src/store/commit_types.rs @@ -237,6 +237,29 @@ impl From for CommitError { Self::Workspace(error) } } +/// The exact original cause a Workspace service port carried, as the product +/// type it is: an owner job, the Store provider's failure, or a typed History +/// failure. A definite pre-effect History refusal (a `Busy` admission, for +/// one) must keep its own class here, so it is never read as an unknown +/// outcome (#647). Anything else stays a Workspace service failure. +pub fn service_cause( + service: Box, + label: ContentError, +) -> Result> { + match service.downcast::() { + Ok(owner) => Ok(CommitError::Owner(*owner)), + Err(other) => match other.downcast::>() { + Ok(port) => Ok(CommitError::Content { + error: label, + provider: Some(*port), + }), + Err(other) => match other.downcast::() { + Ok(history) => Ok(CommitError::History(*history)), + Err(other) => Err(other), + }, + }, + } +} pub fn owner_uncertain(error: &OwnerError) -> bool { match error { OwnerError::Unattempted { .. } => false, diff --git a/engine/crates/layerfs-store-service/src/store/mod.rs b/engine/crates/layerfs-store-service/src/store/mod.rs index 21beeb7bcc..dba2719b9b 100644 --- a/engine/crates/layerfs-store-service/src/store/mod.rs +++ b/engine/crates/layerfs-store-service/src/store/mod.rs @@ -38,3 +38,4 @@ pub use types::{ }; pub use commit_types::owner_uncertain; +pub use commit_types::service_cause; diff --git a/engine/crates/layerfs-store-service/tests/commit_service_busy.rs b/engine/crates/layerfs-store-service/tests/commit_service_busy.rs new file mode 100644 index 0000000000..b515109030 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/commit_service_busy.rs @@ -0,0 +1,47 @@ +//! #647: a definite pre-effect History refusal that reached the Commit through +//! a Workspace service port keeps its own class. The Store's file-digest write +//! meeting SQLite's Busy is `HistoryError::Busy` inside +//! `WorkspaceError::Service` (what `store/digests.rs` returns when +//! `record_file_digests` is refused); read as a `Workspace` service failure it +//! was answered as an unknown outcome and held every owner. +use layerfs_content::ContentError; +use layerfs_history::HistoryError; +use layerfs_store_service::store::{service_cause, CommitError}; + +/// The label the producer's Content result carries; only the provider branch +/// of the classification reads it. +fn label() -> ContentError { + ContentError::OutputRejected +} + +#[test] +fn a_refused_digest_write_classifies_as_history_busy() { + let service: Box = Box::new(HistoryError::Busy); + let classified = service_cause(service, label()).expect("the typed History cause is kept"); + match classified { + CommitError::History(HistoryError::Busy) => {} + other => panic!("{other:?}"), + } + assert!( + !classified.uncertain(), + "a refusal made before any effect is not an unknown outcome" + ); +} + +#[test] +fn a_history_cause_that_is_unknown_stays_unknown() { + let service: Box = Box::new(HistoryError::UnknownOutcome); + let classified = service_cause(service, label()).expect("the typed History cause is kept"); + assert!(matches!( + classified, + CommitError::History(HistoryError::UnknownOutcome) + )); + assert!(classified.uncertain()); +} + +#[test] +fn another_service_cause_is_not_taken_for_a_history_failure() { + let service: Box = + Box::new(std::io::Error::other("not a Store port")); + assert!(service_cause(service, label()).is_err()); +} From 1df9185e685c4569cfd7d2dda7b9d3a6d6275f19 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:47:48 +0800 Subject: [PATCH 04/19] test(store): expect the refused digest write as a definite Busy cause commit_pipeline's construction_failure_abandons_the_save reproduced exactly the value #647 misread: a held Store write lock refuses the new chunked file's digest write during construction, which the digest port carries as WorkspaceError::Service(HistoryError::Busy). It asserted that wrapper, i.e. the old classification. Assert the Commit's first cause as it now is - CommitError::History(HistoryError::Busy) - and that it is not an unknown outcome, so the test keeps checking the same refused write with the class the control boundary answers Busy from. Tests: the same three commit_pipeline cases. Deleted tests: none. Production LOC: 157052 -> 157052 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent 4f9a966f2 vs committed tree; only the external test engine/crates/layerfs-store-service/tests/commit_pipeline.rs changes (tests excluded). --- .../tests/commit_pipeline.rs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/engine/crates/layerfs-store-service/tests/commit_pipeline.rs b/engine/crates/layerfs-store-service/tests/commit_pipeline.rs index 54a1dfe108..9542cdfbc7 100644 --- a/engine/crates/layerfs-store-service/tests/commit_pipeline.rs +++ b/engine/crates/layerfs-store-service/tests/commit_pipeline.rs @@ -21,7 +21,7 @@ use layerfs_store_service::store::{ CommitPhase, CommitSuccess, }; use layerfs_store_service::telemetry::{mark_phases, Recording}; -use layerfs_workspace::{CapturedNamespace, Operation, Outcome, Position, Time, WorkspaceError}; +use layerfs_workspace::{CapturedNamespace, Operation, Outcome, Position, Time}; use std::{cell::Cell, rc::Rc, sync::Once}; const MIB: usize = 1024 * 1024; @@ -330,14 +330,21 @@ fn construction_failure_abandons_the_save() { &[("a.txt", 0, b"alpha\n"), ("c.bin", 0, &chunked)], ); let failure = failed(&fixture, &workspace); - let CommitError::Workspace(WorkspaceError::Service(cause)) = &failure.error else { - panic!("not the producer's own failure: {failure:?}"); - }; + // The producer's own first cause, kept as the product type it is: a + // definite pre-effect refusal is `History(Busy)`, never a Workspace + // service wrapper read as an unknown outcome (#647). + assert!( + matches!( + failure.error, + CommitError::History(layerfs_history::HistoryError::Busy) + ), + "not the producer's own refusal: {:?}", + failure.error + ); assert!( - cause - .downcast_ref::() - .is_some(), - "{cause:?}" + !failure.error.uncertain(), + "a refusal made before any effect is not unknown: {:?}", + failure.error ); fixture.cleanup(); } From 368b10884207d5d53d0d6fdacc7aaa1ed025e744 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:48:31 +0800 Subject: [PATCH 05/19] docs(architecture): describe macOS writer admission in Store pages Ports the flock and Busy-open wording of the #534 review fix (#570) to the current tree, without its 0700 directory, kernel-guard and SDK changes: - docs/architecture/engine/store/README.md: the cross-process writer row names the Store parent directory flock for Linux and macOS; the writer_admission.rs test bullet covers macOS child processes, concurrent opens and reservations, and keeps Cancelled on a signal Linux-only, as the test itself is. - docs/architecture/engine/store/backends.md: Writer::acquire step, the one-writer rule and the Cancelled row say Linux and macOS; the paragraph explains the separate descriptor, writable Session::connect taking the lane, and that only a writer bypassing the lane meets SQLite's Busy. - docs/architecture/engine/store/sqlite.md: Session::connect takes the writer lock when writable and releases it after profile::check; the lane holds the directory lock on both platforms. - docs/architecture/engine/store/lifecycle.md: the idle checkpoint's try_acquire lane and the closing paragraph say Linux and macOS. - docs/architecture/engine/store-calls/service.md: open_direct reports Busy("Store open: ..") before any effect, else Failed("Store open: .."). - Comments only in lane.rs, session/checkpoint.rs and records.rs: the publication lane, the skipped idle checkpoint and Cancelled are Linux and macOS, not Linux alone. Seal is unchanged: Session::seal takes the lane under cfg(target_os = "linux") and the lifecycle page keeps saying so. Deleted tests: none. Production LOC: 157052 -> 157052 (delta +0) Core 39787 -> 39787 (+0); engine 73201 -> 73201 (+0); control 38514 -> 38514 (+0); api 5550 -> 5550 (+0); reference 0 -> 0 (+0); dsh 0 -> 0 (+0). reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Method: python3 -B tools/production_loc.py --detail on exact parent 1df9185e6 and the staged tree. --- .../engine/store-calls/service.md | 3 ++- docs/architecture/engine/store/README.md | 9 +++++---- docs/architecture/engine/store/backends.md | 19 +++++++++++-------- docs/architecture/engine/store/lifecycle.md | 8 +++++--- docs/architecture/engine/store/sqlite.md | 9 +++++---- .../src/backend/records.rs | 2 +- .../src/backend/sqlite/lane/lane.rs | 8 ++++---- .../src/backend/sqlite/session/checkpoint.rs | 8 ++++---- 8 files changed, 37 insertions(+), 29 deletions(-) diff --git a/docs/architecture/engine/store-calls/service.md b/docs/architecture/engine/store-calls/service.md index 3e67765561..183a301bf3 100644 --- a/docs/architecture/engine/store-calls/service.md +++ b/docs/architecture/engine/store-calls/service.md @@ -57,7 +57,8 @@ open_direct(store, binding, cursor_key, template) SqliteHandles::open_writable (the writer session) factory(0) -> control reader (read-only session) Store::new(..).with_merge_scratch(..) - --> Failed("Store open: ..") + --> Busy("Store open: ..") / + Failed("Store open: ..") 4 Owner::start(template, pager 1024 KiB, namespaces 64) --> Failed("overlay template: ..") ``` diff --git a/docs/architecture/engine/store/README.md b/docs/architecture/engine/store/README.md index 838d0383bd..e9c38403cc 100644 --- a/docs/architecture/engine/store/README.md +++ b/docs/architecture/engine/store/README.md @@ -82,7 +82,7 @@ Nothing converts a Store from one kind to the other. | --- | --- | --- | | Platform for SQLite | macOS or Linux; elsewhere `BackendUnavailable` | [`open.rs`](../../../../engine/crates/layerfs-persistence/src/store/open.rs) | | Direct opening for Store calls | Linux local ext4/xfs/btrfs (macOS apfs/hfs); else `Unsupported`, no fallback | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | -| macOS cross-process writers | No directory lock; a second writer gets SQLite `Busy` | [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | +| Cross-process writers (Linux and macOS) | Directory `flock` on the Store's parent directory; an uncoordinated raw SQLite writer still gets SQLite `Busy` | [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | | PostgreSQL without the feature | `BackendUnavailable` | [`selection.rs`](../../../../engine/crates/layerfs-persistence/src/store/selection.rs) | | PostgreSQL lane wait | `lock_timeout_ms`, default 30 s; then `Busy` | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | | Rows per transaction | `TRANSACTION_ROW_LIMIT` 8 191 | [`publication.rs`](../../../../engine/crates/layerfs-persistence/src/storage/publication.rs) | @@ -111,9 +111,10 @@ Commands are written from source and were not run while writing this page. `cargo +1.85.1 test --locked -p layerfs-persistence --test heads` - **`writer_admission.rs`** checks first-come order, `Closed` for queued writers on close, writers queued behind `publish_lane`, refusal of a - thread re-entering its own turn, and (Linux only, with child processes) the - directory lock across processes, its hand-over when a holder exits and - `Cancelled` on a signal. + thread re-entering its own turn, and (with child processes on Linux and + macOS) the directory lock across processes, concurrent opens and + reservations, and its hand-over when a holder exits; on Linux it also + covers `Cancelled` when a signal interrupts the wait. `cargo +1.85.1 test --locked -p layerfs-persistence --test writer_admission` - **`diagnostics_during_lane.rs`** holds the lane for 3 s on one thread and requires `storage.diagnostics()` to return within 750 ms. Any platform the diff --git a/docs/architecture/engine/store/backends.md b/docs/architecture/engine/store/backends.md index e1853446a9..54060eb785 100644 --- a/docs/architecture/engine/store/backends.md +++ b/docs/architecture/engine/store/backends.md @@ -52,7 +52,7 @@ PostgreSQL runs them unchanged except the few it replaces in | `Busy` | `Busy` | `Busy` | Another writer held the lock; not retried. | | `Closed` | `Refused "Closed"` | `OwnershipUnavailable` | Store closed before this call's turn. | | `Rejected` | `Refused "Rejected"` | `OwnershipUnavailable` | PostgreSQL refused a new connection. | -| `Cancelled` | `Refused "Cancelled"` | `OwnershipUnavailable` | A signal interrupted the Linux writer wait. | +| `Cancelled` | `Refused "Cancelled"` | `OwnershipUnavailable` | A signal interrupted the writer wait (Linux and macOS). | | `ReadOnly` | `Refused "ReadOnly"` | `ContinuityUnavailable` | Write on a read-only session. | | `Capacity` | `Refused "Capacity"` | `Capacity("persistence")` | A bound or the disk margin was reached. | | `Integrity` | `Malformed` | `Integrity("persistence")` | Stored or supplied data broke a rule. | @@ -97,8 +97,8 @@ it never queues behind a held lane. SQLite write: Session::run(true, body) any process, one host 1 admission.enter in-process FIFO turn on this session 2 state mutex the connection - 3 Writer::acquire Linux: flock(LOCK_EX) on the Store's parent - directory; macOS: none (skipped if nested) + 3 Writer::acquire flock(LOCK_EX) on the Store's parent directory + (Linux and macOS) (skipped if nested) 4 BEGIN IMMEDIATE SQLite's write lock; busy_timeout 0 -> Busy 5 body; COMMIT or ROLLBACK; Unknown -> quarantine 6 finish_writer release the flock @@ -116,17 +116,20 @@ it never queues behind a held lane. | Rule | Enforced by | | --- | --- | | One write transaction at a time per session | `Admission` turn: [`admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/admission.rs) | -| One writer across processes on one Linux host | Directory `flock`: [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | +| One writer across processes on one Linux or macOS host | Directory `flock`: [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | | One writer in the SQLite file | `BEGIN IMMEDIATE` with `busy_timeout=0`: [`transaction.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs) | | One writer across PostgreSQL clients | `store_lane` row `FOR UPDATE`: [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/session/profile.rs) | | No busy handler, no retry | `busy_timeout` reads back 0; `Busy` is returned as is | The lock is taken on a separate descriptor of the Store's directory, never on the main, WAL or SHM file: closing a main-file descriptor would drop SQLite's -own POSIX locks. Several Store files in one directory share this lock. On -macOS `flock` conflicts with SQLite's locks, so there is no cross-process -writer wait: a second writer gets SQLite's `Busy`. Reads (`BEGIN`, or one -implicit snapshot) take the turn but never the directory lock or the lane. +own POSIX locks, and on macOS `flock` on that file would conflict with +SQLite's `fcntl` locks. Several Store files in one directory share this lock. +A writable `Session::connect` takes it too, so concurrent first openers +serialize their opening PRAGMA statements instead of meeting SQLite's `Busy`; +the schema statements a first open runs then take it once per transaction. +Reads (`BEGIN`, or one implicit snapshot) take the turn but never the +directory lock or the lane. The publication lane holds steps 1 and 3 across several transactions; see [History publication](history-publication.md#the-publish-lane). diff --git a/docs/architecture/engine/store/lifecycle.md b/docs/architecture/engine/store/lifecycle.md index b908155275..2e095537e1 100644 --- a/docs/architecture/engine/store/lifecycle.md +++ b/docs/architecture/engine/store/lifecycle.md @@ -126,7 +126,7 @@ StoreFile::checkpoint(&self) (live shared session; stays open) -wal empty or absent? ------------------------------------> Ok(true), no SQL FIFO turn (Writer); this thread holds the lane? ----------> Ok(false) quarantined or open transaction? ---------------------------> Unknown - Writer::try_acquire: Linux directory lane, one attempt + Writer::try_acquire: directory lane (Linux and macOS), one attempt another holder (EWOULDBLOCK) ----------------------------> Ok(false) a background checkpoint pass running? wait for it to end PRAGMA wal_checkpoint(TRUNCATE), once; release the lane @@ -141,8 +141,10 @@ live daemon still keeps its Store across Saves and Commits, and the 60 s idle close is unchanged. A checkpoint left by `Ok(false)` is never retried; its frames stay valid and the last close checkpoints them. A process-level close (`close_idle`, which the CLI runs before it exits) waits for a checkpoint still running and then closes the Store; it never skips it. The lane is taken first because a TRUNCATE checkpoint holds SQLite's WAL write lock, which a -writer of another process would otherwise meet as `Busy`; off Linux there is -no lane, and concurrent writers keep exact `Busy`, as for any write there. +writer of another process would otherwise meet as `Busy`; every host that +opens a SQLite Store takes this lane, so a writer there waits for it or the +checkpoint is left, and only a writer that bypasses the lane meets exact +`Busy`. Sources: [`session/checkpoint.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs), [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs). diff --git a/docs/architecture/engine/store/sqlite.md b/docs/architecture/engine/store/sqlite.md index 80d39de066..85cdb26da1 100644 --- a/docs/architecture/engine/store/sqlite.md +++ b/docs/architecture/engine/store/sqlite.md @@ -73,6 +73,7 @@ text ([`schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sq | Step in `Session::connect` | Detail | Source | | --- | --- | --- | +| Writer lock | When `writable`, takes `Writer::acquire(path)` across connect and releases it after `profile::check`; a read-only connect takes none | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Open | `READ_WRITE` or `READ_ONLY`, `NO_MUTEX`; statement cache 64 | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Guard | `busy_timeout=0`; `SQLITE_DBCONFIG_DEFENSIVE` on | same | | Settings | `profile::apply`, then read back and `profile::check` | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | @@ -109,10 +110,10 @@ also adds its SQL work to the calling thread's counters v inside: run(true, ..) enters nested: no flock, BEGIN IMMEDIATE ``` -The lane is one held admission turn plus, on Linux, the directory lock, kept -by one thread across several short transactions. No publication of any -process interleaves. `LaneHold` is `!Send`: it belongs to the thread that took -it. Source: [`lane.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs). +The lane is one held admission turn plus the directory lock (Linux and +macOS), kept by one thread across several short transactions. No publication +of any process interleaves. `LaneHold` is `!Send`: it belongs to the thread +that took it. Source: [`lane.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs). ## Heads and snapshots diff --git a/engine/crates/layerfs-persistence/src/backend/records.rs b/engine/crates/layerfs-persistence/src/backend/records.rs index 8d9d105f5d..9efc76cd55 100644 --- a/engine/crates/layerfs-persistence/src/backend/records.rs +++ b/engine/crates/layerfs-persistence/src/backend/records.rs @@ -114,7 +114,7 @@ pub(crate) enum BackendError { /// database, failed authentication): no session, so nothing ran. #[cfg_attr(not(feature = "postgresql"), allow(dead_code))] Rejected, - /// A signal interrupted the cross-process writer wait (Linux only). + /// A signal interrupted the cross-process writer wait (Linux or macOS). #[cfg_attr(not(target_os = "linux"), allow(dead_code))] Cancelled, ReadOnly, diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs index 8c9e0b32a9..1c31440b0b 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs @@ -1,7 +1,7 @@ -//! The Store's publication lane: one connection turn and, on Linux, the -//! cooperative writer lock, held by one thread across a short read-decide-write -//! sequence. Its own SQL work runs as ordinary transactions inside the held -//! turn; no other publication of any process can interleave. +//! The Store's publication lane: one connection turn and the cooperative +//! writer lock (Linux and macOS), held by one thread across a short +//! read-decide-write sequence. Its own SQL work runs as ordinary transactions +//! inside the held turn; no other publication of any process can interleave. use super::writer_admission::Writer; use crate::backend::records::BackendError; use crate::backend::sqlite::session::connection::Session; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs index 8a9fa3ca4d..151d5e3da3 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs @@ -12,10 +12,10 @@ impl Session { /// empty (it was, or it is now). `Ok(false)`: it could not complete now /// and is left as it is (its frames stay valid; the last close checkpoints /// them): this thread holds the publication lane, another process holds - /// the writer lane (Linux, tried without waiting), or a reader in another - /// process still reads a snapshot in the log (`busy`). Nothing is retried - /// and no busy handler waits. Short holds: the FIFO turn and the - /// connection lock are held for this one statement only. + /// the writer lane (Linux or macOS, tried without waiting), or a reader + /// in another process still reads a snapshot in the log (`busy`). + /// Nothing is retried and no busy handler waits. Short holds: the FIFO + /// turn and the connection lock are held for this one statement only. pub(crate) fn checkpoint_idle(&self) -> Result { if !self.writable { return Err(BackendError::ReadOnly); From c0bd28bd2f251a7d3eca59842e0fd385067d93a7 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:50:32 +0800 Subject: [PATCH 06/19] fix(persistence): take the directory lane for a seal on macOS too Session::seal is the sole owner's offline handoff: it checkpoints the WAL, closes the one connection and removes the sidecars. On Linux it has always held the writers' advisory lane (a flock on the Store's directory) across that sequence; the block was still gated to Linux, so a macOS seal ran with no advisory ownership while another opener could write. Widen the gate to any(linux, macos), as writer_admission, the publication lane and the idle checkpoint already are. Off those two platforms the binding is unchanged. Tests: layerfs-persistence writer_admission (12), idle_checkpoint (4), seal_allocation (5), serverless_store (5) on macOS. Deleted tests: none. Production LOC: 157052 -> 157052 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent 1df9185e6 vs committed tree; only comment lines are added to lifecycle/seal.rs, so the production total is unchanged. --- .../layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs index e0c9969e4b..bb73427003 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs @@ -19,7 +19,10 @@ impl Session { if state.quarantined || !state.connection.is_autocommit() { return Err(PersistenceError::Uncertain); } - #[cfg(target_os = "linux")] + // The same advisory lane as a write transaction: a seal is the sole + // owner's offline handoff, so no other opener may write while the + // checkpoint, close and sidecar removal below run. + #[cfg(any(target_os = "linux", target_os = "macos"))] let state = { let mut state = state; state.writer_hold = From 42aa0ca5a30b01f993ff2fe0bc97b816ffd750e8 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:52:01 +0800 Subject: [PATCH 07/19] test(api): hold the writer lane on macOS for the running-call open cache test a_running_call_keeps_its_store_open_against_a_push_or_remove keeps one call running by holding the Store directory's writer lane itself, then checks that store.remove and store.push are refused InUse while a lease is held. Since the lane exists on macOS as well, enable the case there instead of skipping it. Tests: layerfs-api store_open_cache, 2 passed on macOS (the case was gated to Linux before). Deleted tests: none. Production LOC: 157052 -> 157052 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent c0bd28bd2 vs committed tree; only the external test api/crates/layerfs-api/tests/store_open_cache.rs changes (tests excluded). --- api/crates/layerfs-api/tests/store_open_cache.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/crates/layerfs-api/tests/store_open_cache.rs b/api/crates/layerfs-api/tests/store_open_cache.rs index 06b913bb3b..e9af86ceed 100644 --- a/api/crates/layerfs-api/tests/store_open_cache.rs +++ b/api/crates/layerfs-api/tests/store_open_cache.rs @@ -156,9 +156,9 @@ fn a_worker_closes_its_open_copy_so_a_store_is_pulled_at_once_and_returns_fresh( let _ = std::fs::remove_dir_all(folder); } -/// A write waits for the Store directory's writer lock (Linux): holding it -/// here keeps one call running for as long as this test wants. -#[cfg(target_os = "linux")] +/// A write waits for the Store directory's writer lock (Linux and macOS): +/// holding it here keeps one call running for as long as this test wants. +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn a_running_call_keeps_its_store_open_against_a_push_or_remove() { use nix::fcntl::{Flock, FlockArg}; From 97a02e199e93d3a6b4687ab1523abc69f2f1f046 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:52:57 +0800 Subject: [PATCH 08/19] test(store-service): cover direct open refusals on both lane platforms Port the two kernel_guard cases that the macOS writer admission makes true, without the Store-directory 0700 changes they were originally written next to (this tree has no private-directory guard, so the fixture's default directory mode is enough): - open_direct_reports_busy_when_locked_and_failed_when_corrupt: another process holds BEGIN IMMEDIATE, so the open is refused OpenRefusal::Busy with "Store open: Store busy before effect" before any effect; a corrupt Store file is still OpenRefusal::Failed. - three_processes_open_direct_and_commit_concurrently plus its child helper open_concurrent_child: three processes open the same Store directly at one barrier and each commits twenty times; all opens wait instead of failing and each branch ends with exactly twenty commits. The file's OpenRefusal/StoreService import is now unconditional because the macOS cases use both. Tests: layerfs-store-service kernel_guard, 4 passed on macOS. Deleted tests: none. Production LOC: 157052 -> 157052 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent 42aa0ca5a vs committed tree; only the external test engine/crates/layerfs-store-service/tests/kernel_guard.rs changes (tests excluded). --- .../tests/kernel_guard.rs | 207 +++++++++++++++++- 1 file changed, 206 insertions(+), 1 deletion(-) diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index e99d46ea85..6b3adff582 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -3,7 +3,6 @@ //! a filesystem outside the allowlist (here tmpfs). The refusal comes before //! any file is opened: the Store file's bytes are unchanged and no overlay //! template is created. On macOS a local apfs or hfs directory is allowed. -#[cfg(not(target_os = "macos"))] use layerfs_store_service::{OpenRefusal, StoreService}; use std::path::{Path, PathBuf}; @@ -41,6 +40,212 @@ fn direct_access_is_allowed_on_a_local_macos_filesystem() { std::fs::remove_dir_all(directory).unwrap(); } +#[allow(dead_code)] +#[path = "support/installed_store.rs"] +mod installed; + +/// An open that finds the Store held by another writer's unwritten transaction +/// is a definite pre-effect `Busy` the caller may repeat; any other open +/// failure stays `Failed`. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn open_direct_reports_busy_when_locked_and_failed_when_corrupt() { + let fixture = installed::Fixture::new(1, "open-refusal"); + let sql = rusqlite::Connection::open(&fixture.config.path).unwrap(); + sql.busy_timeout(std::time::Duration::ZERO).unwrap(); + sql.execute_batch("BEGIN IMMEDIATE").unwrap(); + let template = fixture.directory.join("overlay.sqlite"); + let busy = StoreService::open_direct( + &fixture.config.path, + installed::BINDING, + installed::CURSOR, + &template, + ); + match busy { + Err(OpenRefusal::Busy(reason)) => { + assert_eq!(reason, "Store open: Store busy before effect"); + } + Err(other) => panic!("expected OpenRefusal::Busy, got {other:?}"), + Ok(_) => panic!("expected OpenRefusal::Busy, Store opened"), + } + sql.execute_batch("ROLLBACK").unwrap(); + drop(sql); + + let corrupt = fixture.directory.join("corrupt.sqlite"); + std::fs::write(&corrupt, b"not a valid sqlite store").unwrap(); + let failed = + StoreService::open_direct(&corrupt, installed::BINDING, installed::CURSOR, &template); + match failed { + Err(OpenRefusal::Failed(reason)) => assert!(reason.starts_with("Store open:"), "{reason}"), + Err(other) => panic!("expected OpenRefusal::Failed, got {other:?}"), + Ok(_) => panic!("expected OpenRefusal::Failed, Store opened"), + } + fixture.cleanup(); +} + +#[test] +fn open_concurrent_child() { + let Some(store) = std::env::var_os("LAYERFS_OPEN_CONCURRENT_STORE") else { + return; + }; + use layerfs_bridge::control::{SdkCall, SdkValue}; + use std::io::{Read, Write}; + use std::os::unix::net::UnixStream; + let id = std::env::var("LAYERFS_OPEN_CONCURRENT_ID").unwrap(); + let template = + std::path::PathBuf::from(std::env::var_os("LAYERFS_OPEN_CONCURRENT_TPL").unwrap()); + let mut sock = + UnixStream::connect(std::env::var_os("LAYERFS_OPEN_CONCURRENT_SOCK").unwrap()).unwrap(); + sock.write_all(b"R").unwrap(); + let mut go = [0u8; 1]; + sock.read_exact(&mut go).unwrap(); + let service = StoreService::open_direct( + Path::new(&store), + installed::BINDING, + installed::CURSOR, + &template, + ) + .unwrap_or_else(|e| panic!("agent-{id} open_direct: {e:?}")); + let call = |method: &str, fields: Vec<(&str, SdkValue)>| { + let mut map: Vec<(String, SdkValue)> = + fields.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(); + map.push(("project".to_owned(), SdkValue::text("installed"))); + layerfs_store_service::methods::call( + &service, + &SdkCall { + method: method.to_owned(), + args: SdkValue::Map(map), + }, + ) + .unwrap_or_else(|e| panic!("agent-{id} {method}: {e:?}")) + }; + let branch = format!("agent-{id}"); + call("branch.fork", vec![("name", SdkValue::text(&branch))]); + for step in 0..20 { + call( + "fs.write", + vec![ + ("branch", SdkValue::text(&branch)), + ("path", SdkValue::text(format!("/agent-{id}-{step}.txt"))), + ( + "bytes", + SdkValue::Bytes(format!("step {step}\n").into_bytes()), + ), + ( + "message", + SdkValue::text(format!("agent-{id} commit {step}")), + ), + ], + ); + } + service.close().unwrap(); +} + +/// Three processes open the same Store directly and each commits twenty times +/// at once: opening waits on the directory lane instead of failing, and no +/// Commit is lost or doubled. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn three_processes_open_direct_and_commit_concurrently() { + use std::io::{Read, Write}; + use std::os::unix::net::UnixListener; + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + let fixture = installed::Fixture::new(2, "concurrent-open"); + let sock_path = fixture.directory.join("barrier.sock"); + let listener = UnixListener::bind(&sock_path).unwrap(); + listener.set_nonblocking(true).unwrap(); + + let exe = std::env::current_exe().unwrap(); + let mut children = Vec::new(); + for id in 0..3 { + let overlay_dir = fixture.directory.join(format!("overlay-{id}")); + std::fs::create_dir(&overlay_dir).unwrap(); + let tpl = overlay_dir.join("overlay.sqlite"); + let child = Command::new(&exe) + .args(["--exact", "open_concurrent_child", "--nocapture"]) + .env("LAYERFS_OPEN_CONCURRENT_STORE", &fixture.config.path) + .env("LAYERFS_OPEN_CONCURRENT_ID", id.to_string()) + .env("LAYERFS_OPEN_CONCURRENT_TPL", &tpl) + .env("LAYERFS_OPEN_CONCURRENT_SOCK", &sock_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + children.push(child); + } + let mut sockets = Vec::new(); + let deadline = Instant::now() + Duration::from_secs(10); + while sockets.len() < 3 { + match listener.accept() { + Ok((mut stream, _)) => { + stream.set_nonblocking(false).unwrap(); + let mut ready = [0u8; 1]; + stream.read_exact(&mut ready).unwrap(); + assert_eq!(&ready, b"R"); + sockets.push(stream); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { + std::thread::yield_now(); + } + Err(e) => panic!("barrier accept: {e}"), + } + } + for mut stream in sockets { + stream.write_all(b"G").unwrap(); + } + let mut failures = Vec::new(); + for (id, child) in children.into_iter().enumerate() { + let output = child.wait_with_output().unwrap(); + if !output.status.success() { + failures.push(format!( + "agent-{id}: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + } + assert!( + failures.is_empty(), + "concurrent open_direct + 20 commits failed:\n{}", + failures.join("\n") + ); + let verify_dir = fixture.directory.join("overlay-verify"); + std::fs::create_dir(&verify_dir).unwrap(); + let service = StoreService::open_direct( + &fixture.config.path, + installed::BINDING, + installed::CURSOR, + &verify_dir.join("overlay.sqlite"), + ) + .unwrap(); + for id in 0..3 { + let branch = format!("agent-{id}"); + let list = layerfs_store_service::methods::call( + &service, + &layerfs_bridge::control::SdkCall { + method: "branch.list_commits".to_owned(), + args: layerfs_bridge::control::SdkValue::map([ + ( + "project", + layerfs_bridge::control::SdkValue::text("installed"), + ), + ("branch", layerfs_bridge::control::SdkValue::text(&branch)), + ("pageSize", layerfs_bridge::control::SdkValue::Int(50)), + ]), + }, + ) + .unwrap(); + let Some(layerfs_bridge::control::SdkValue::List(items)) = list.get("items") else { + panic!("missing items: {list:?}"); + }; + assert_eq!(items.len(), 20, "branch {branch} commit count"); + } + service.close().unwrap(); + fixture.cleanup(); +} + #[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] fn direct_access_is_unsupported_off_linux() { From 7d2bfbda9add05f81bd19ce113f56214cdaeb2de Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:39:11 +0800 Subject: [PATCH 09/19] docs(architecture): take a seal's directory lane on both lock platforms c0bd28bd2 widened Session::seal to any(linux, macos), but the two Seal flow boxes still said Linux-only: lifecycle.md's "## Seal" block read "Linux: take the directory writer lock" and seal-and-install.md's Init flow read "Linux: take the cross-process writer hold". Both now say the hold is taken on Linux and macOS, matching lifecycle/seal.rs. This corrects 368b10884, whose message states that seal is unchanged and that Session::seal takes the lane under cfg(target_os = "linux"). That was true of its base 1df9185e6 and stopped being true when c0bd28bd2 merged with it at 07113483b; the commit message itself is left as written and this one carries the correction. Two remaining Linux-only wordings go with it: store_open_cache.rs's module doc ("(Linux, where a writer waits on its Store directory's lock)") and the glossary's Publish lane row ("the flock on the Store's directory on Linux"). kernel_guard's three-process barrier socket was 101 bytes of Darwin's 103-byte sun_path limit under the default 48-byte TMPDIR, so any longer TMPDIR reached SUN_LEN before the test could run. The fixture label shrinks to "co" and the socket to "s" (78 bytes with a 57-byte TMPDIR, verified there), with the reason in a comment; open_concurrent_child now carries the same any(linux, macos) gate as its only caller. No test assertion changes. Tests: layerfs-store-service kernel_guard (4 passed) on macOS, and again with a 57-byte TMPDIR; layerfs-api store_open_cache (2 passed); layerfs-persistence writer_admission (12), seal_allocation (5), serverless_store (5); cargo clippy -D warnings --tests for layerfs-store-service and layerfs-api; cargo fmt --all -- --check; tools/check_product_boundary.py (1343 production files). Deleted tests: none. Production LOC: 157052 -> 157052 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at parent 07113483b vs the committed tree; only docs and external tests change, so the production total is unchanged. Merge commit 07113483b ("Merge docs sync for macOS writer admission") carries no Production LOC line. Its history is not rewritten; the record is here instead: Production LOC: 157052 -> 157052 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 73201, control 38514, api 5550, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the merge's first parent 97a02e199 (157052) and at its merged tree 32df2a943 (157052); 368b10884's side touches records.rs, lane.rs and checkpoint.rs in comments only, and that tree also counts 157052, so the merge changes no production line. --- api/crates/layerfs-api/tests/store_open_cache.rs | 4 ++-- docs/architecture/engine/project-init/seal-and-install.md | 2 +- docs/architecture/engine/store/lifecycle.md | 2 +- docs/architecture/glossary.md | 2 +- engine/crates/layerfs-store-service/tests/kernel_guard.rs | 7 +++++-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/api/crates/layerfs-api/tests/store_open_cache.rs b/api/crates/layerfs-api/tests/store_open_cache.rs index e9af86ceed..2622c3f9ca 100644 --- a/api/crates/layerfs-api/tests/store_open_cache.rs +++ b/api/crates/layerfs-api/tests/store_open_cache.rs @@ -2,8 +2,8 @@ //! open copy holds a lease under its root and is closed by the steps that //! read, replace or delete the file, so a Store is pulled back at once after a //! write and a Store that returns is never served from a stale handle; and -//! (Linux, where a writer waits on its Store directory's lock) a call that -//! is running keeps its Store open against a push or remove. +//! (where a writer waits on its Store directory's lock, Linux and macOS) a +//! call that is running keeps its Store open against a push or remove. use layerfs_api::home::{self, paths, write_record}; use layerfs_bridge::control::SdkValue; use std::net::TcpListener; diff --git a/docs/architecture/engine/project-init/seal-and-install.md b/docs/architecture/engine/project-init/seal-and-install.md index 59462a5f19..11ee9a3c57 100644 --- a/docs/architecture/engine/project-init/seal-and-install.md +++ b/docs/architecture/engine/project-init/seal-and-install.md @@ -24,7 +24,7 @@ live Store instead, and what each route leaves when its outcome is unknown. Arc::try_unwrap(storage, session) ----- still shared? --> Busy Session::seal writable, not quarantined, autocommit --- else ReadOnly/Uncertain - Linux: take the cross-process writer hold + take the cross-process writer hold (Linux and macOS) macOS: capture file identity; disable persistent WAL PRAGMA wal_checkpoint(TRUNCATE) ------- busy != 0 --> Busy close connection; release writer hold diff --git a/docs/architecture/engine/store/lifecycle.md b/docs/architecture/engine/store/lifecycle.md index 2e095537e1..4006822384 100644 --- a/docs/architecture/engine/store/lifecycle.md +++ b/docs/architecture/engine/store/lifecycle.md @@ -108,7 +108,7 @@ SqliteHandles::seal(self) Session::seal read-only session? ----------------------> Refused "ReadOnly" quarantined or open transaction? ---------------------------> Uncertain - Linux: take the directory writer lock + take the directory writer lock (Linux and macOS) macOS: capture file identity; clear SQLITE_FCNTL_PERSIST_WAL PRAGMA wal_checkpoint(TRUNCATE) -- not complete -----------> Busy close the connection, then release the writer lock diff --git a/docs/architecture/glossary.md b/docs/architecture/glossary.md index 468af21e96..c7c9f13016 100644 --- a/docs/architecture/glossary.md +++ b/docs/architecture/glossary.md @@ -50,7 +50,7 @@ LayerFS (backend, install, scope), the row gives both. | Platform | A machine's kind (`native`, `docker`, `apple-container`, `lima`). It fixes the Sandbox backend, the worker's address and what the support matrix allows. | [Machines: platforms and profiles](control/machines/platforms-and-profiles.md#platform) | | Project | A named LayerStack in a Store, written `/`. | [Project Init](engine/project-init/README.md) | | Publish | `branch.publish`: puts a Branch's head Commit on `main` as a new Layer. | [Store calls: branches](engine/store-calls/branches.md#publish) | -| Publish lane | The Store's admission turn plus its cross-process writer lock (the `flock` on the Store's directory on Linux; the `store_lane` row on PostgreSQL), held by a Commit across its head check, last merge step and publish. | [Store: history publication](engine/store/history-publication.md#the-publish-lane) | +| Publish lane | The Store's admission turn plus its cross-process writer lock (the `flock` on the Store's directory on Linux and macOS; the `store_lane` row on PostgreSQL), held by a Commit across its head check, last merge step and publish. | [Store: history publication](engine/store/history-publication.md#the-publish-lane) | | Reach | How a daemon on a machine dials a PostgreSQL Store: the URL rewritten for that platform and checked once at Sandbox create. | [Network: reach](control/network/reach.md) | | Sandbox | One daemon started by a backend on a machine, plus the commands run in it. The daemon serves the Workspaces; the Sandbox runtime owns the commands. | [Sandboxes](control/sandboxes/README.md) | | Save | A Storage write session: it accepts objects, packs them and `Save::finish` completes them while the Store stays open. | [Physical storage: saves](core/physical-storage/saves.md) | diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index 6b3adff582..4dba5e7088 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -83,6 +83,7 @@ fn open_direct_reports_busy_when_locked_and_failed_when_corrupt() { fixture.cleanup(); } +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn open_concurrent_child() { let Some(store) = std::env::var_os("LAYERFS_OPEN_CONCURRENT_STORE") else { @@ -152,8 +153,10 @@ fn three_processes_open_direct_and_commit_concurrently() { use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; - let fixture = installed::Fixture::new(2, "concurrent-open"); - let sock_path = fixture.directory.join("barrier.sock"); + // A short fixture label and socket name: `sockaddr_un.sun_path` is 104 bytes on + // Darwin (103 of path), and this directory sits under the host's TMPDIR. + let fixture = installed::Fixture::new(2, "co"); + let sock_path = fixture.directory.join("s"); let listener = UnixListener::bind(&sock_path).unwrap(); listener.set_nonblocking(true).unwrap(); From 523f7fda1317ff1714993295d3c87f401bcb3c34 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:39:39 +0000 Subject: [PATCH 10/19] test: bound the concurrent-open children and make the Store fixture private Review of #647, items 1 and 2. Bounded waits (item 1). three_processes_open_and_reserve_concurrently (layerfs-persistence) and three_processes_open_direct_and_commit_concurrently (layerfs-store-service) each collected their children with Child::wait_with_output(), which has no deadline, so a lane deadlock would hang the test instead of failing it. Both now call one wait_bounded helper per test crate: it polls try_wait against a 60 s deadline (the per-invocation ceiling is 120 s), kills and reaps the child and panics with the child's label when the deadline expires, and otherwise reads the pipes of a child that has already exited, so no wait can outlive a live child. Private Store fixture (item 2). installed::Fixture created its Store directory with the process umask (0755 under the CI umask of 022). #663's kernel guard refuses any Store directory that holds a group or other bit, so once both land the fixture's Store answered Unsupported and the three-process children panicked. Both fixtures now create it with mode 0700 through one private_directory helper: layerfs-store-service's tests/support/installed_store (which layerfs-daemon tests also include by path) covers Fixture::built_with and Fixture::cloned, and layerfs-daemon's own fixture covers Fixture::built. The kernel source directory and the overlay directories keep their previous modes; the guard looks only at the Store file's own directory. Tests: layerfs-persistence writer_admission 13 passed, heads 1, idle_checkpoint 4, serverless_store 5, snapshot 1, concurrent_first_installs 1, sqlite_lock_custody 2, disposable_startup_sync 1; layerfs-store-service kernel_guard 4 (both lane platforms on Linux), commit_service_busy 3, commit_pipeline 3, commit_reclaim 2, write_refusals 3, sdk_method_golden 1 under umask 022; layerfs-api store_open_cache 2; cargo +1.85.1 clippy -D warnings --tests for layerfs-persistence, layerfs-store-service and layerfs-daemon; fmt --check clean for every file this commit touches. Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the merge commit 95d0aef7f (160034) and at this commit's tree (160034); only external tests change, so the production total is unchanged. --- .../tests/support/installed_store.rs | 14 +++++++++- .../tests/writer_admission.rs | 26 ++++++++++++++++--- .../tests/kernel_guard.rs | 22 +++++++++++++++- .../tests/support/installed_store.rs | 16 ++++++++++-- 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/engine/crates/layerfs-daemon/tests/support/installed_store.rs b/engine/crates/layerfs-daemon/tests/support/installed_store.rs index a8322b9b04..06988a0087 100644 --- a/engine/crates/layerfs-daemon/tests/support/installed_store.rs +++ b/engine/crates/layerfs-daemon/tests/support/installed_store.rs @@ -16,6 +16,18 @@ use std::{ pub const BINDING: &[u8] = b"daemon-installed-store"; pub const CURSOR: [u8; 32] = [43; 32]; +/// Creates a Store's own directory owner-only (`0700`): the kernel guard +/// refuses a Store whose directory grants any group/other bit (#663), and the +/// daemon installs every Store that way. +fn private_directory(path: &std::path::Path) { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path).unwrap(); + } + #[cfg(not(unix))] + std::fs::create_dir(path).unwrap(); +} pub struct Fixture { pub directory: PathBuf, pub config: PersistenceConfig, @@ -36,7 +48,7 @@ impl Fixture { pub fn built(label: &str, build: impl FnOnce(&std::path::Path) -> usize) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - std::fs::create_dir(&directory).unwrap(); + private_directory(&directory); let source = directory.join("source"); std::fs::create_dir(&source).unwrap(); let count = build(&source); diff --git a/engine/crates/layerfs-persistence/tests/writer_admission.rs b/engine/crates/layerfs-persistence/tests/writer_admission.rs index 2a9c6cc860..2e9bfd33fc 100644 --- a/engine/crates/layerfs-persistence/tests/writer_admission.rs +++ b/engine/crates/layerfs-persistence/tests/writer_admission.rs @@ -202,6 +202,26 @@ impl Drop for Held { } } +/// Waits for a spawned test child to exit, bounded, then returns its output. +/// A lane deadlock has to fail this test instead of hanging it, so the wait +/// carries the same kind of deadline as `Held::release`; the 60 s ceiling also +/// stays under the per-invocation test limit. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn wait_bounded(mut child: Child, what: &str) -> std::process::Output { + let deadline = Instant::now() + Duration::from_secs(60); + while child.try_wait().unwrap().is_none() { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("{what}: the child did not exit before its deadline"); + } + std::thread::yield_now(); + } + // The child has exited already: the cached status returns at once and the + // pipes reach EOF, so collecting the output cannot wait on a live child. + child.wait_with_output().unwrap() +} + #[test] #[cfg(any(target_os = "linux", target_os = "macos"))] fn another_process_admission_precedes_the_single_sqlite_begin() { @@ -674,11 +694,11 @@ fn three_processes_open_and_reserve_concurrently() { for mut stream in sockets { stream.write_all(b"G").unwrap(); } - for child in children { - let output = child.wait_with_output().unwrap(); + for (id, child) in children.into_iter().enumerate() { + let output = wait_bounded(child, &format!("opener-{id}")); assert!( output.status.success(), - "{}", + "opener-{id}: {}", String::from_utf8_lossy(&output.stderr) ); } diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index 4dba5e7088..edc3136778 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -142,6 +142,26 @@ fn open_concurrent_child() { service.close().unwrap(); } +/// Waits for a spawned test child to exit, bounded, then returns its output. +/// A lane deadlock has to fail this test instead of hanging it, so the wait +/// carries the same kind of deadline as the barrier accept above; the 60 s +/// ceiling also stays under the per-invocation test limit. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn wait_bounded(mut child: std::process::Child, what: &str) -> std::process::Output { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while child.try_wait().unwrap().is_none() { + if std::time::Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("{what}: the child did not exit before its deadline"); + } + std::thread::yield_now(); + } + // The child has exited already: the cached status returns at once and the + // pipes reach EOF, so collecting the output cannot wait on a live child. + child.wait_with_output().unwrap() +} + /// Three processes open the same Store directly and each commits twenty times /// at once: opening waits on the directory lane instead of failing, and no /// Commit is lost or doubled. @@ -201,7 +221,7 @@ fn three_processes_open_direct_and_commit_concurrently() { } let mut failures = Vec::new(); for (id, child) in children.into_iter().enumerate() { - let output = child.wait_with_output().unwrap(); + let output = wait_bounded(child, &format!("agent-{id}")); if !output.status.success() { failures.push(format!( "agent-{id}: {}", diff --git a/engine/crates/layerfs-store-service/tests/support/installed_store.rs b/engine/crates/layerfs-store-service/tests/support/installed_store.rs index e609c8258e..126a5235ab 100644 --- a/engine/crates/layerfs-store-service/tests/support/installed_store.rs +++ b/engine/crates/layerfs-store-service/tests/support/installed_store.rs @@ -16,6 +16,18 @@ use std::{ pub const BINDING: &[u8] = b"daemon-installed-store"; pub const CURSOR: [u8; 32] = [43; 32]; +/// Creates a Store's own directory owner-only (`0700`): the kernel guard +/// refuses a Store whose directory grants any group/other bit (#663), and the +/// daemon installs every Store that way. +fn private_directory(path: &std::path::Path) { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path).unwrap(); + } + #[cfg(not(unix))] + std::fs::create_dir(path).unwrap(); +} pub struct Fixture { pub directory: PathBuf, pub config: PersistenceConfig, @@ -45,7 +57,7 @@ impl Fixture { ) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - std::fs::create_dir(&directory).unwrap(); + private_directory(&directory); let source = directory.join("source"); std::fs::create_dir(&source).unwrap(); let count = build(&source); @@ -116,7 +128,7 @@ impl Fixture { pub fn cloned(label: &str, store: &std::path::Path, count: usize) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - std::fs::create_dir(&directory).unwrap(); + private_directory(&directory); let installed = directory.join("store.sqlite"); std::fs::copy(store, &installed).unwrap(); Self { From 9030b8ef312290cc302d6c4cdb76591c274b1e57 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:39:55 +0000 Subject: [PATCH 11/19] fix(persistence): hold a seal's lane across its sidecar check Review of #647, smaller item: pre-existing on Linux and now reachable on macOS because this branch moved a seal onto the directory lane there. Session::seal released the directory writer lock after the checkpoint and the close, but before checking that no -wal, -shm or -journal sidecar remains. An opener waiting on the same lane was admitted in that gap, created -wal for its own writable session, and the check then answered Refused for a Store that had sealed correctly. The release now follows the sidecar check, so an opener that takes the lane is admitted only after the sealed Store is known to have no sidecar, and the check can only observe another process's sidecar (or a remaining file), never one an admitted opener just made. Nothing else about the seal changes: a found sidecar still answers Refused, that early return drops the retained State owner, which closes the descriptor and releases the lock, and the checkpoint, the close, the macOS persistent-WAL clear and the unused-extent release keep their order. The Seal flows in lifecycle.md and seal-and-install.md now show the check before the release. Tests: the seal's own paths on Linux - serverless_store 5 passed (including seal_refuses_retained_provider_without_effect and disposable_create_publish_seal_and_reopen_all_layouts), disposable_startup_sync 1, heads 1, snapshot 1, concurrent_first_installs 1, sqlite_lock_custody 2, idle_checkpoint 4, writer_admission 13; layerfs-store-service kernel_guard 4, commit_service_busy 3, commit_pipeline 3 (each fixture seals a Store); layerfs-api store_open_cache 2; clippy -D warnings --tests for layerfs-persistence, layerfs-store-service and layerfs-daemon. NOT_RUN: every macOS-only path here (seal_allocation, the macOS lane and sync_volume_np coverage) needs a Darwin host. Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the parent 523f7fda1 (160034) and at this commit's tree (160034); the only production file here is lifecycle/seal.rs, which counts 79 production lines before and after (it gains 4 comment lines and moves existing code), so the total is unchanged. --- .../engine/project-init/seal-and-install.md | 3 ++- docs/architecture/engine/store/lifecycle.md | 3 ++- .../src/backend/sqlite/lifecycle/seal.rs | 10 +++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/architecture/engine/project-init/seal-and-install.md b/docs/architecture/engine/project-init/seal-and-install.md index 11ee9a3c57..3b27013a44 100644 --- a/docs/architecture/engine/project-init/seal-and-install.md +++ b/docs/architecture/engine/project-init/seal-and-install.md @@ -27,8 +27,9 @@ live Store instead, and what each route leaves when its outcome is unknown. take the cross-process writer hold (Linux and macOS) macOS: capture file identity; disable persistent WAL PRAGMA wal_checkpoint(TRUNCATE) ------- busy != 0 --> Busy - close connection; release writer hold + close connection -wal, -shm, -journal must be absent --- else Refused + release writer hold macOS: release unused allocation past the logical end --> SealedStore { path, profile, sqlite_version, bytes } ``` diff --git a/docs/architecture/engine/store/lifecycle.md b/docs/architecture/engine/store/lifecycle.md index 4006822384..2a6ffe168e 100644 --- a/docs/architecture/engine/store/lifecycle.md +++ b/docs/architecture/engine/store/lifecycle.md @@ -111,8 +111,9 @@ SqliteHandles::seal(self) take the directory writer lock (Linux and macOS) macOS: capture file identity; clear SQLITE_FCNTL_PERSIST_WAL PRAGMA wal_checkpoint(TRUNCATE) -- not complete -----------> Busy - close the connection, then release the writer lock + close the connection a -wal, -shm or -journal sidecar left? --------------------> Refused + release the writer lock macOS: release unused extents past EOF (seal_allocation) -> SealedStore { path, profile, sqlite_version, bytes } ``` diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs index bb73427003..e7259fb616 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs @@ -49,9 +49,10 @@ impl Session { .map_err(|(_, error)| rows::error(error))?; // SQLite has checked-closed before advisory ownership is released. // Earlier error exits drop State.connection before its retained owner. - if let Some(writer) = state.writer_hold { - writer.release().map_err(|_| PersistenceError::Uncertain)?; - } + // The sidecar check below runs while the lane is still held: an opener + // waiting on it takes the same lane before it may create `-wal`, so + // releasing first would let one land between the close and this check + // and turn a sealed Store into a refused one. for suffix in ["-wal", "-shm", "-journal"] { let mut sidecar = self.path.as_os_str().to_os_string(); sidecar.push(suffix); @@ -69,6 +70,9 @@ impl Session { } } } + if let Some(writer) = state.writer_hold { + writer.release().map_err(|_| PersistenceError::Uncertain)?; + } #[cfg(target_os = "macos")] let bytes = super::seal_allocation::release(&self.path, identity)?; #[cfg(not(target_os = "macos"))] From 59b143fda6f61af3cbbefc85dd06f3d53aaf82f9 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:40:08 +0000 Subject: [PATCH 12/19] docs(engine, api): name the export-volume flush and the Busy open Review of #647, item 4 and the Linux names in the smaller list. Three engine pages still described the default pre-Commit flush as `sync(2)`: daemon/control-calls.md's Commit row, kernel-routes/coherence.md's writeback box and kernel-routes/nfs.md's Mount box. They now say what the branch does: with no LAYERFS_NFS_FLUSH helper, macOS flushes only the export directory's volume with `sync_volume_np(, SYNC_VOLUME_WAIT)` and falls back to `sync(2)` if that call fails, and Linux keeps `sync(2)`. The daemon's own module doc in control/export.rs said "`sync(2)` by default" the same way and is corrected with them. The Busy open code was missing from the two call-level tables a reader checks first. flows/call.md row 9a now names the lane a writable open takes (Linux and macOS) and its failure row adds `Busy` for an open that meets SQLite's busy lock before any effect; api/call-table/errors.md adds that case to the `Busy` catalog row (with store_service.rs as its source) and to the uncertain-outcome table, where a Busy open is a known no-effect refusal like the other Busy cases. Tests: none run - documentation and a doc comment only. `cargo +1.85.1 --locked fmt --check` unaffected; the pages were checked against control/export.rs `flush_default`/`flush_export`, store_service.rs `open_refusal` and control/crates/layerfs-store/src/service.rs `open`. Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the parent 9030b8ef3 (160034) and at this commit's tree (160034); the only production file touched is control/export.rs, and only its module doc comment changes, so no production line moves. --- docs/architecture/api/call-table/errors.md | 4 ++-- docs/architecture/engine/daemon/control-calls.md | 2 +- docs/architecture/engine/kernel-routes/coherence.md | 13 +++++++------ docs/architecture/engine/kernel-routes/nfs.md | 4 +++- docs/architecture/flows/call.md | 4 ++-- engine/crates/layerfs-daemon/src/control/export.rs | 5 +++-- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/architecture/api/call-table/errors.md b/docs/architecture/api/call-table/errors.md index f68f9f5fd2..8a0f8ef59d 100644 --- a/docs/architecture/api/call-table/errors.md +++ b/docs/architecture/api/call-table/errors.md @@ -46,7 +46,7 @@ Store engine's refusals that pass through Store calls. | `NoSuchPath`, `IsDirectory`, `NotDirectory`, `NotFile`, `NotEmpty`, `Binary` | path refusals of the `file` calls | `write/mutation.rs`, [`store-service read/`](../../../../engine/crates/layerfs-store-service/src/read) | | `Exists`, `NameTaken`, `ReservedName` | a name is taken or reserved; a database already connected | [`postgresql/connect.rs`](../../../../control/crates/layerfs-store/src/postgresql/connect.rs), [`history/branch.rs`](../../../../engine/crates/layerfs-store-service/src/history/branch.rs) | | `InUse` | held: running Sandboxes, processes with the Store open, Stores on a machine | [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs), `inventory.rs` | -| `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), or no daemon connection came free | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | +| `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), no daemon connection came free, or a Store's writable open met SQLite's busy lock before any effect (the open wrote nothing; the caller may repeat the call) | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) | | `Unresolved` | an earlier operation on the entity ended with its outcome unknown and its process is gone | `layerfs-home` `op.rs` | | `Conflict`, `HeadMoved` | a merge or Commit conflict, or the Branch head moved; for `backup.restore`, the Store changed since its last backup | [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), `history/lookup.rs` | | `Dirty`, `MountsActive` | unmount with uncommitted changes; `sandbox.close` while Workspaces are mounted | `daemon/values.rs`, [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) | @@ -94,7 +94,7 @@ The 2.0.0 manual keeps a reader's version of this table in | `Unresolved` | an earlier `Unknown` on this entity stands; this call did nothing | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs) `blocks` | | daemon `Unknown` | a Workspace operation's outcome is uncertain at the daemon | [`control_types.rs`](../../../../engine/crates/layerfs-bridge/src/control_types.rs) `ControlCode::Unknown` | | `Failed` after a daemon exchange with `attempted: true` | uncertain, though the code is `Failed`: a lost daemon reply is not mapped to `Unknown` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation` | -| `Busy` | known: nothing was attempted | `layerfs-home` `op.rs`, `daemon/pool.rs` | +| `Busy` | known: nothing was attempted, including a `Busy` Store open (the open changed nothing) | `layerfs-home` `op.rs`, `daemon/pool.rs`, [`service.rs`](../../../../control/crates/layerfs-store/src/service.rs) | | `Unavailable`, `Denied`, `TooLarge` from a worker | known: the request was not sent | `worker/client.rs` (`Unreached`) | Intent sources: [`sqlite/family.rs`](../../../../control/crates/layerfs-store/src/sqlite/family.rs), diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index 44eed0c3e0..af2405917f 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -87,7 +87,7 @@ differs. | Step | NFS route | Source | | --- | --- | --- | | Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the host client mounts it with the mount helper; Ready only after the helper reports the mount. An NFS receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| Commit | Flush first (flush helper, or `sync(2)`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | +| Commit | Flush first (flush helper, or the export volume's own flush: `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | | Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | ForceUnmount | Force helper once, then the same teardown | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | Helper run | One attempt, at most `HELPER_WAIT` (60 s), then killed and reported | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | diff --git a/docs/architecture/engine/kernel-routes/coherence.md b/docs/architecture/engine/kernel-routes/coherence.md index 930394a455..070966cf25 100644 --- a/docs/architecture/engine/kernel-routes/coherence.md +++ b/docs/architecture/engine/kernel-routes/coherence.md @@ -105,12 +105,13 @@ holds. ```text FUSE NFS - write(2) -> WRITE request write(2) -> client page cache - -> published by its owner job -> WRITE (FILE_SYNC answer: - -> reply -> write(2) returns each window published) - msync / mapped store -> WRITE Commit: flush helper (sync(2) by - with FUSE_WRITE_CACHE (cached=true) default) first, then - FLUSH/FSYNC/FSYNCDIR -> ENOSYS once close_files, then Commit + write(2) -> WRITE request write(2) -> client page cache + -> published by its owner job-> WRITE (FILE_SYNC answer: + -> reply -> write(2) returns each window published) + msync / mapped store -> WRITE Commit: flush helper, else the export + with FUSE_WRITE_CACHE (cached=true) volume's own flush (sync_volume_np + FLUSH/FSYNC/FSYNCDIR -> ENOSYS once on macOS, sync(2) elsewhere), then + close_files, then Commit ``` | Rule | Source | diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 7b64bf49ed..9fdbcc0a0f 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -207,7 +207,9 @@ it starts with `._` and has at least one more byte. bind 127.0.0.1:0 -> Server::start(...) port P helper: mount_nfs -o vers=3,tcp,port=P,mountport=P,... 127.0.0.1:/ client --MOUNT MNT "/"--> root handle + AUTH_UNIX - daemon Commit: flush helper (sync(2) by default), close_files, Commit + daemon Commit: flush helper (else the export volume's own flush: + sync_volume_np on macOS, sync(2) elsewhere), close_files, + Commit daemon Unmount: unmount helper (busy is reversible), Server::stop, revoke the engine mount, Close ``` diff --git a/docs/architecture/flows/call.md b/docs/architecture/flows/call.md index 26c78fe943..54f9ad0f49 100644 --- a/docs/architecture/flows/call.md +++ b/docs/architecture/flows/call.md @@ -64,7 +64,7 @@ Sandbox daemon; see [A Workspace session](workspace.md). Terms are in the | 6 | The family function translates the call into one Store method (`branch.fork`, `branch.diff_commits`, `fs.read`, …). | [Control: families](../control/management/families.md), [SDK: branch](../api/sdk/branch/README.md) | [`calls.rs`](../../../control/crates/layerfs-store/src/calls.rs) | | 7 | `resolve::target` reads the Store and project records; `store_call` checks the record holds no blocking intent and takes this process's lease on the Store under the Store's lock. | [Routing table: by Store kind](../control/routing/routing-table.md#by-store-kind) | [`resolve.rs`](../../../control/crates/layerfs-store/src/resolve.rs), [`call.rs`](../../../control/crates/layerfs-store/src/call.rs), [`lease.rs`](../../../control/crates/layerfs-store/src/lease.rs) | | 8 | `place::plan` decides where the call runs, once, from the record: a PostgreSQL Store runs here; a sqlite Store runs here when its machine's platform runs calls locally on this host, else in that machine's worker. | [Routing table](../control/routing/routing-table.md) | [`place.rs`](../../../control/crates/layerfs-store/src/place.rs) | -| 9a | Local: `open::run` keeps the Store open in this process (closed after 60 s unused) and `service::acquire` opens a `StoreService`; the kernel guard runs before any file is opened. `service::call` strips the routing fields and calls `methods::call` once. | [Store calls: service](../engine/store-calls/service.md#open-and-bind), [SDK facade: dispatch](../engine/store-calls/sdk-facade.md#dispatch) | [`open.rs`](../../../control/crates/layerfs-store/src/open.rs), [`service.rs`](../../../control/crates/layerfs-store/src/service.rs) | +| 9a | Local: `open::run` keeps the Store open in this process (closed after 60 s unused) and `service::acquire` opens a `StoreService`; the kernel guard runs before any file is opened, and a writable open takes the Store directory's writer lane (Linux and macOS) so two openers queue instead of meeting SQLite's busy lock. `service::call` strips the routing fields and calls `methods::call` once. | [Store calls: service](../engine/store-calls/service.md#open-and-bind), [SDK facade: dispatch](../engine/store-calls/sdk-facade.md#dispatch) | [`open.rs`](../../../control/crates/layerfs-store/src/open.rs), [`service.rs`](../../../control/crates/layerfs-store/src/service.rs) | | 9b | Worker: one new connection to the loopback address the machine's `worker.json` records. The secret goes first, alone; the worker answers with its protocol and bundle; a worker of another build is sent nothing. The request goes once. The worker holds its own lease, sets the Store path under its root and runs step 9a. | [Network: workers](../control/network/workers.md#one-request-per-connection), [Routing: workers](../control/routing/workers.md) | [`worker/client.rs`](../../../control/crates/layerfs-machine/src/worker/client.rs), [`worker_server.rs`](../../../control/crates/layerfs-control/src/worker_server.rs) | | 10 | The engine runs the method. Reads use the published read session and never take the lane. A write waits for the session's admission turn, then the Linux `flock` on the Store's directory, then `BEGIN IMMEDIATE` (PostgreSQL: the `store_lane` row `FOR UPDATE`). A Commit holds the publish lane across its head check and publish. | [Store: one writer](../engine/store/backends.md#one-writer), [Store: the publish lane](../engine/store/history-publication.md#the-publish-lane) | [`methods/dispatch.rs`](../../../engine/crates/layerfs-store-service/src/methods/dispatch.rs), [`backend/sqlite/lane/lane.rs`](../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs) | | 11 | The answer's keys become snake_case. Events (progress, command output) went to the call's event stream as they came. The CLI closes the Stores the call opened before it exits and maps a refusal to exit code 125. | [Call table: keys](../api/call-table/table.md#keys) | [`calls/dispatch.rs`](../../../api/crates/layerfs-api/src/calls/dispatch.rs), [`run.rs`](../../../api/crates/layerfs-cli/src/run.rs) | @@ -100,7 +100,7 @@ Timing trees themselves are described in [Core: Telemetry](../core/telemetry/REA | 9b | The worker is of another build | refusal naming how to replace it; nothing was sent | | 9b | Request larger than a frame | `TooLarge`; nothing was sent | | 9b | Connection lost after the request was sent | `Unknown`: the call may have run; it is not resent | -| 9a | Kernel guard refuses the Store's filesystem; Store file missing | `Unsupported`; `NoStore` | +| 9a | Kernel guard refuses the Store's filesystem; Store file missing; a writable open meets SQLite's busy lock before any effect (another process holds the Store outside the lane) | `Unsupported`; `NoStore`; `Busy` | | 10 | SQLite busy (no cross-process wait on macOS), PostgreSQL `lock_timeout` | `Busy` | | 10 | The engine cannot tell whether a write happened | `Unknown`; a file write keeps its private Workspace | | 10 | Any other engine refusal | its stable code (`NoSuchBranch`, `HeadMoved`, `Conflict`, …), see [Call table: errors](../api/call-table/errors.md) | diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 5c84dbf1f7..2bb3e391a4 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -10,8 +10,9 @@ //! - Ready carries the same receipt record; an NFS receipt is recognized by //! `abi_major == 0`, and its `mount_id` is the export's TCP port. //! - Commit first has the client write back what it buffered (the flush -//! helper, `sync(2)` by default) and closes the export's implicit file -//! descriptors, then runs the unchanged Commit. A Commit that installed +//! helper, else the export volume's own flush: `sync_volume_np(..., +//! SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere) and closes the +//! export's implicit file descriptors, then runs the unchanged Commit. A Commit that installed //! another root closes the export's descriptors again; the client //! revalidates attributes on its own schedule (close-to-open, `actimeo`). //! - Unmount asks the helper to unmount (a busy answer is reversible and the From 2d4bac02518edf3a37ee85b7b3a5218b3138e118 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:40:23 +0000 Subject: [PATCH 13/19] test(nfs): stop claiming the mounter test observes the flush fallback Review of #647, smaller item: default_flush_syncs_volume_and_falls_back_ when_directory_is_missing wrote to the export through its own NFS RPC client, so the host mount client held no writeback for either flush to push and no part of the test could see which call ran. It only ever observed that a Commit succeeds in both cases, which is what its title denied. The test is renamed to default_flush_commits_with_the_export_volume_and_ without_it and says what it checks: a Commit with the live export directory (the default `sync_volume_np` succeeding) and one after the directory is removed (that call answering ENOENT, so `flush_default` falls back to `sync(2)`) both run the Commit and succeed instead of failing on the flush. The comment also names where the flush itself is covered instead: `layerfs- bridge`'s process_identity test for `sync_volume_wait`'s Ok/Err contract, and the single fallback branch in export.rs `flush_default`. No assertion changes. Tests: NOT_RUN on this host - the file, like the rest of the native macOS client coverage, is `#![cfg(target_os = "macos")]`, so nothing in it compiles on a Linux box and the rename is verified only by review. Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the parent 59b143fda (160034) and at this commit's tree (160034); only an external macOS-only test changes. --- engine/crates/layerfs-nfs/tests/mounter.rs | 25 ++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/engine/crates/layerfs-nfs/tests/mounter.rs b/engine/crates/layerfs-nfs/tests/mounter.rs index 55407db40b..775e78c153 100644 --- a/engine/crates/layerfs-nfs/tests/mounter.rs +++ b/engine/crates/layerfs-nfs/tests/mounter.rs @@ -1,6 +1,7 @@ //! The native macOS client must not send atime updates the export ignores, -//! and the default export flush syncs the export directory volume (falling -//! back to `sync(2)` if the directory is missing). +//! and a Commit with the default export flush (the export volume's own +//! `sync_volume_np` on macOS, `sync(2)` elsewhere) never fails on the flush +//! itself, even when the export's directory is gone. #![cfg(target_os = "macos")] #[allow(dead_code)] #[path = "support/client.rs"] @@ -35,7 +36,17 @@ fn host_mount_disables_access_time_updates() { } #[test] -fn default_flush_syncs_volume_and_falls_back_when_directory_is_missing() { +fn default_flush_commits_with_the_export_volume_and_without_it() { + // What this observes, and what it cannot: the Workspace is driven through + // this test's own NFS RPC client, so the host's mount client holds no + // writeback for either flush to push and neither syscall leaves a trace + // here. The two Commits below therefore check the contract that is + // observable - a flush that succeeds (`sync_volume_np` of the live + // directory) and one that fails (`ENOENT` after the directory is removed, + // which falls back to `sync(2)`) both leave the Commit to run and + // succeed. The flush calls themselves are covered where they live: + // `sync_volume_wait`'s `Ok`/error by `layerfs-bridge`'s + // `process_identity` test, and the fallback branch by `flush_default`. let base = std::env::temp_dir().join(format!("layerfs-nfs-mounter-{}", std::process::id())); let source = base.join("src"); let work = base.join("work"); @@ -84,7 +95,8 @@ fn default_flush_syncs_volume_and_falls_back_when_directory_is_missing() { let mut nfs = Client::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, port)), 501, 20); let root = nfs.mnt("/"); - // 1. Existing directory: `sync_volume_wait` succeeds (`Ok(())`). + // 1. Live export directory: the default flush is the volume's own + // `sync_volume_np`, which succeeds; the Commit runs. let f1 = nfs.touch(&root, b"one.txt", 0o644).unwrap(); nfs.write_all(&f1.handle, 0, b"first\n", 6); assert!(Path::new(&ready.directory).is_dir()); @@ -101,8 +113,9 @@ fn default_flush_syncs_volume_and_falls_back_when_directory_is_missing() { other => panic!("{other:?}"), } - // 2. Missing directory: `sync_volume_wait` returns `Err(NotFound)` and - // `flush_export` falls back to `sync(2)` rather than failing or skipping. + // 2. Missing export directory: `sync_volume_np` answers `ENOENT` and + // `flush_default` falls back to `sync(2)` instead of failing or skipping + // the client writeback; the Commit still runs and succeeds. let f2 = nfs.touch(&root, b"two.txt", 0o644).unwrap(); nfs.write_all(&f2.handle, 0, b"second\n", 7); fs::remove_dir_all(&ready.directory).unwrap(); From 180ef5930d36461d071d67b83babeeb604103b46 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:48:07 +0000 Subject: [PATCH 14/19] docs(flows): a macOS writer waits on the directory lane too The call flow's step-10 failure row still said "SQLite busy (no cross-process wait on macOS)". This branch is what removed that exception: a writable open and every write transaction on macOS take the Store directory's `flock`, so the busy lock is left for a writer that bypasses the lane (a raw SQLite connection, or any process that does not go through Writer::acquire). The row now says that, matching engine/store/backends.md and engine/store/README.md, which the branch already rewrote. Tests: none run - a table cell only. Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the parent 2d4bac025 (160034) and at this commit's tree (160034); documentation only. --- docs/architecture/flows/call.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/flows/call.md b/docs/architecture/flows/call.md index 54f9ad0f49..ed911e4cbf 100644 --- a/docs/architecture/flows/call.md +++ b/docs/architecture/flows/call.md @@ -101,7 +101,7 @@ Timing trees themselves are described in [Core: Telemetry](../core/telemetry/REA | 9b | Request larger than a frame | `TooLarge`; nothing was sent | | 9b | Connection lost after the request was sent | `Unknown`: the call may have run; it is not resent | | 9a | Kernel guard refuses the Store's filesystem; Store file missing; a writable open meets SQLite's busy lock before any effect (another process holds the Store outside the lane) | `Unsupported`; `NoStore`; `Busy` | -| 10 | SQLite busy (no cross-process wait on macOS), PostgreSQL `lock_timeout` | `Busy` | +| 10 | SQLite busy (a writer that bypasses the directory lane), PostgreSQL `lock_timeout` | `Busy` | | 10 | The engine cannot tell whether a write happened | `Unknown`; a file write keeps its private Workspace | | 10 | Any other engine refusal | its stable code (`NoSuchBranch`, `HeadMoved`, `Conflict`, …), see [Call table: errors](../api/call-table/errors.md) | From 30c0e56723f2bc2370fc3a25a7947605fcd69d7d Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:05:11 +0000 Subject: [PATCH 15/19] test: keep a fixture's Store in its own private directory Review item 2, corrected. 523f7fda1 made the whole installed::Fixture directory 0700 so the Store inside it would satisfy #663's guard. That directory is also where the daemon harness puts its mount points and where daemon tests run `bash` as another uid: in the mount lane, 24 of 45 binaries then failed with EACCES from `bash_in`'s chdir into the mount, because the parent no longer let that uid traverse. The fixture now keeps everything it owns in `directory` at the process umask (0755 under the CI umask) - kernel source, overlay template, mount points - and puts the sealed and installed Store files in `directory/store` at 0700. The store-service fixture exposes it as `Fixture::store_directory()`, and kernel_guard's corrupt copy moves there: #663's guard checks the Store file's own parent, which is now the private directory, while every path a test reaches as another uid keeps its old mode. The daemon's own fixture gets the same split as a field. Both three-process cases now own their children as one group instead of waiting on each in turn: wait_bounded becomes Children::wait_all, with one deadline for the whole set (60 s, under the per-invocation ceiling), a Drop that stops any child still running, and a failure that names the child which hung together with what the group wrote to stderr. The barrier handshake is bounded too: the parent's accepted streams and the child's socket get read and write timeouts, so a child that connects and never writes `R` fails the case instead of blocking the accept loop. With this, the remaining review items are in the same tree: api/call-table's Busy open also reaches the 2.0 manual (versioned/2.0.0/calls.md), engine AGENTS.md states that macOS holds the parent-directory lane over short writer jobs and writable opens so only a bypassing writer keeps SQLite's Busy, the coherence box is re-aligned, and the mounter test's comment no longer implies the fallback branch is observed by a test of its own. Tests: layerfs-persistence writer_admission 13, heads 1, idle_checkpoint 4, serverless_store 5, snapshot 1, concurrent_first_installs 1, sqlite_lock_custody 2, disposable_startup_sync 1; layerfs-store-service kernel_guard 4, commit_service_busy 3, commit_pipeline 3, commit_reclaim 2, write_refusals 3, sdk_method_golden 1; layerfs-api store_open_cache 2 - 46 passed, 0 failed. cargo +1.85.1 clippy -D warnings --tests for layerfs-persistence, layerfs-store-service and layerfs-daemon; fmt --check clean for every file here. Mount lane, selected: the 26 binaries that failed at 523f7fda1 pass 26/26 with this fixture layout (/data/work/ci-logs/pr647-r2-r, 05:44). Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py at the parent 180ef5930 (160034) and at this commit's tree (160034); this commit touches external tests, docs and one test-support module only. --- .../engine/kernel-routes/coherence.md | 14 +- docs/versioned/2.0.0/calls.md | 2 +- engine/AGENTS.md | 7 +- .../tests/support/installed_store.rs | 19 ++- engine/crates/layerfs-nfs/tests/mounter.rs | 4 +- .../tests/writer_admission.rs | 122 +++++++++++++----- .../tests/kernel_guard.rs | 113 +++++++++++----- .../tests/support/installed_store.rs | 32 +++-- 8 files changed, 221 insertions(+), 92 deletions(-) diff --git a/docs/architecture/engine/kernel-routes/coherence.md b/docs/architecture/engine/kernel-routes/coherence.md index 070966cf25..1c97633800 100644 --- a/docs/architecture/engine/kernel-routes/coherence.md +++ b/docs/architecture/engine/kernel-routes/coherence.md @@ -105,13 +105,13 @@ holds. ```text FUSE NFS - write(2) -> WRITE request write(2) -> client page cache - -> published by its owner job-> WRITE (FILE_SYNC answer: - -> reply -> write(2) returns each window published) - msync / mapped store -> WRITE Commit: flush helper, else the export - with FUSE_WRITE_CACHE (cached=true) volume's own flush (sync_volume_np - FLUSH/FSYNC/FSYNCDIR -> ENOSYS once on macOS, sync(2) elsewhere), then - close_files, then Commit + write(2) -> WRITE request write(2) -> client page cache + -> published by its owner job -> WRITE (FILE_SYNC answer: + -> reply -> write(2) returns each window published) + msync / mapped store -> WRITE Commit: flush helper, else the export + with FUSE_WRITE_CACHE (cached=true) volume's own flush (sync_volume_np + FLUSH/FSYNC/FSYNCDIR -> ENOSYS once on macOS, sync(2) elsewhere), then + close_files, then Commit ``` | Rule | Source | diff --git a/docs/versioned/2.0.0/calls.md b/docs/versioned/2.0.0/calls.md index 1fe4379dc7..687d493c5e 100644 --- a/docs/versioned/2.0.0/calls.md +++ b/docs/versioned/2.0.0/calls.md @@ -562,7 +562,7 @@ Codes seen in the 2.0 library include: | `NoSuchPath`, `NoSuchSandbox`, `NoSuchProcess`, … | 1.1.0's codes for paths, Sandboxes and processes | | `Exists` | the name is taken, the same database or bucket is already connected (under any name), or a bucket object was exported from another home | | `InUse` | held: running Sandboxes, processes with the Store open, Stores on a machine; a layout 2 home with any of these, or with a move, removal or create still running, which is not converted yet ([the home](home.md#a-layout-2-home)) | -| `Busy` | an operation runs on the entity: a Store being pushed, pulled or removed, a machine being made or removed, a Sandbox being created or closed; the message names it (`op` id and phase) and its process (#536 §7.1); `detail` is `{entity: {kind, name}, phase, op}`. Also: another project is being added to the same Store | +| `Busy` | an operation runs on the entity: a Store being pushed, pulled or removed, a machine being made or removed, a Sandbox being created or closed; the message names it (`op` id and phase) and its process (#536 §7.1); `detail` is `{entity: {kind, name}, phase, op}`. Also: another project is being added to the same Store; a Store's writable open met SQLite's busy lock before any effect (the open wrote nothing, so the caller may repeat the call) | | `Unresolved` | an earlier operation on the entity ended with its outcome not known; its process is gone and its `op` stays. The message names the call that shows it (`store status`, `machine status`, `sandbox get`) and the one that clears it (`store remove`, `machine remove`, `sandbox close`); `detail` as for `Busy` | | `Unsupported` | not possible here, or not built (see [limitations](limitations.md)); a missing tool is listed in `detail.missing` | | `Unavailable`, `Denied` | a server, bucket, worker or host cannot be reached, or refused the login or secret | diff --git a/engine/AGENTS.md b/engine/AGENTS.md index 8bbe4178a6..38db1a5237 100644 --- a/engine/AGENTS.md +++ b/engine/AGENTS.md @@ -454,8 +454,11 @@ fixtures, examples and tools do not contribute to production LOC. separate main/WAL/SHM descriptor for advisory ownership: closing a main-file descriptor can cancel SQLite's own process POSIX locks. This admission does not gate whole Saves or Commits or replay - a failed SQL attempt. macOS has Session admission only; separate macOS writers - and uncoordinated raw SQLite writers retain exact Busy refusal. Follow the + a failed SQL attempt. macOS holds the same parent-directory lane as Linux, + over the same short writer jobs and its writable opens (#647), so separate + macOS writers queue instead of meeting SQLite's Busy; only a writer that + bypasses the lane (an uncoordinated raw SQLite process) retains the exact + Busy refusal. Follow the [integration contract](../core/docs/issues/303/06-cluster-one-integration.md) and its [deepest-file plan](../core/docs/issues/307/SERVERLESS-STORE-PLAN-20261007.md); do not extend the host-mediated SDK runtime, Bridge data codec or daemon upstream. diff --git a/engine/crates/layerfs-daemon/tests/support/installed_store.rs b/engine/crates/layerfs-daemon/tests/support/installed_store.rs index 06988a0087..166b02dc24 100644 --- a/engine/crates/layerfs-daemon/tests/support/installed_store.rs +++ b/engine/crates/layerfs-daemon/tests/support/installed_store.rs @@ -16,9 +16,9 @@ use std::{ pub const BINDING: &[u8] = b"daemon-installed-store"; pub const CURSOR: [u8; 32] = [43; 32]; -/// Creates a Store's own directory owner-only (`0700`): the kernel guard -/// refuses a Store whose directory grants any group/other bit (#663), and the -/// daemon installs every Store that way. +/// Creates one directory owner-only (`0700`) as any directory holding a Store +/// file must be: the kernel guard refuses one that grants a group or other bit +/// (#663), and the daemon installs every Store that way. fn private_directory(path: &std::path::Path) { #[cfg(unix)] { @@ -28,7 +28,12 @@ fn private_directory(path: &std::path::Path) { #[cfg(not(unix))] std::fs::create_dir(path).unwrap(); } +/// A sealed Store installed as a consumer sees it. pub struct Fixture { + /// Everything this fixture owns. It stays traversable (`0755`): mounted + /// Workspaces, the overlay template and the kernel source live here, and + /// tests run commands as another uid inside them. Only `store/` below is + /// private. pub directory: PathBuf, pub config: PersistenceConfig, pub branch: BranchId, @@ -48,11 +53,13 @@ impl Fixture { pub fn built(label: &str, build: impl FnOnce(&std::path::Path) -> usize) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - private_directory(&directory); + std::fs::create_dir(&directory).unwrap(); + let store_directory = directory.join("store"); + private_directory(&store_directory); let source = directory.join("source"); std::fs::create_dir(&source).unwrap(); let count = build(&source); - let sealed = directory.join("sealed.sqlite"); + let sealed = store_directory.join("sealed.sqlite"); let config = PersistenceConfig::sqlite(&sealed) .with_sqlite_profile(SqlitePersistenceProfile::Disposable) .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); @@ -97,7 +104,7 @@ impl Fixture { .unwrap(); drop(storage); let sealed = handles.seal().unwrap(); - let installed = directory.join("store.sqlite"); + let installed = store_directory.join("store.sqlite"); std::fs::copy(sealed.path, &installed).unwrap(); // No source or sealed original remains available to the consumer. std::fs::remove_dir_all(source).unwrap(); diff --git a/engine/crates/layerfs-nfs/tests/mounter.rs b/engine/crates/layerfs-nfs/tests/mounter.rs index 775e78c153..2309013b34 100644 --- a/engine/crates/layerfs-nfs/tests/mounter.rs +++ b/engine/crates/layerfs-nfs/tests/mounter.rs @@ -46,7 +46,9 @@ fn default_flush_commits_with_the_export_volume_and_without_it() { // which falls back to `sync(2)`) both leave the Commit to run and // succeed. The flush calls themselves are covered where they live: // `sync_volume_wait`'s `Ok`/error by `layerfs-bridge`'s - // `process_identity` test, and the fallback branch by `flush_default`. + // `process_identity` test. The `ENOENT` to `sync(2)` fallback inside + // `flush_default` has no test of its own, so all this test observes of it + // is its outcome: the Commit still runs. let base = std::env::temp_dir().join(format!("layerfs-nfs-mounter-{}", std::process::id())); let source = base.join("src"); let work = base.join("work"); diff --git a/engine/crates/layerfs-persistence/tests/writer_admission.rs b/engine/crates/layerfs-persistence/tests/writer_admission.rs index 2e9bfd33fc..790c664311 100644 --- a/engine/crates/layerfs-persistence/tests/writer_admission.rs +++ b/engine/crates/layerfs-persistence/tests/writer_admission.rs @@ -202,24 +202,73 @@ impl Drop for Held { } } -/// Waits for a spawned test child to exit, bounded, then returns its output. -/// A lane deadlock has to fail this test instead of hanging it, so the wait -/// carries the same kind of deadline as `Held::release`; the 60 s ceiling also -/// stays under the per-invocation test limit. +/// The test children of one concurrency case, owned as one group: the wait +/// carries a single deadline for the whole set, a child that outlives it or +/// exits unsuccessfully fails the case, and dropping the group while any child +/// still runs stops it. A lane deadlock therefore fails the case instead of +/// hanging it or leaving a live child that holds the Store, and the failure +/// names the child that hung together with what the group wrote to stderr. #[cfg(any(target_os = "linux", target_os = "macos"))] -fn wait_bounded(mut child: Child, what: &str) -> std::process::Output { - let deadline = Instant::now() + Duration::from_secs(60); - while child.try_wait().unwrap().is_none() { - if Instant::now() >= deadline { +struct Children(Vec<(String, Child)>); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Children { + fn new() -> Self { + Self(Vec::new()) + } + fn push(&mut self, what: String, child: Child) { + self.0.push((what, child)); + } + /// Waits for every child, `seconds` for the group as a whole; the 60 s + /// ceiling stays under the per-invocation test limit. + fn wait_all(mut self, seconds: u64) { + let deadline = Instant::now() + Duration::from_secs(seconds); + let mut late = Vec::new(); + for (what, child) in self.0.iter_mut() { + while child.try_wait().unwrap().is_none() { + if Instant::now() >= deadline { + late.push(what.clone()); + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + if !late.is_empty() { + self.stop(); + } + let mut failed = Vec::new(); + for (what, child) in std::mem::take(&mut self.0) { + let output = child.wait_with_output().unwrap(); + if !output.status.success() { + failed.push(format!( + "{what}: {}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr).trim_end() + )); + } + } + assert!( + late.is_empty(), + "the group's deadline passed with {late:?} still running" + ); + assert!(failed.is_empty(), "opener failed:\n{}", failed.join("\n")); + } + /// Stops every child of the group, exited or not. + fn stop(&mut self) { + for (_, child) in self.0.iter_mut() { let _ = child.kill(); + } + for (_, child) in self.0.iter_mut() { let _ = child.wait(); - panic!("{what}: the child did not exit before its deadline"); } - std::thread::yield_now(); } - // The child has exited already: the cached status returns at once and the - // pipes reach EOF, so collecting the output cannot wait on a live child. - child.wait_with_output().unwrap() +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for Children { + fn drop(&mut self) { + self.stop(); + } } #[test] @@ -639,6 +688,8 @@ fn concurrent_open_child() { }; let mut sock = UnixStream::connect(std::env::var_os("LAYERFS_ADMISSION_OPEN_SOCK").unwrap()).unwrap(); + sock.set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); sock.write_all(b"R").unwrap(); let mut go = [0u8; 1]; sock.read_exact(&mut go).unwrap(); @@ -661,32 +712,40 @@ fn three_processes_open_and_reserve_concurrently() { let listener = UnixListener::bind(&sock_path).unwrap(); listener.set_nonblocking(true).unwrap(); let exe = std::env::current_exe().unwrap(); - let children: Vec<_> = (0..3) - .map(|_| { - Command::new(&exe) - .args(["--exact", "concurrent_open_child", "--nocapture"]) - .env("LAYERFS_ADMISSION_OPEN_STORE", &path) - .env("LAYERFS_ADMISSION_OPEN_SOCK", &sock_path) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .unwrap() - }) - .collect(); + let mut children = Children::new(); + for id in 0..3 { + let child = Command::new(&exe) + .args(["--exact", "concurrent_open_child", "--nocapture"]) + .env("LAYERFS_ADMISSION_OPEN_STORE", &path) + .env("LAYERFS_ADMISSION_OPEN_SOCK", &sock_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + children.push(format!("opener-{id}"), child); + } let mut sockets = Vec::new(); let deadline = Instant::now() + Duration::from_secs(5); while sockets.len() < 3 { match listener.accept() { Ok((mut stream, _)) => { stream.set_nonblocking(false).unwrap(); + // The handshake itself is bounded too: a child that connects + // and never writes `R` must fail the case, not wait forever. + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); let mut ready = [0u8; 1]; stream.read_exact(&mut ready).unwrap(); assert_eq!(&ready, b"R"); sockets.push(stream); } Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { - std::thread::yield_now(); + std::thread::sleep(Duration::from_millis(1)); } Err(e) => panic!("barrier accept: {e}"), } @@ -694,14 +753,7 @@ fn three_processes_open_and_reserve_concurrently() { for mut stream in sockets { stream.write_all(b"G").unwrap(); } - for (id, child) in children.into_iter().enumerate() { - let output = wait_bounded(child, &format!("opener-{id}")); - assert!( - output.status.success(), - "opener-{id}: {}", - String::from_utf8_lossy(&output.stderr) - ); - } + children.wait_all(60); let handles = open(&path); let next = handles.storage.reserve(reserve()).unwrap(); assert_eq!(next.first_pack_id, 61); diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index edc3136778..266afca8bb 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -71,7 +71,7 @@ fn open_direct_reports_busy_when_locked_and_failed_when_corrupt() { sql.execute_batch("ROLLBACK").unwrap(); drop(sql); - let corrupt = fixture.directory.join("corrupt.sqlite"); + let corrupt = fixture.store_directory().join("corrupt.sqlite"); std::fs::write(&corrupt, b"not a valid sqlite store").unwrap(); let failed = StoreService::open_direct(&corrupt, installed::BINDING, installed::CURSOR, &template); @@ -97,6 +97,8 @@ fn open_concurrent_child() { std::path::PathBuf::from(std::env::var_os("LAYERFS_OPEN_CONCURRENT_TPL").unwrap()); let mut sock = UnixStream::connect(std::env::var_os("LAYERFS_OPEN_CONCURRENT_SOCK").unwrap()).unwrap(); + sock.set_read_timeout(Some(std::time::Duration::from_secs(30))) + .unwrap(); sock.write_all(b"R").unwrap(); let mut go = [0u8; 1]; sock.read_exact(&mut go).unwrap(); @@ -142,24 +144,77 @@ fn open_concurrent_child() { service.close().unwrap(); } -/// Waits for a spawned test child to exit, bounded, then returns its output. -/// A lane deadlock has to fail this test instead of hanging it, so the wait -/// carries the same kind of deadline as the barrier accept above; the 60 s -/// ceiling also stays under the per-invocation test limit. +/// The test children of one concurrency case, owned as one group: the wait +/// carries a single deadline for the whole set, a child that outlives it or +/// exits unsuccessfully fails the case, and dropping the group while any child +/// still runs stops it. A lane deadlock therefore fails the case instead of +/// hanging it or leaving a live child that holds the Store, and the failure +/// names the child that hung together with what the group wrote to stderr. +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct Children(Vec<(String, std::process::Child)>); + #[cfg(any(target_os = "linux", target_os = "macos"))] -fn wait_bounded(mut child: std::process::Child, what: &str) -> std::process::Output { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); - while child.try_wait().unwrap().is_none() { - if std::time::Instant::now() >= deadline { +impl Children { + fn new() -> Self { + Self(Vec::new()) + } + fn push(&mut self, what: String, child: std::process::Child) { + self.0.push((what, child)); + } + /// Waits for every child, `seconds` for the group as a whole; the 60 s + /// ceiling stays under the per-invocation test limit. + fn wait_all(mut self, seconds: u64) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(seconds); + let mut late = Vec::new(); + for (what, child) in self.0.iter_mut() { + while child.try_wait().unwrap().is_none() { + if std::time::Instant::now() >= deadline { + late.push(what.clone()); + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + if !late.is_empty() { + self.stop(); + } + let mut failed = Vec::new(); + for (what, child) in std::mem::take(&mut self.0) { + let output = child.wait_with_output().unwrap(); + if !output.status.success() { + failed.push(format!( + "{what}: {}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr).trim_end() + )); + } + } + assert!( + late.is_empty(), + "the group's deadline passed with {late:?} still running" + ); + assert!( + failed.is_empty(), + "concurrent open_direct + 20 commits failed:\n{}", + failed.join("\n") + ); + } + /// Stops every child of the group, exited or not. + fn stop(&mut self) { + for (_, child) in self.0.iter_mut() { let _ = child.kill(); + } + for (_, child) in self.0.iter_mut() { let _ = child.wait(); - panic!("{what}: the child did not exit before its deadline"); } - std::thread::yield_now(); } - // The child has exited already: the cached status returns at once and the - // pipes reach EOF, so collecting the output cannot wait on a live child. - child.wait_with_output().unwrap() +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for Children { + fn drop(&mut self) { + self.stop(); + } } /// Three processes open the same Store directly and each commits twenty times @@ -181,7 +236,7 @@ fn three_processes_open_direct_and_commit_concurrently() { listener.set_nonblocking(true).unwrap(); let exe = std::env::current_exe().unwrap(); - let mut children = Vec::new(); + let mut children = Children::new(); for id in 0..3 { let overlay_dir = fixture.directory.join(format!("overlay-{id}")); std::fs::create_dir(&overlay_dir).unwrap(); @@ -197,7 +252,7 @@ fn three_processes_open_direct_and_commit_concurrently() { .stderr(Stdio::piped()) .spawn() .unwrap(); - children.push(child); + children.push(format!("agent-{id}"), child); } let mut sockets = Vec::new(); let deadline = Instant::now() + Duration::from_secs(10); @@ -205,13 +260,21 @@ fn three_processes_open_direct_and_commit_concurrently() { match listener.accept() { Ok((mut stream, _)) => { stream.set_nonblocking(false).unwrap(); + // The handshake itself is bounded too: a child that connects + // and never writes `R` must fail the case, not wait forever. + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); let mut ready = [0u8; 1]; stream.read_exact(&mut ready).unwrap(); assert_eq!(&ready, b"R"); sockets.push(stream); } Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { - std::thread::yield_now(); + std::thread::sleep(Duration::from_millis(1)); } Err(e) => panic!("barrier accept: {e}"), } @@ -219,21 +282,7 @@ fn three_processes_open_direct_and_commit_concurrently() { for mut stream in sockets { stream.write_all(b"G").unwrap(); } - let mut failures = Vec::new(); - for (id, child) in children.into_iter().enumerate() { - let output = wait_bounded(child, &format!("agent-{id}")); - if !output.status.success() { - failures.push(format!( - "agent-{id}: {}", - String::from_utf8_lossy(&output.stderr) - )); - } - } - assert!( - failures.is_empty(), - "concurrent open_direct + 20 commits failed:\n{}", - failures.join("\n") - ); + children.wait_all(60); let verify_dir = fixture.directory.join("overlay-verify"); std::fs::create_dir(&verify_dir).unwrap(); let service = StoreService::open_direct( diff --git a/engine/crates/layerfs-store-service/tests/support/installed_store.rs b/engine/crates/layerfs-store-service/tests/support/installed_store.rs index 126a5235ab..1e47893110 100644 --- a/engine/crates/layerfs-store-service/tests/support/installed_store.rs +++ b/engine/crates/layerfs-store-service/tests/support/installed_store.rs @@ -16,9 +16,9 @@ use std::{ pub const BINDING: &[u8] = b"daemon-installed-store"; pub const CURSOR: [u8; 32] = [43; 32]; -/// Creates a Store's own directory owner-only (`0700`): the kernel guard -/// refuses a Store whose directory grants any group/other bit (#663), and the -/// daemon installs every Store that way. +/// Creates one directory owner-only (`0700`) as any directory holding a Store +/// file must be: the kernel guard refuses one that grants a group or other bit +/// (#663), and the daemon installs every Store that way. fn private_directory(path: &std::path::Path) { #[cfg(unix)] { @@ -28,13 +28,25 @@ fn private_directory(path: &std::path::Path) { #[cfg(not(unix))] std::fs::create_dir(path).unwrap(); } +/// A sealed Store installed as a consumer sees it. pub struct Fixture { + /// Everything this fixture owns. It stays traversable (`0755`): the kernel + /// source, the overlay template and any mount live here, and tests run + /// commands as another uid inside them. Only `store/` below is private. pub directory: PathBuf, pub config: PersistenceConfig, pub branch: BranchId, pub count: usize, } impl Fixture { + /// `directory/store`, where this fixture keeps its Store files: `0700`, as + /// every directory a daemon opens a Store in must be (#663). + pub fn store_directory(&self) -> &std::path::Path { + self.config + .path + .parent() + .expect("the fixture's Store file stands in its own directory") + } pub fn new(count: usize, label: &str) -> Self { Self::built(label, |source| { for n in 0..count { @@ -57,11 +69,13 @@ impl Fixture { ) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - private_directory(&directory); + std::fs::create_dir(&directory).unwrap(); + let store_directory = directory.join("store"); + private_directory(&store_directory); let source = directory.join("source"); std::fs::create_dir(&source).unwrap(); let count = build(&source); - let sealed = directory.join("sealed.sqlite"); + let sealed = store_directory.join("sealed.sqlite"); let config = PersistenceConfig::sqlite(&sealed) .with_sqlite_profile(SqlitePersistenceProfile::Disposable) .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); @@ -107,7 +121,7 @@ impl Fixture { .unwrap(); drop(storage); let sealed = handles.seal().unwrap(); - let installed = directory.join("store.sqlite"); + let installed = store_directory.join("store.sqlite"); std::fs::copy(sealed.path, &installed).unwrap(); // No source or sealed original remains available to the consumer. std::fs::remove_dir_all(source).unwrap(); @@ -128,8 +142,10 @@ impl Fixture { pub fn cloned(label: &str, store: &std::path::Path, count: usize) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - private_directory(&directory); - let installed = directory.join("store.sqlite"); + std::fs::create_dir(&directory).unwrap(); + let store_directory = directory.join("store"); + private_directory(&store_directory); + let installed = store_directory.join("store.sqlite"); std::fs::copy(store, &installed).unwrap(); Self { directory, From 5534c3ef3ab8a745fb4ac396440c0a43357000e6 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:51:28 +0000 Subject: [PATCH 16/19] test: stop concurrent-open child group on early failure and attach stderr on timeout Completes review item 1 (F1) in both three-process concurrency tests: - Children::wait_all polls the group together, stops the whole group (kill + wait) as soon as any child fails or the 60 s group deadline expires, and includes every child's status and stderr in the timeout or failure panic. - Children::abort stops the whole group and attaches every child's stderr when the barrier accept, read, or write fails or times out. - Child sockets also set a 10 s write timeout alongside the 30 s read timeout; writer_admission's barrier accept uses 10 s like kernel_guard. Tests: writer_admission (13 passed), kernel_guard (4 passed); rustfmt --check clean. Deleted tests: none. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py --detail at 30c0e5672 (160034) and this staged tree (160034); external tests only. --- .../tests/writer_admission.rs | 98 ++++++++++++++----- .../tests/kernel_guard.rs | 94 +++++++++++++----- 2 files changed, 140 insertions(+), 52 deletions(-) diff --git a/engine/crates/layerfs-persistence/tests/writer_admission.rs b/engine/crates/layerfs-persistence/tests/writer_admission.rs index 790c664311..77a90b227d 100644 --- a/engine/crates/layerfs-persistence/tests/writer_admission.rs +++ b/engine/crates/layerfs-persistence/tests/writer_admission.rs @@ -220,38 +220,76 @@ impl Children { self.0.push((what, child)); } /// Waits for every child, `seconds` for the group as a whole; the 60 s - /// ceiling stays under the per-invocation test limit. + /// ceiling stays under the per-invocation test limit. Any child failure or + /// timeout stops the entire group (`kill` + `wait`) and attaches every + /// child's stderr to the failure panic. fn wait_all(mut self, seconds: u64) { let deadline = Instant::now() + Duration::from_secs(seconds); + let mut early_failure = false; let mut late = Vec::new(); - for (what, child) in self.0.iter_mut() { - while child.try_wait().unwrap().is_none() { - if Instant::now() >= deadline { - late.push(what.clone()); - break; + loop { + let mut any_running = false; + for (what, child) in self.0.iter_mut() { + match child.try_wait().unwrap() { + Some(status) if !status.success() => { + early_failure = true; + } + Some(_) => {} + None => { + any_running = true; + if Instant::now() >= deadline { + late.push(what.clone()); + } + } } - std::thread::sleep(Duration::from_millis(1)); } + if early_failure || !late.is_empty() || !any_running { + break; + } + std::thread::sleep(Duration::from_millis(5)); } - if !late.is_empty() { + if early_failure || !late.is_empty() { self.stop(); } - let mut failed = Vec::new(); - for (what, child) in std::mem::take(&mut self.0) { - let output = child.wait_with_output().unwrap(); - if !output.status.success() { - failed.push(format!( - "{what}: {}\n{}", - output.status, - String::from_utf8_lossy(&output.stderr).trim_end() - )); - } - } + let details = self.collect_details(); assert!( late.is_empty(), - "the group's deadline passed with {late:?} still running" + "the group's deadline passed with {late:?} still running\n{}", + details + .iter() + .map(|(line, _)| line.as_str()) + .collect::>() + .join("\n") + ); + assert!( + !early_failure && details.iter().all(|(_, ok)| *ok), + "opener failed:\n{}", + details + .into_iter() + .map(|(line, _)| line) + .collect::>() + .join("\n") ); - assert!(failed.is_empty(), "opener failed:\n{}", failed.join("\n")); + } + fn collect_details(&mut self) -> Vec<(String, bool)> { + let mut details = Vec::new(); + for (what, child) in std::mem::take(&mut self.0) { + let output = child.wait_with_output().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + let line = format!("{what}: {}\n{}", output.status, stderr.trim_end()); + details.push((line, output.status.success())); + } + details + } + /// Stops every child and panics with `reason` plus each child's stderr. + fn abort(mut self, reason: impl std::fmt::Display) -> ! { + self.stop(); + let details = self + .collect_details() + .into_iter() + .map(|(line, _)| line) + .collect::>(); + panic!("{reason}\n{}", details.join("\n")); } /// Stops every child of the group, exited or not. fn stop(&mut self) { @@ -690,6 +728,8 @@ fn concurrent_open_child() { UnixStream::connect(std::env::var_os("LAYERFS_ADMISSION_OPEN_SOCK").unwrap()).unwrap(); sock.set_read_timeout(Some(Duration::from_secs(30))) .unwrap(); + sock.set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); sock.write_all(b"R").unwrap(); let mut go = [0u8; 1]; sock.read_exact(&mut go).unwrap(); @@ -726,7 +766,7 @@ fn three_processes_open_and_reserve_concurrently() { children.push(format!("opener-{id}"), child); } let mut sockets = Vec::new(); - let deadline = Instant::now() + Duration::from_secs(5); + let deadline = Instant::now() + Duration::from_secs(10); while sockets.len() < 3 { match listener.accept() { Ok((mut stream, _)) => { @@ -740,18 +780,24 @@ fn three_processes_open_and_reserve_concurrently() { .set_write_timeout(Some(Duration::from_secs(10))) .unwrap(); let mut ready = [0u8; 1]; - stream.read_exact(&mut ready).unwrap(); - assert_eq!(&ready, b"R"); + if let Err(e) = stream.read_exact(&mut ready) { + children.abort(format!("barrier read: {e}")); + } + if &ready != b"R" { + children.abort(format!("unexpected barrier byte: {ready:?}")); + } sockets.push(stream); } Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { std::thread::sleep(Duration::from_millis(1)); } - Err(e) => panic!("barrier accept: {e}"), + Err(e) => children.abort(format!("barrier accept: {e}")), } } for mut stream in sockets { - stream.write_all(b"G").unwrap(); + if let Err(e) = stream.write_all(b"G") { + children.abort(format!("barrier write: {e}")); + } } children.wait_all(60); let handles = open(&path); diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index 266afca8bb..329782f7c5 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -99,6 +99,8 @@ fn open_concurrent_child() { UnixStream::connect(std::env::var_os("LAYERFS_OPEN_CONCURRENT_SOCK").unwrap()).unwrap(); sock.set_read_timeout(Some(std::time::Duration::from_secs(30))) .unwrap(); + sock.set_write_timeout(Some(std::time::Duration::from_secs(10))) + .unwrap(); sock.write_all(b"R").unwrap(); let mut go = [0u8; 1]; sock.read_exact(&mut go).unwrap(); @@ -162,43 +164,77 @@ impl Children { self.0.push((what, child)); } /// Waits for every child, `seconds` for the group as a whole; the 60 s - /// ceiling stays under the per-invocation test limit. + /// ceiling stays under the per-invocation test limit. Any child failure or + /// timeout stops the entire group (`kill` + `wait`) and attaches every + /// child's stderr to the failure panic. fn wait_all(mut self, seconds: u64) { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(seconds); + let mut early_failure = false; let mut late = Vec::new(); - for (what, child) in self.0.iter_mut() { - while child.try_wait().unwrap().is_none() { - if std::time::Instant::now() >= deadline { - late.push(what.clone()); - break; + loop { + let mut any_running = false; + for (what, child) in self.0.iter_mut() { + match child.try_wait().unwrap() { + Some(status) if !status.success() => { + early_failure = true; + } + Some(_) => {} + None => { + any_running = true; + if std::time::Instant::now() >= deadline { + late.push(what.clone()); + } + } } - std::thread::sleep(std::time::Duration::from_millis(1)); } + if early_failure || !late.is_empty() || !any_running { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); } - if !late.is_empty() { + if early_failure || !late.is_empty() { self.stop(); } - let mut failed = Vec::new(); - for (what, child) in std::mem::take(&mut self.0) { - let output = child.wait_with_output().unwrap(); - if !output.status.success() { - failed.push(format!( - "{what}: {}\n{}", - output.status, - String::from_utf8_lossy(&output.stderr).trim_end() - )); - } - } + let details = self.collect_details(); assert!( late.is_empty(), - "the group's deadline passed with {late:?} still running" + "the group's deadline passed with {late:?} still running\n{}", + details + .iter() + .map(|(line, _)| line.as_str()) + .collect::>() + .join("\n") ); assert!( - failed.is_empty(), + !early_failure && details.iter().all(|(_, ok)| *ok), "concurrent open_direct + 20 commits failed:\n{}", - failed.join("\n") + details + .into_iter() + .map(|(line, _)| line) + .collect::>() + .join("\n") ); } + fn collect_details(&mut self) -> Vec<(String, bool)> { + let mut details = Vec::new(); + for (what, child) in std::mem::take(&mut self.0) { + let output = child.wait_with_output().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + let line = format!("{what}: {}\n{}", output.status, stderr.trim_end()); + details.push((line, output.status.success())); + } + details + } + /// Stops every child and panics with `reason` plus each child's stderr. + fn abort(mut self, reason: impl std::fmt::Display) -> ! { + self.stop(); + let details = self + .collect_details() + .into_iter() + .map(|(line, _)| line) + .collect::>(); + panic!("{reason}\n{}", details.join("\n")); + } /// Stops every child of the group, exited or not. fn stop(&mut self) { for (_, child) in self.0.iter_mut() { @@ -269,18 +305,24 @@ fn three_processes_open_direct_and_commit_concurrently() { .set_write_timeout(Some(Duration::from_secs(10))) .unwrap(); let mut ready = [0u8; 1]; - stream.read_exact(&mut ready).unwrap(); - assert_eq!(&ready, b"R"); + if let Err(e) = stream.read_exact(&mut ready) { + children.abort(format!("barrier read: {e}")); + } + if &ready != b"R" { + children.abort(format!("unexpected barrier byte: {ready:?}")); + } sockets.push(stream); } Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { std::thread::sleep(Duration::from_millis(1)); } - Err(e) => panic!("barrier accept: {e}"), + Err(e) => children.abort(format!("barrier accept: {e}")), } } for mut stream in sockets { - stream.write_all(b"G").unwrap(); + if let Err(e) = stream.write_all(b"G") { + children.abort(format!("barrier write: {e}")); + } } children.wait_all(60); let verify_dir = fixture.directory.join("overlay-verify"); From e24c1b40fe83438328464503c81a50a6f95e33e7 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:01:13 +0000 Subject: [PATCH 17/19] docs(architecture): port #647's macOS flush, lane and seal notes into the rebuilt pages Upstream #671 replaced the pages this branch had edited, so the substantive text is carried into their rebuilt successors; only this PR's content moves, and the new structure is kept. - engine/kernel-routes/README.md: the NFS "Durability of a reply" row now names the Commit flush: the flush helper when set, else the export volume's own `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS and `sync(2)` elsewhere, with `sync(2)` also when that call fails. - engine/store-calls/README.md: `open_direct` answers `Busy("Store open: ..")` or `Failed(..)`, with a paragraph on why a `Busy` open is a definite pre-effect refusal and what bypasses the lane. - engine/store/sqlite.md: the Lane paragraph merges upstream's description of the lock (a separate directory descriptor, the macOS `fcntl` reason, `Cancelled`, `LaneHold` drop) with this branch's lane: `flock` on the parent directory on Linux and macOS, taken by a writable open, and only a bypassing writer keeps exact `Busy`. The lane diagram, the idle checkpoint and the lifecycle block say "Linux and macOS" / "release the writer lock". - engine/store/README.md: the direct-open row keeps macOS apfs/hfs; a cross-process-writers row and the invariant say Linux and macOS; `Cancelled` leaves "Linux"; the `writer_admission` demonstration no longer claims its cross-process cases are Linux only. - engine/project-init/README.md: seal takes the lane before the checkpoint and releases it after the sidecar check, and says why (a waiting opener could otherwise create a `-wal` and make a sealed Store Refused). - api/calls/errors-and-events.md: `Busy` covers a writable open that met SQLite's busy lock before any effect, and stays a known outcome. - flows/call.md: hop 7a names the writable open's lane, hop 8 the writer on both hosts. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py --detail at the parent 19a6398ad (160034) and at this commit's tree (160034). Documentation only. --- docs/architecture/api/calls/errors-and-events.md | 4 ++-- docs/architecture/engine/kernel-routes/README.md | 2 +- docs/architecture/engine/project-init/README.md | 13 +++++++------ docs/architecture/engine/store-calls/README.md | 9 ++++++++- docs/architecture/engine/store/README.md | 2 +- docs/architecture/engine/store/sqlite.md | 13 +++++++------ docs/architecture/flows/call.md | 4 ++-- 7 files changed, 28 insertions(+), 19 deletions(-) diff --git a/docs/architecture/api/calls/errors-and-events.md b/docs/architecture/api/calls/errors-and-events.md index 729bf29562..6cc4ac3b76 100644 --- a/docs/architecture/api/calls/errors-and-events.md +++ b/docs/architecture/api/calls/errors-and-events.md @@ -35,7 +35,7 @@ front ends carry it unchanged. | `NoSuchPath`, `IsDirectory`, `NotDirectory`, `NotFile`, `NotEmpty`, `Binary` | path refusals of the `file` calls | [`write/mutation.rs`](../../../../engine/crates/layerfs-store-service/src/write/mutation.rs), [`read/`](../../../../engine/crates/layerfs-store-service/src/read) | | `Exists`, `NameTaken`, `ReservedName` | a name is taken or reserved; a database already connected | [`postgresql/connect.rs`](../../../../control/crates/layerfs-store/src/postgresql/connect.rs), [`history/branch.rs`](../../../../engine/crates/layerfs-store-service/src/history/branch.rs) | | `InUse` | held: running Sandboxes, processes with the Store open, Stores on a machine | [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs), [`inventory.rs`](../../../../control/crates/layerfs-control/src/inventory.rs) | -| `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), or no daemon connection came free; nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | +| `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), no daemon connection came free, or a Store's writable open met SQLite's busy lock before any effect (the open wrote nothing, so the caller may repeat the call); nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) | | `Unresolved` | an earlier operation on the entity ended `Unknown` and its process is gone | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs) | | `Conflict`, `HeadMoved` | a merge or Commit conflict, or the Branch head moved; for `backup.restore`, the Store changed since its last transfer | [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), [`history/lookup.rs`](../../../../engine/crates/layerfs-store-service/src/history/lookup.rs) | | `Dirty`, `MountsActive` | unmount with uncommitted changes; `sandbox.close` while Workspaces are mounted | [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) | @@ -100,7 +100,7 @@ The CLI refuses a wrong command line itself, with exit 2, before any call | `Unresolved` | an earlier `Unknown` on this entity stands; this call did nothing | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs) `blocks` | | daemon `Unknown` | a Workspace operation's outcome is uncertain at the daemon | [`control_types.rs`](../../../../engine/crates/layerfs-bridge/src/control_types.rs) `ControlCode::Unknown` | | `Failed` with `attempted: true` after a daemon exchange | uncertain, though the code is `Failed`: a lost daemon reply is not mapped to `Unknown` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation` | -| `Busy` | known: nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | +| `Busy` | known: nothing was attempted, including a `Busy` Store open (the open changed nothing) | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`service.rs`](../../../../control/crates/layerfs-store/src/service.rs) | | `Unavailable`, `Denied`, `TooLarge` from a worker | known: the request was not sent | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) (`Unreached`) | Intents are written by diff --git a/docs/architecture/engine/kernel-routes/README.md b/docs/architecture/engine/kernel-routes/README.md index da57b59e01..59a86a2a86 100644 --- a/docs/architecture/engine/kernel-routes/README.md +++ b/docs/architecture/engine/kernel-routes/README.md @@ -133,7 +133,7 @@ so NFS calls charge nothing. | Hard links | Aliases share one serial, so one kernel inode | `fileid` is the serial | [`link.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/link.rs), [`attributes.rs`](../../../../engine/crates/layerfs-nfs/src/attributes.rs) | | Ticket release | After the one reply attempt (fuser reports no send result) | After `write_record` returns | [`reply_order.rs`](../../../../engine/crates/layerfs-fuse/src/coherence/reply_order.rs), [`server.rs`](../../../../engine/crates/layerfs-nfs/src/server.rs) | | After a Commit installs a root other than the one shown | `inval_inode` / `inval_entry` per stale serial or name before the answer | Mode-preserving SETATTR refresh through the client | [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs); [FUSE](fuse.md), [NFS](nfs.md) | -| Durability of a reply | `FLUSH`/`FSYNC`/`FSYNCDIR` answer `ENOSYS` once; the kernel then reports success and stops sending them | WRITE answers `FILE_SYNC`, meaning published; the daemon flushes the client before each Commit | [`flush.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/flush.rs), [`changing.rs`](../../../../engine/crates/layerfs-nfs/src/changing.rs) | +| Durability of a reply | `FLUSH`/`FSYNC`/`FSYNCDIR` answer `ENOSYS` once; the kernel then reports success and stops sending them | WRITE answers `FILE_SYNC`, meaning published; the daemon flushes the client before each Commit: the flush helper when set, else the export volume's own flush (`sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere, and `sync(2)` when that call fails) | [`flush.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/flush.rs), [`changing.rs`](../../../../engine/crates/layerfs-nfs/src/changing.rs) | An install of the Workspace's own captured root sends nothing: the tree the mount shows is unchanged. The overlay is disposable and never synchronized to diff --git a/docs/architecture/engine/project-init/README.md b/docs/architecture/engine/project-init/README.md index 677604907b..96f6d98b00 100644 --- a/docs/architecture/engine/project-init/README.md +++ b/docs/architecture/engine/project-init/README.md @@ -120,17 +120,18 @@ readiness wait, not a retry ([`import/batch.rs`](../../../../engine/crates/layer ```text SqliteHandles::seal install_file --> daemon sole session (else Busy) manifest Request, Claim - wal_checkpoint(TRUNCATE) (busy -> Busy) <= 65,519 B Transfer - close; no -wal/-shm/-journal remains per record - macOS: release unused allocation "LFSF\x01" Publish (rename) - --> SealedStore{path, profile, ..} Open, check binding - <-- Installed(manifest + daemon + take the writer lane (Linux and macOS) <= 65,519 B Transfer + wal_checkpoint(TRUNCATE) (busy -> Busy) per record + close; no -wal/-shm/-journal remains "LFSF\x01" Publish (rename) + release the writer lane Open, check binding + macOS: release unused allocation + --> SealedStore{path, profile, ..} <-- Installed(manifest + daemon SQLite version) ``` | Rule | Source | | --- | --- | -| Seal consumes the sole session after `Save::finish` and publication; it is never called on a live daemon's shared Store. `initialize` then builds a `StoreManifest` and returns a `SealedProject`. | [`store/seal.rs`](../../../../engine/crates/layerfs-persistence/src/store/seal.rs), [`store/init.rs`](../../../../engine/crates/layerfs-project/src/store/init.rs) | +| Seal consumes the sole session after `Save::finish` and publication; it is never called on a live daemon's shared Store. `initialize` then builds a `StoreManifest` and returns a `SealedProject`. The lane is held across the checkpoint, the close and the sidecar check, so an opener waiting for the lane cannot create a `-wal` in the gap and turn a sealed Store into `Refused`. | [`store/seal.rs`](../../../../engine/crates/layerfs-persistence/src/store/seal.rs), [`store/init.rs`](../../../../engine/crates/layerfs-project/src/store/init.rs) | | Install checks Disposable, file kind and length against the manifest, streams `MAX_PLAINTEXT_BYTES` records from one fixed buffer, then the finish marker; it never opens the file as a database. A reply manifest that differs from the sent one plus the daemon's SQLite version is a protocol mismatch. | [`store/install.rs`](../../../../engine/crates/layerfs-project/src/store/install.rs), [`control.rs`](../../../../engine/crates/layerfs-bridge/src/control.rs) | | On a failure after the request was attempted the caller closes its send side (the fence), unless the failure was a channel error. | [`store/install.rs`](../../../../engine/crates/layerfs-project/src/store/install.rs) | diff --git a/docs/architecture/engine/store-calls/README.md b/docs/architecture/engine/store-calls/README.md index 0a80d2974b..79ee10824e 100644 --- a/docs/architecture/engine/store-calls/README.md +++ b/docs/architecture/engine/store-calls/README.md @@ -62,11 +62,18 @@ StoreService::open_direct(store, binding, cursor_key, template) 2 store.is_file() -> Missing 3 open_store_observed(sqlite, Disposable, CALLS=64, cache 64 KiB) writer session + read-only control reader - -> Failed("Store open: ..") + -> Busy("Store open: ..") / + Failed("Store open: ..") 4 Owner::start(template, pager 4096 KiB, namespaces 64) -> Failed("overlay template: ..") ``` +A writable `Session::connect` takes the Store directory's writer lane, so two +openers queue instead of meeting SQLite's `Busy`; a `Busy` open therefore +means a writer that bypasses the lane (an uncoordinated raw SQLite process) +held the lock, and it is a definite refusal that changed nothing: the caller +may repeat the call. Any other open failure stays `Failed`. + | Field or constant | Meaning | Source | | --- | --- | --- | | `CALLS` = 64 | Private Workspaces one process holds at once (one per SDK binding worker thread); `bind` refuses `Capacity` at the limit. | [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) | diff --git a/docs/architecture/engine/store/README.md b/docs/architecture/engine/store/README.md index 87c2ba6ea9..b2a670667e 100644 --- a/docs/architecture/engine/store/README.md +++ b/docs/architecture/engine/store/README.md @@ -157,7 +157,7 @@ A PostgreSQL Store has no profile; its write setting is `synchronous_commit`. | Limit or failure | Value or error | Source | | --- | --- | --- | | Platform for SQLite | macOS or Linux; elsewhere `BackendUnavailable` | [`open.rs`](../../../../engine/crates/layerfs-persistence/src/store/open.rs) | -| Direct opening for Store calls | Linux, Store directory on ext4, xfs or btrfs; else `Unsupported`, no fallback | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | +| Direct opening for Store calls | Linux on ext4, xfs or btrfs, macOS on apfs or hfs (one shared kernel); else `Unsupported`, no fallback | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | | Cross-process writers (Linux and macOS) | Directory `flock` on the Store's parent directory; an uncoordinated raw SQLite writer still gets SQLite `Busy` | [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | | PostgreSQL without the feature | `BackendUnavailable` | [`selection.rs`](../../../../engine/crates/layerfs-persistence/src/store/selection.rs) | | PostgreSQL lane wait | `lock_timeout_ms`, default 30,000; then `Busy` | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | diff --git a/docs/architecture/engine/store/sqlite.md b/docs/architecture/engine/store/sqlite.md index ecf4e7522d..f2a04f2a8e 100644 --- a/docs/architecture/engine/store/sqlite.md +++ b/docs/architecture/engine/store/sqlite.md @@ -83,8 +83,8 @@ SQL work to the calling thread's counters SQLite write: Session::run(true, body) 1 admission.enter in-process FIFO turn on this session 2 state mutex the connection - 3 Writer::acquire Linux: flock(LOCK_EX) on the Store's parent - directory; macOS: none (skipped if nested) + 3 Writer::acquire flock(LOCK_EX) on the Store's parent directory + (Linux and macOS) (skipped if nested) 4 BEGIN IMMEDIATE busy_timeout 0 -> Busy 5 body; COMMIT or ROLLBACK; Unknown -> quarantine 6 release the flock; drop the turn @@ -123,7 +123,7 @@ quarantines the session and keeps the lock StoreFile::checkpoint() (idle: last caller in the process released it) read-only? Refused "ReadOnly" -wal empty? Ok(true), no SQL this thread holds the lane? Ok(false) - Linux writer lane, one try; held elsewhere? Ok(false) + writer lane (Linux and macOS), one try; held elsewhere? Ok(false) wait for a running PASSIVE pass; wal_checkpoint(TRUNCATE) once busy (a reader in the log)? Ok(false) else Ok(true): -wal is 0 B ``` @@ -132,7 +132,7 @@ quarantines the session and keeps the lock | --- | --- | | A pass that cannot copy every frame leaves them valid; the next wake, the idle checkpoint or the last close copies them. Each wake is a new pass, never a retry. | [`background.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs) | | A PASSIVE checkpoint takes no lock a writer needs, so it adds no `Busy`. | same | -| The idle TRUNCATE takes the writer lane first, because it holds SQLite's WAL write lock; `Ok(false)` is never retried. The Store stays open. | [`checkpoint.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs) | +| The idle TRUNCATE takes the writer lane first, because it holds SQLite's WAL write lock; `Ok(false)` is never retried. The Store stays open. Every host that opens a SQLite Store takes this lane, so a writer waits for it or the checkpoint is left, and only a writer that bypasses the lane meets exact `Busy`. | [`checkpoint.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs) | | Control's open-Store cache calls it when the last call or Sandbox on a Store returns. | [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs) | ## Lifecycle @@ -143,11 +143,12 @@ quarantines the session and keeps the lock own READ_ONLY connection storage still shared? Busy VACUUM INTO destination read-only? Refused "ReadOnly" reopen copy, journal_mode=wal quarantined / open tx? Uncertain - wal_checkpoint(TRUNCATE), close Linux: take the writer lock + wal_checkpoint(TRUNCATE), close take the writer lock (both OSes) no sidecar wal_checkpoint(TRUNCATE) or Busy \ close; sidecar left? Refused - \ macOS: release extents past EOF + \ release the writer lock +-------> SealedStore <-----------+ + macOS: release extents past EOF ``` | Operation | Bound or rule | Source | diff --git a/docs/architecture/flows/call.md b/docs/architecture/flows/call.md index afdc8ad845..42397c5b3f 100644 --- a/docs/architecture/flows/call.md +++ b/docs/architecture/flows/call.md @@ -53,9 +53,9 @@ retries. A lost answer after the request was sent is `Unknown`, never resent. | 4 | The home is opened (explicit path, else `LAYERFS_HOME`, else `~/.layerfs`). | [Home](../control/home/README.md) | | 5 | The family function reads the Store and project records, refuses a blocking intent and takes this process's lease on the Store. | [Stores and projects](../control/stores-and-projects/README.md#what-it-does), [Home: the `op` intent](../control/home/README.md#the-op-intent) | | 6 | Where the call runs is decided once from the record: the caller's process, or the Store machine's worker. | [Control: where each call runs](../control/README.md#where-each-call-runs) | -| 7a | Local: the process keeps the Store open and runs the method once through the Store service; the kernel guard runs before any file is opened. | [Store calls: opening the service](../engine/store-calls/README.md#opening-the-service), [Store calls: method table](../engine/store-calls/README.md#method-table) | +| 7a | Local: the process keeps the Store open and runs the method once through the Store service; the kernel guard runs before any file is opened, and a writable open takes the Store directory's writer lane (Linux and macOS), so two openers queue instead of meeting SQLite's busy lock. | [Store calls: opening the service](../engine/store-calls/README.md#opening-the-service), [Store calls: method table](../engine/store-calls/README.md#method-table) | | 7b | Worker: one loopback connection, the secret first, then one request; the worker holds its own lease and runs step 7a. | [Workers and reach](../control/machines/workers-and-reach.md), [Bridge](../engine/bridge/README.md) | -| 8 | Reads use the published read session and never take the lane; a write takes the session's admission turn and the Store's writer. | [Store calls: read sessions](../engine/store-calls/README.md#read-sessions), [Store: invariants](../engine/store/README.md#invariants) | +| 8 | Reads use the published read session and never take the lane; a write takes the session's admission turn and the Store's writer (a directory `flock` on Linux and macOS, or PostgreSQL's `store_lane` row). | [Store calls: read sessions](../engine/store-calls/README.md#read-sessions), [Store: invariants](../engine/store/README.md#invariants) | | 9 | The answer's keys become snake_case; events went out as they came; a refusal carries its stable code. | [API: calling conventions](../api/README.md#calling-conventions), [Errors and events](../api/calls/errors-and-events.md) | ## Related From 19aaeb1b548be3f936d28d48e48679fcf901a476 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:44:39 +0000 Subject: [PATCH 18/19] docs(project-init): align the seal block's two columns The seal and install steps gained two lines ("take the writer lane", "release the writer lane"), which left the paired `install_file` column one space off. Realign both, content unchanged. Production LOC: 160034 -> 160034 (delta 0) Scope: first-party core/engine/control/API (core 39787, engine 74475, control 39796, api 5976, dsh 0; reference absent, retired R9c, last counted 65,417 at tree 498dd1917812). Method: tools/production_loc.py --detail at the parent e24c1b40f (160034) and this commit's tree (160034). Documentation only. --- docs/architecture/engine/project-init/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/architecture/engine/project-init/README.md b/docs/architecture/engine/project-init/README.md index 96f6d98b00..f5c50a69af 100644 --- a/docs/architecture/engine/project-init/README.md +++ b/docs/architecture/engine/project-init/README.md @@ -118,15 +118,15 @@ readiness wait, not a retry ([`import/batch.rs`](../../../../engine/crates/layer ### Seal and install ```text - SqliteHandles::seal install_file --> daemon - sole session (else Busy) manifest Request, Claim - take the writer lane (Linux and macOS) <= 65,519 B Transfer - wal_checkpoint(TRUNCATE) (busy -> Busy) per record - close; no -wal/-shm/-journal remains "LFSF\x01" Publish (rename) - release the writer lane Open, check binding + SqliteHandles::seal install_file --> daemon + sole session (else Busy) manifest Request, Claim + take the writer lane (Linux and macOS) <= 65,519 B Transfer + wal_checkpoint(TRUNCATE) (busy -> Busy) per record + close; no -wal/-shm/-journal remains "LFSF\x01" Publish (rename) + release the writer lane Open, check binding macOS: release unused allocation - --> SealedStore{path, profile, ..} <-- Installed(manifest + daemon - SQLite version) + --> SealedStore{path, profile, ..} <-- Installed(manifest + daemon + SQLite version) ``` | Rule | Source | From 11b30f0d760595998ade79b013ddc30a4efa79b9 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Sat, 14 Nov 2026 16:17:52 +0800 Subject: [PATCH 19/19] fix(daemon): refuse the Commit when the macOS volume flush fails A failed sync_volume_np no longer falls back to sync(2) and runs the Commit: on macOS sync(2) may return before the writes are flushed, so the Commit could capture without the client's buffered writes and hide the error. The default flush now refuses the Commit as Failed at commit:flush, as a failed flush helper already does, and leaves the Workspace idle. Linux keeps sync(2). Tests: layerfs-nfs mounter's second case now expects the refusal (renamed default_flush_commits_with_the_export_volume_and_refuses_without_it). Docs: kernel-routes/README.md and daemon/control-calls.md state the refusal. Production LOC: 160034 -> 160034 (delta 0); scope core 39787, engine 74475, control 39796, api 5976; reference absent (retired R9c; last counted 65,417 at tree 498dd1917812); method tools/production_loc.py on the parent and the committed tree. Co-Authored-By: Claude Opus 5.5 --- .../engine/daemon/control-calls.md | 2 +- .../engine/kernel-routes/README.md | 2 +- .../layerfs-daemon/src/control/export.rs | 25 ++++---- engine/crates/layerfs-nfs/tests/mounter.rs | 63 ++++++++++--------- 4 files changed, 50 insertions(+), 42 deletions(-) diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index b4c04bf0dd..aec17d14b8 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -97,7 +97,7 @@ differs. `Exported`, `ExportLeaving` and `ExportKept` mirror `Ready`, | Step | NFS route | Source | | --- | --- | --- | | Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| Commit | Flush first (flush helper, or the export volume's own flush: `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | +| Commit | Flush first (flush helper, or the export volume's own flush: `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere; a failed flush refuses the Commit as `Failed` at `commit:flush`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | | Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | ForceUnmount | Force helper once, then the same teardown | same | | Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | diff --git a/docs/architecture/engine/kernel-routes/README.md b/docs/architecture/engine/kernel-routes/README.md index 59a86a2a86..c345942caf 100644 --- a/docs/architecture/engine/kernel-routes/README.md +++ b/docs/architecture/engine/kernel-routes/README.md @@ -133,7 +133,7 @@ so NFS calls charge nothing. | Hard links | Aliases share one serial, so one kernel inode | `fileid` is the serial | [`link.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/link.rs), [`attributes.rs`](../../../../engine/crates/layerfs-nfs/src/attributes.rs) | | Ticket release | After the one reply attempt (fuser reports no send result) | After `write_record` returns | [`reply_order.rs`](../../../../engine/crates/layerfs-fuse/src/coherence/reply_order.rs), [`server.rs`](../../../../engine/crates/layerfs-nfs/src/server.rs) | | After a Commit installs a root other than the one shown | `inval_inode` / `inval_entry` per stale serial or name before the answer | Mode-preserving SETATTR refresh through the client | [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs); [FUSE](fuse.md), [NFS](nfs.md) | -| Durability of a reply | `FLUSH`/`FSYNC`/`FSYNCDIR` answer `ENOSYS` once; the kernel then reports success and stops sending them | WRITE answers `FILE_SYNC`, meaning published; the daemon flushes the client before each Commit: the flush helper when set, else the export volume's own flush (`sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere, and `sync(2)` when that call fails) | [`flush.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/flush.rs), [`changing.rs`](../../../../engine/crates/layerfs-nfs/src/changing.rs) | +| Durability of a reply | `FLUSH`/`FSYNC`/`FSYNCDIR` answer `ENOSYS` once; the kernel then reports success and stops sending them | WRITE answers `FILE_SYNC`, meaning published; the daemon flushes the client before each Commit: the flush helper when set, else the export volume's own flush (`sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere); a failed flush refuses the Commit as `Failed` at `commit:flush` | [`flush.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/flush.rs), [`changing.rs`](../../../../engine/crates/layerfs-nfs/src/changing.rs) | An install of the Workspace's own captured root sends nothing: the tree the mount shows is unchanged. The overlay is disposable and never synchronized to diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index 2bb3e391a4..ead80216dd 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -79,7 +79,7 @@ pub struct Mounter { pub unmount: Helper, pub force: Helper, /// Writes back what the client buffered before a Commit; `sync_volume_np` - /// of the export directory on macOS (falling back to `sync(2)`) or + /// of the export directory on macOS (its failure refuses the Commit) or /// `sync(2)` elsewhere when absent. pub flush: Option, } @@ -585,7 +585,11 @@ impl Service { return Ok(()); }; match &serving.config.mounter.flush { - None => flush_default(std::path::Path::new(&directory)), + None => { + if let Err(detail) = flush_default(std::path::Path::new(&directory)) { + return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)); + } + } Some(helper) => { match serving .helpers @@ -935,18 +939,17 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result Result<(), String> { + layerfs_bridge::process::sync_volume_wait(directory) + .map_err(|error| format!("sync_volume_np({}): {error}", directory.display())) } #[cfg(not(target_os = "macos"))] -fn flush_default(_directory: &std::path::Path) { +fn flush_default(_directory: &std::path::Path) -> Result<(), String> { nix::unistd::sync(); + Ok(()) } diff --git a/engine/crates/layerfs-nfs/tests/mounter.rs b/engine/crates/layerfs-nfs/tests/mounter.rs index 2309013b34..2daffa7624 100644 --- a/engine/crates/layerfs-nfs/tests/mounter.rs +++ b/engine/crates/layerfs-nfs/tests/mounter.rs @@ -1,7 +1,7 @@ //! The native macOS client must not send atime updates the export ignores, //! and a Commit with the default export flush (the export volume's own -//! `sync_volume_np` on macOS, `sync(2)` elsewhere) never fails on the flush -//! itself, even when the export's directory is gone. +//! `sync_volume_np` on macOS) runs when that flush succeeds and is refused, +//! with nothing committed, when it fails. #![cfg(target_os = "macos")] #[allow(dead_code)] #[path = "support/client.rs"] @@ -10,8 +10,8 @@ mod client; #[path = "support/rig.rs"] mod rig; use client::Client; -use layerfs_bridge::control::{CommitControl, Reply, Request, UnmountControl}; -use layerfs_daemon::control::{ExportServing, ExportServingConfig, Helper, Mounter}; +use layerfs_bridge::control::{CommitControl, ControlCode, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Failure, Helper, Mounter}; use layerfs_history::WorkspaceId; use std::{ fs, @@ -36,19 +36,16 @@ fn host_mount_disables_access_time_updates() { } #[test] -fn default_flush_commits_with_the_export_volume_and_without_it() { +fn default_flush_commits_with_the_export_volume_and_refuses_without_it() { // What this observes, and what it cannot: the Workspace is driven through // this test's own NFS RPC client, so the host's mount client holds no - // writeback for either flush to push and neither syscall leaves a trace - // here. The two Commits below therefore check the contract that is - // observable - a flush that succeeds (`sync_volume_np` of the live - // directory) and one that fails (`ENOENT` after the directory is removed, - // which falls back to `sync(2)`) both leave the Commit to run and - // succeed. The flush calls themselves are covered where they live: - // `sync_volume_wait`'s `Ok`/error by `layerfs-bridge`'s - // `process_identity` test. The `ENOENT` to `sync(2)` fallback inside - // `flush_default` has no test of its own, so all this test observes of it - // is its outcome: the Commit still runs. + // writeback for the flush to push and the syscall leaves no trace here. + // The two Commits below therefore check the contract that is observable: + // a flush that succeeds (`sync_volume_np` of the live directory) lets + // the Commit run, and one that fails (`ENOENT` after the directory is + // removed) refuses it as `Failed` at `commit:flush`, before it runs, and + // leaves the Workspace idle. `sync_volume_wait`'s `Ok`/error is covered + // where it lives, by `layerfs-bridge`'s `process_identity` test. let base = std::env::temp_dir().join(format!("layerfs-nfs-mounter-{}", std::process::id())); let source = base.join("src"); let work = base.join("work"); @@ -115,24 +112,32 @@ fn default_flush_commits_with_the_export_volume_and_without_it() { other => panic!("{other:?}"), } - // 2. Missing export directory: `sync_volume_np` answers `ENOENT` and - // `flush_default` falls back to `sync(2)` instead of failing or skipping - // the client writeback; the Commit still runs and succeeds. + // 2. Missing export directory: `sync_volume_np` answers `ENOENT`; the + // Commit is refused before it runs instead of skipping the writeback. let f2 = nfs.touch(&root, b"two.txt", 0o644).unwrap(); nfs.write_all(&f2.handle, 0, b"second\n", 7); fs::remove_dir_all(&ready.directory).unwrap(); assert!(!Path::new(&ready.directory).exists()); - match service - .execute_control(&Request::Commit(CommitControl { - token, - force: false, - message: Some("second".into()), - })) - .unwrap() - .reply - { - Reply::Committed(_) => {} - other => panic!("{other:?}"), + match service.execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("second".into()), + })) { + Err(Failure::Kernel(failure)) => { + assert_eq!( + (failure.code, failure.phase), + (ControlCode::Failed, "commit:flush"), + "{}", + failure.detail + ); + assert!( + failure.detail.contains("sync_volume_np"), + "{}", + failure.detail + ); + } + Ok(outcome) => panic!("the Commit ran without its flush: {:?}", outcome.reply), + Err(other) => panic!("{other:?}"), } fs::create_dir_all(&ready.directory).unwrap();