diff --git a/api/crates/layerfs-api/tests/store_open_cache.rs b/api/crates/layerfs-api/tests/store_open_cache.rs index 06b913bb3b..2622c3f9ca 100644 --- a/api/crates/layerfs-api/tests/store_open_cache.rs +++ b/api/crates/layerfs-api/tests/store_open_cache.rs @@ -2,8 +2,8 @@ //! open copy holds a lease under its root and is closed by the steps that //! read, replace or delete the file, so a Store is pulled back at once after a //! write and a Store that returns is never served from a stale handle; and -//! (Linux, where a writer waits on its Store directory's lock) a call that -//! is running keeps its Store open against a push or remove. +//! (where a writer waits on its Store directory's lock, Linux and macOS) a +//! call that is running keeps its Store open against a push or remove. use layerfs_api::home::{self, paths, write_record}; use layerfs_bridge::control::SdkValue; use std::net::TcpListener; @@ -156,9 +156,9 @@ fn a_worker_closes_its_open_copy_so_a_store_is_pulled_at_once_and_returns_fresh( let _ = std::fs::remove_dir_all(folder); } -/// A write waits for the Store directory's writer lock (Linux): holding it -/// here keeps one call running for as long as this test wants. -#[cfg(target_os = "linux")] +/// A write waits for the Store directory's writer lock (Linux and macOS): +/// holding it here keeps one call running for as long as this test wants. +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn a_running_call_keeps_its_store_open_against_a_push_or_remove() { use nix::fcntl::{Flock, FlockArg}; diff --git a/control/crates/layerfs-store/src/service.rs b/control/crates/layerfs-store/src/service.rs index 02069bda6b..e57360725a 100644 --- a/control/crates/layerfs-store/src/service.rs +++ b/control/crates/layerfs-store/src/service.rs @@ -50,6 +50,7 @@ fn open(store: &Path, args: &OpenArgs) -> Result { .map_err(|refusal| match refusal { OpenRefusal::Unsupported(reason) => refused("Unsupported", reason.0), OpenRefusal::Missing(reason) => refused("NoStore", reason), + OpenRefusal::Busy(reason) => refused("Busy", reason), OpenRefusal::Failed(reason) => refused("Failed", reason), }) } diff --git a/docs/architecture/api/calls/errors-and-events.md b/docs/architecture/api/calls/errors-and-events.md index 729bf29562..6cc4ac3b76 100644 --- a/docs/architecture/api/calls/errors-and-events.md +++ b/docs/architecture/api/calls/errors-and-events.md @@ -35,7 +35,7 @@ front ends carry it unchanged. | `NoSuchPath`, `IsDirectory`, `NotDirectory`, `NotFile`, `NotEmpty`, `Binary` | path refusals of the `file` calls | [`write/mutation.rs`](../../../../engine/crates/layerfs-store-service/src/write/mutation.rs), [`read/`](../../../../engine/crates/layerfs-store-service/src/read) | | `Exists`, `NameTaken`, `ReservedName` | a name is taken or reserved; a database already connected | [`postgresql/connect.rs`](../../../../control/crates/layerfs-store/src/postgresql/connect.rs), [`history/branch.rs`](../../../../engine/crates/layerfs-store-service/src/history/branch.rs) | | `InUse` | held: running Sandboxes, processes with the Store open, Stores on a machine | [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs), [`inventory.rs`](../../../../control/crates/layerfs-control/src/inventory.rs) | -| `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), or no daemon connection came free; nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | +| `Busy` | another operation runs on the entity (`detail {entity, phase, op}`), no daemon connection came free, or a Store's writable open met SQLite's busy lock before any effect (the open wrote nothing, so the caller may repeat the call); nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) | | `Unresolved` | an earlier operation on the entity ended `Unknown` and its process is gone | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs) | | `Conflict`, `HeadMoved` | a merge or Commit conflict, or the Branch head moved; for `backup.restore`, the Store changed since its last transfer | [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), [`history/lookup.rs`](../../../../engine/crates/layerfs-store-service/src/history/lookup.rs) | | `Dirty`, `MountsActive` | unmount with uncommitted changes; `sandbox.close` while Workspaces are mounted | [`daemon/values.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/values.rs), [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) | @@ -100,7 +100,7 @@ The CLI refuses a wrong command line itself, with exit 2, before any call | `Unresolved` | an earlier `Unknown` on this entity stands; this call did nothing | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs) `blocks` | | daemon `Unknown` | a Workspace operation's outcome is uncertain at the daemon | [`control_types.rs`](../../../../engine/crates/layerfs-bridge/src/control_types.rs) `ControlCode::Unknown` | | `Failed` with `attempted: true` after a daemon exchange | uncertain, though the code is `Failed`: a lost daemon reply is not mapped to `Unknown` | [`lifecycle.rs`](../../../../control/crates/layerfs-sandbox/src/lifecycle.rs) `operation` | -| `Busy` | known: nothing was attempted | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs) | +| `Busy` | known: nothing was attempted, including a `Busy` Store open (the open changed nothing) | [`op.rs`](../../../../control/crates/layerfs-home/src/op.rs), [`daemon/pool.rs`](../../../../control/crates/layerfs-sandbox/src/daemon/pool.rs), [`service.rs`](../../../../control/crates/layerfs-store/src/service.rs) | | `Unavailable`, `Denied`, `TooLarge` from a worker | known: the request was not sent | [`worker/client.rs`](../../../../control/crates/layerfs-machine/src/worker/client.rs) (`Unreached`) | Intents are written by diff --git a/docs/architecture/engine/daemon/control-calls.md b/docs/architecture/engine/daemon/control-calls.md index 3ff411ae5c..b814f8aa7b 100644 --- a/docs/architecture/engine/daemon/control-calls.md +++ b/docs/architecture/engine/daemon/control-calls.md @@ -97,7 +97,7 @@ differs. `Exported`, `ExportLeaving` and `ExportKept` mirror `Ready`, | Step | NFS route | Source | | --- | --- | --- | | Attach | Serve the engine mount as one loopback NFSv3 export on a fresh port; the client mounts it with the mount helper; Ready only after the helper reports the mount. The receipt has `abi_major == 0` and `mount_id` = the port | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -| Commit | Flush first (flush helper, or `sync(2)`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | +| Commit | Flush first (flush helper, or the export volume's own flush: `sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere; a failed flush refuses the Commit as `Failed` at `commit:flush`), close the export's descriptors, run the unchanged Commit; after an install of another root, close them again | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs), [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs) | | Unmount | Helper unmount (busy is reversible), stop the export, revoke, Close | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | | ForceUnmount | Force helper once, then the same teardown | same | | Helper run | One attempt, at most `HELPER_WAIT` 60 s, then killed and reported | [`export_helper.rs`](../../../../engine/crates/layerfs-daemon/src/control/export_helper.rs) | diff --git a/docs/architecture/engine/kernel-routes/README.md b/docs/architecture/engine/kernel-routes/README.md index da57b59e01..c345942caf 100644 --- a/docs/architecture/engine/kernel-routes/README.md +++ b/docs/architecture/engine/kernel-routes/README.md @@ -133,7 +133,7 @@ so NFS calls charge nothing. | Hard links | Aliases share one serial, so one kernel inode | `fileid` is the serial | [`link.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/link.rs), [`attributes.rs`](../../../../engine/crates/layerfs-nfs/src/attributes.rs) | | Ticket release | After the one reply attempt (fuser reports no send result) | After `write_record` returns | [`reply_order.rs`](../../../../engine/crates/layerfs-fuse/src/coherence/reply_order.rs), [`server.rs`](../../../../engine/crates/layerfs-nfs/src/server.rs) | | After a Commit installs a root other than the one shown | `inval_inode` / `inval_entry` per stale serial or name before the answer | Mode-preserving SETATTR refresh through the client | [`invalidate.rs`](../../../../engine/crates/layerfs-daemon/src/control/invalidate.rs); [FUSE](fuse.md), [NFS](nfs.md) | -| Durability of a reply | `FLUSH`/`FSYNC`/`FSYNCDIR` answer `ENOSYS` once; the kernel then reports success and stops sending them | WRITE answers `FILE_SYNC`, meaning published; the daemon flushes the client before each Commit | [`flush.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/flush.rs), [`changing.rs`](../../../../engine/crates/layerfs-nfs/src/changing.rs) | +| Durability of a reply | `FLUSH`/`FSYNC`/`FSYNCDIR` answer `ENOSYS` once; the kernel then reports success and stops sending them | WRITE answers `FILE_SYNC`, meaning published; the daemon flushes the client before each Commit: the flush helper when set, else the export volume's own flush (`sync_volume_np(, SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere); a failed flush refuses the Commit as `Failed` at `commit:flush` | [`flush.rs`](../../../../engine/crates/layerfs-mount-ports/src/operations/flush.rs), [`changing.rs`](../../../../engine/crates/layerfs-nfs/src/changing.rs) | An install of the Workspace's own captured root sends nothing: the tree the mount shows is unchanged. The overlay is disposable and never synchronized to diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index f3b9a5f30d..4c8f4e0325 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -186,7 +186,8 @@ reaches the Workspace or a Commit; everything is dropped with the export. Attach: bind 127.0.0.1:0 -> Server::start -> port P helper mount_nfs -o vers=3,tcp,port=P,mountport=P,... client MOUNT MNT "/" -> root handle, AUTH_UNIX - Commit: flush helper (sync(2) by default), close_files, Commit + Commit: flush helper (else the export volume's own flush: sync_volume_np + on macOS, sync(2) elsewhere), close_files, Commit Unmount: unmount helper (busy is reversible), Server::stop, revoke the engine mount, Close ``` diff --git a/docs/architecture/engine/project-init/README.md b/docs/architecture/engine/project-init/README.md index 677604907b..f5c50a69af 100644 --- a/docs/architecture/engine/project-init/README.md +++ b/docs/architecture/engine/project-init/README.md @@ -118,19 +118,20 @@ readiness wait, not a retry ([`import/batch.rs`](../../../../engine/crates/layer ### Seal and install ```text - SqliteHandles::seal install_file --> daemon - sole session (else Busy) manifest Request, Claim - wal_checkpoint(TRUNCATE) (busy -> Busy) <= 65,519 B Transfer - close; no -wal/-shm/-journal remains per record - macOS: release unused allocation "LFSF\x01" Publish (rename) - --> SealedStore{path, profile, ..} Open, check binding - <-- Installed(manifest + daemon - SQLite version) + SqliteHandles::seal install_file --> daemon + sole session (else Busy) manifest Request, Claim + take the writer lane (Linux and macOS) <= 65,519 B Transfer + wal_checkpoint(TRUNCATE) (busy -> Busy) per record + close; no -wal/-shm/-journal remains "LFSF\x01" Publish (rename) + release the writer lane Open, check binding + macOS: release unused allocation + --> SealedStore{path, profile, ..} <-- Installed(manifest + daemon + SQLite version) ``` | Rule | Source | | --- | --- | -| Seal consumes the sole session after `Save::finish` and publication; it is never called on a live daemon's shared Store. `initialize` then builds a `StoreManifest` and returns a `SealedProject`. | [`store/seal.rs`](../../../../engine/crates/layerfs-persistence/src/store/seal.rs), [`store/init.rs`](../../../../engine/crates/layerfs-project/src/store/init.rs) | +| Seal consumes the sole session after `Save::finish` and publication; it is never called on a live daemon's shared Store. `initialize` then builds a `StoreManifest` and returns a `SealedProject`. The lane is held across the checkpoint, the close and the sidecar check, so an opener waiting for the lane cannot create a `-wal` in the gap and turn a sealed Store into `Refused`. | [`store/seal.rs`](../../../../engine/crates/layerfs-persistence/src/store/seal.rs), [`store/init.rs`](../../../../engine/crates/layerfs-project/src/store/init.rs) | | Install checks Disposable, file kind and length against the manifest, streams `MAX_PLAINTEXT_BYTES` records from one fixed buffer, then the finish marker; it never opens the file as a database. A reply manifest that differs from the sent one plus the daemon's SQLite version is a protocol mismatch. | [`store/install.rs`](../../../../engine/crates/layerfs-project/src/store/install.rs), [`control.rs`](../../../../engine/crates/layerfs-bridge/src/control.rs) | | On a failure after the request was attempted the caller closes its send side (the fence), unless the failure was a channel error. | [`store/install.rs`](../../../../engine/crates/layerfs-project/src/store/install.rs) | diff --git a/docs/architecture/engine/store-calls/README.md b/docs/architecture/engine/store-calls/README.md index 71bfec018f..d7eecb0ed3 100644 --- a/docs/architecture/engine/store-calls/README.md +++ b/docs/architecture/engine/store-calls/README.md @@ -62,11 +62,18 @@ StoreService::open_direct(store, binding, cursor_key, template) 2 store.is_file() -> Missing 3 open_store_observed(sqlite, Disposable, CALLS=64, cache 64 KiB) writer session + read-only control reader - -> Failed("Store open: ..") + -> Busy("Store open: ..") / + Failed("Store open: ..") 4 Owner::start(template, pager 4096 KiB, namespaces 64) -> Failed("overlay template: ..") ``` +A writable `Session::connect` takes the Store directory's writer lane, so two +openers queue instead of meeting SQLite's `Busy`; a `Busy` open therefore +means a writer that bypasses the lane (an uncoordinated raw SQLite process) +held the lock, and it is a definite refusal that changed nothing: the caller +may repeat the call. Any other open failure stays `Failed`. + | Field or constant | Meaning | Source | | --- | --- | --- | | `CALLS` = 64 | Private Workspaces one process holds at once (one per SDK binding worker thread); `bind` refuses `Capacity` at the limit. | [`store_service.rs`](../../../../engine/crates/layerfs-store-service/src/store_service.rs) | diff --git a/docs/architecture/engine/store/README.md b/docs/architecture/engine/store/README.md index 0c3a137816..b2a670667e 100644 --- a/docs/architecture/engine/store/README.md +++ b/docs/architecture/engine/store/README.md @@ -129,7 +129,7 @@ own write transaction, which the one writer already orders. | Rule | Enforced by | | --- | --- | | One global database per Store; no sharding or second database | One `Session` per open; the exact table set is checked at open: [`sqlite/schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs), [`postgresql/schema.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs) | -| One writer at a time: FIFO admission per session, then a cross-process lock (Linux directory `flock`, or the `store_lane` row) | [`admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/admission.rs), [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs), [`postgresql/session/profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/session/profile.rs) | +| One writer at a time: FIFO admission per session, then a cross-process lock (a directory `flock` on Linux and macOS, or the `store_lane` row) | [`admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/admission.rs), [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs), [`postgresql/session/profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/session/profile.rs) | | Waiting for a turn is readiness, never a retry; the queue has no time limit and a holder that unwinds releases its turn | [`admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/admission.rs) | | Short holds: only head check, merge step and publish run under the lane | [`commit.rs`](../../../../engine/crates/layerfs-store-service/src/store/commit.rs) | | Reads never take the lane or the cross-process lock | [`sqlite/session/transaction.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/transaction.rs), [`read_service.rs`](../../../../engine/crates/layerfs-store-service/src/store/read_service.rs) | @@ -157,8 +157,8 @@ A PostgreSQL Store has no profile; its write setting is `synchronous_commit`. | Limit or failure | Value or error | Source | | --- | --- | --- | | Platform for SQLite | macOS or Linux; elsewhere `BackendUnavailable` | [`open.rs`](../../../../engine/crates/layerfs-persistence/src/store/open.rs) | -| Direct opening for Store calls | Linux, Store directory on ext4, xfs or btrfs; else `Unsupported`, no fallback | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | -| macOS cross-process writers | No directory lock; a second writer gets SQLite `Busy` | [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | +| Direct opening for Store calls | Linux on ext4, xfs or btrfs, macOS on apfs or hfs (one shared kernel); else `Unsupported`, no fallback | [`kernel_guard.rs`](../../../../engine/crates/layerfs-store-service/src/kernel_guard.rs) | +| Cross-process writers (Linux and macOS) | Directory `flock` on the Store's parent directory; an uncoordinated raw SQLite writer still gets SQLite `Busy` | [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs) | | PostgreSQL without the feature | `BackendUnavailable` | [`selection.rs`](../../../../engine/crates/layerfs-persistence/src/store/selection.rs) | | PostgreSQL lane wait | `lock_timeout_ms`, default 30,000; then `Busy` | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs) | | Rows per publication or `reserve` | `TRANSACTION_ROW_LIMIT` 8,191 | [`publication.rs`](../../../../engine/crates/layerfs-persistence/src/storage/publication.rs) | @@ -175,7 +175,7 @@ A PostgreSQL Store has no profile; its write setting is `synchronous_commit`. | `Busy` | `Busy` | `Busy` | Another writer held the lock | | `Closed` | `Refused "Closed"` | `OwnershipUnavailable` | Store closed before this call's turn | | `Rejected` | `Refused "Rejected"` | `OwnershipUnavailable` | PostgreSQL refused a new connection | -| `Cancelled` | `Refused "Cancelled"` | `OwnershipUnavailable` | A signal interrupted the Linux writer wait | +| `Cancelled` | `Refused "Cancelled"` | `OwnershipUnavailable` | A signal interrupted the writer wait (Linux and macOS) | | `ReadOnly` | `Refused "ReadOnly"` | `ContinuityUnavailable` | Write on a read-only session | | `Capacity` | `Refused "Capacity"` | `Capacity("persistence")` | A bound or the disk margin was reached | | `Integrity` | `Malformed` | `Integrity("persistence")` | Stored or supplied data broke a rule | @@ -190,7 +190,7 @@ Source: [`records.rs`](../../../../engine/crates/layerfs-persistence/src/backend | Test or example | Command | Platform | Shows | | --- | --- | --- | --- | | [`heads.rs`](../../../../engine/crates/layerfs-persistence/tests/heads.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test heads` | macOS, Linux (no-sidecar check Linux only) | Project, fork and Commit change the heads exactly; a snapshot has the same heads | -| [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/tests/writer_admission.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test writer_admission` | macOS, Linux (cross-process cases Linux only) | FIFO order, `Closed` on close, writers queued behind the lane, self re-entry refused, directory lock hand-over and `Cancelled` | +| [`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/tests/writer_admission.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test writer_admission` | macOS, Linux (`Cancelled` Linux only) | FIFO order, `Closed` on close, writers queued behind the lane, self re-entry refused, the directory lock across processes and concurrent opens, its hand-over, and `Cancelled` | | [`diagnostics_during_lane.rs`](../../../../engine/crates/layerfs-persistence/tests/diagnostics_during_lane.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test diagnostics_during_lane` | any | Counter reads return while another thread holds the lane | | [`layerstack_removal.rs`](../../../../engine/crates/layerfs-persistence/tests/layerstack_removal.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test layerstack_removal` | macOS, Linux; also PostgreSQL when `LAYERFS_TEST_POSTGRES_URL` is set | Removing one of two projects leaves the other whole, frees the name and takes its logical bytes out of the totals | | [`binding_hint_blocks.rs`](../../../../engine/crates/layerfs-persistence/tests/binding_hint_blocks.rs) | `cargo +1.85.1 test --locked -p layerfs-persistence --test binding_hint_blocks` | macOS, Linux | Hints merge across blocks without repeats | @@ -202,4 +202,4 @@ Source: [`records.rs`](../../../../engine/crates/layerfs-persistence/src/backend | --- | --- | | [SQLite](sqlite.md) | The SQLite backend: schema, sessions, lane, profile, checkpoints, seal, snapshot, heads, acquisition and merge records. | | [PostgreSQL](postgresql.md) | The PostgreSQL backend: schema, open checks and authority, connections, transactions, publication, acquisition, access and status. | -| [Diagnostics](diagnostics.md) | Read-only Store audits: observations, the SQLite and PostgreSQL audits, table coverage and the structural fallback. | +| [Diagnostics](diagnostics.md) | Read-only Store audits: observations, the SQLite and PostgreSQL audits, table coverage and the structural fallback. | \ No newline at end of file diff --git a/docs/architecture/engine/store/sqlite.md b/docs/architecture/engine/store/sqlite.md index 5be13e1d3c..f2a04f2a8e 100644 --- a/docs/architecture/engine/store/sqlite.md +++ b/docs/architecture/engine/store/sqlite.md @@ -51,6 +51,7 @@ UPDATE and DELETE on `pack` and `pack_unit`. Open checks `application_id`, | Step in `Session::connect` | Detail | Source | | --- | --- | --- | +| Writer lock | When `writable`, takes `Writer::acquire(path)` across connect and releases it after `profile::check`; a read-only connect takes none | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Open | `READ_WRITE` or `READ_ONLY`, `NO_MUTEX`; statement cache 64 | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Guard | `busy_timeout=0`; `SQLITE_DBCONFIG_DEFENSIVE` on | same | | Settings | `profile::apply`, then read back into `ConnectionProfile`; a mismatch is `Integrity`; an existing Store not in `wal` mode is refused | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | @@ -82,8 +83,8 @@ SQL work to the calling thread's counters SQLite write: Session::run(true, body) 1 admission.enter in-process FIFO turn on this session 2 state mutex the connection - 3 Writer::acquire Linux: flock(LOCK_EX) on the Store's parent - directory; macOS: none (skipped if nested) + 3 Writer::acquire flock(LOCK_EX) on the Store's parent directory + (Linux and macOS) (skipped if nested) 4 BEGIN IMMEDIATE busy_timeout 0 -> Busy 5 body; COMMIT or ROLLBACK; Unknown -> quarantine 6 release the flock; drop the turn @@ -94,13 +95,20 @@ SQL work to the calling thread's counters inside: run(true, ..) is nested: no flock, BEGIN IMMEDIATE only ``` -The lock is taken on a separate descriptor of the Store's directory, never on -the main, WAL or SHM file: closing a main-file descriptor would drop -SQLite's own POSIX locks. Several Store files in one directory share it. A -holder process that exits releases it; a signal during the wait is -`Cancelled`. On macOS `flock` conflicts with SQLite's locks, so there is no -cross-process wait and a second writer gets `Busy`. A failed release on -`LaneHold` drop quarantines the session and keeps the lock +The lock is taken on a separate descriptor of the Store's directory (on +macOS, a `flock` on the main file itself would meet SQLite's own `fcntl` +locks), never on the main, WAL or SHM file: closing a main-file descriptor +would drop SQLite's own POSIX locks. Several Store files in one directory +share it. It is the same lane on Linux and macOS: a holder process that +exits releases it, a signal during the wait is `Cancelled`, and a writable +open takes it too, so concurrent first openers serialize their opening +PRAGMA statements instead of meeting SQLite's `Busy`; the schema statements +a first open runs then take it once per transaction. Only a writer that +bypasses the lane is left with the exact `Busy`. The lane is one held +admission turn plus that lock, kept by one thread across several short +transactions, and no publication of any process interleaves. `LaneHold` is +`!Send`: it belongs to the thread that took it; a failed release on its drop +quarantines the session and keeps the lock ([`writer_admission.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs), [`lane.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs)). @@ -115,7 +123,7 @@ cross-process wait and a second writer gets `Busy`. A failed release on StoreFile::checkpoint() (idle: last caller in the process released it) read-only? Refused "ReadOnly" -wal empty? Ok(true), no SQL this thread holds the lane? Ok(false) - Linux writer lane, one try; held elsewhere? Ok(false) + writer lane (Linux and macOS), one try; held elsewhere? Ok(false) wait for a running PASSIVE pass; wal_checkpoint(TRUNCATE) once busy (a reader in the log)? Ok(false) else Ok(true): -wal is 0 B ``` @@ -124,7 +132,7 @@ cross-process wait and a second writer gets `Busy`. A failed release on | --- | --- | | A pass that cannot copy every frame leaves them valid; the next wake, the idle checkpoint or the last close copies them. Each wake is a new pass, never a retry. | [`background.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs) | | A PASSIVE checkpoint takes no lock a writer needs, so it adds no `Busy`. | same | -| The idle TRUNCATE takes the writer lane first, because it holds SQLite's WAL write lock; `Ok(false)` is never retried. The Store stays open. | [`checkpoint.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs) | +| The idle TRUNCATE takes the writer lane first, because it holds SQLite's WAL write lock; `Ok(false)` is never retried. The Store stays open. Every host that opens a SQLite Store takes this lane, so a writer waits for it or the checkpoint is left, and only a writer that bypasses the lane meets exact `Busy`. | [`checkpoint.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs) | | Control's open-Store cache calls it when the last call or Sandbox on a Store returns. | [`open.rs`](../../../../control/crates/layerfs-store/src/open.rs) | ## Lifecycle @@ -135,11 +143,12 @@ cross-process wait and a second writer gets `Busy`. A failed release on own READ_ONLY connection storage still shared? Busy VACUUM INTO destination read-only? Refused "ReadOnly" reopen copy, journal_mode=wal quarantined / open tx? Uncertain - wal_checkpoint(TRUNCATE), close Linux: take the writer lock + wal_checkpoint(TRUNCATE), close take the writer lock (both OSes) no sidecar wal_checkpoint(TRUNCATE) or Busy \ close; sidecar left? Refused - \ macOS: release extents past EOF + \ release the writer lock +-------> SealedStore <-----------+ + macOS: release extents past EOF ``` | Operation | Bound or rule | Source | diff --git a/docs/architecture/flows/call.md b/docs/architecture/flows/call.md index afdc8ad845..42397c5b3f 100644 --- a/docs/architecture/flows/call.md +++ b/docs/architecture/flows/call.md @@ -53,9 +53,9 @@ retries. A lost answer after the request was sent is `Unknown`, never resent. | 4 | The home is opened (explicit path, else `LAYERFS_HOME`, else `~/.layerfs`). | [Home](../control/home/README.md) | | 5 | The family function reads the Store and project records, refuses a blocking intent and takes this process's lease on the Store. | [Stores and projects](../control/stores-and-projects/README.md#what-it-does), [Home: the `op` intent](../control/home/README.md#the-op-intent) | | 6 | Where the call runs is decided once from the record: the caller's process, or the Store machine's worker. | [Control: where each call runs](../control/README.md#where-each-call-runs) | -| 7a | Local: the process keeps the Store open and runs the method once through the Store service; the kernel guard runs before any file is opened. | [Store calls: opening the service](../engine/store-calls/README.md#opening-the-service), [Store calls: method table](../engine/store-calls/README.md#method-table) | +| 7a | Local: the process keeps the Store open and runs the method once through the Store service; the kernel guard runs before any file is opened, and a writable open takes the Store directory's writer lane (Linux and macOS), so two openers queue instead of meeting SQLite's busy lock. | [Store calls: opening the service](../engine/store-calls/README.md#opening-the-service), [Store calls: method table](../engine/store-calls/README.md#method-table) | | 7b | Worker: one loopback connection, the secret first, then one request; the worker holds its own lease and runs step 7a. | [Workers and reach](../control/machines/workers-and-reach.md), [Bridge](../engine/bridge/README.md) | -| 8 | Reads use the published read session and never take the lane; a write takes the session's admission turn and the Store's writer. | [Store calls: read sessions](../engine/store-calls/README.md#read-sessions), [Store: invariants](../engine/store/README.md#invariants) | +| 8 | Reads use the published read session and never take the lane; a write takes the session's admission turn and the Store's writer (a directory `flock` on Linux and macOS, or PostgreSQL's `store_lane` row). | [Store calls: read sessions](../engine/store-calls/README.md#read-sessions), [Store: invariants](../engine/store/README.md#invariants) | | 9 | The answer's keys become snake_case; events went out as they came; a refusal carries its stable code. | [API: calling conventions](../api/README.md#calling-conventions), [Errors and events](../api/calls/errors-and-events.md) | ## Related diff --git a/docs/architecture/glossary.md b/docs/architecture/glossary.md index 175b59dd4b..00cc847f3d 100644 --- a/docs/architecture/glossary.md +++ b/docs/architecture/glossary.md @@ -51,7 +51,7 @@ LayerFS (backend, install, scope), the row gives both. | Platform | A machine's kind (`native`, `docker`, `apple-container`, `lima`). It fixes the Sandbox backend, the worker's address and what the support table allows. | [Machines: platforms and profiles](control/machines/README.md#platforms-and-profiles), [Platforms](control/machines/platforms.md) | | Project | A named LayerStack in a Store, written `/`. | [Project Init](engine/project-init/README.md), [Projects](control/stores-and-projects/projects.md) | | Publish | `branch.publish`: puts a Branch's head Commit on `main` as a new Layer. | [Store calls: Branch history](engine/store-calls/README.md#branch-history), [SDK: `branch.publish`](api/sdk/branch.md#branchpublish) | -| Publish lane | One admission turn plus the Store's cross-process writer (the `flock` on the Store's directory on Linux; the `store_lane` row on PostgreSQL), held by a Commit across its head check, last merge step and publish. | [Store: the publish lane in a Commit](engine/store/README.md#the-publish-lane-in-a-commit) | +| Publish lane | One admission turn plus the Store's cross-process writer (the `flock` on the Store's directory on Linux and macOS; the `store_lane` row on PostgreSQL), held by a Commit across its head check, last merge step and publish. | [Store: the publish lane in a Commit](engine/store/README.md#the-publish-lane-in-a-commit) | | Reach | How a daemon on a machine dials a PostgreSQL Store: the caller's URL rewritten for that machine's platform, checked once at Sandbox create. | [Workers and reach: reach](control/machines/workers-and-reach.md#reach-reachrs) | | Sandbox | One daemon started by a backend on a machine, plus the commands run beside it. The daemon serves the Workspaces; the Sandbox runtime owns the commands. | [Sandboxes](control/sandboxes/README.md) | | Save | A Storage write session: it accepts objects, packs them and `Save::finish` completes them while the Store stays open. | [Physical storage: saves](core/physical-storage/README.md#saves) | diff --git a/docs/versioned/2.0.0/calls.md b/docs/versioned/2.0.0/calls.md index 851a952652..c315445ec6 100644 --- a/docs/versioned/2.0.0/calls.md +++ b/docs/versioned/2.0.0/calls.md @@ -563,7 +563,7 @@ Codes seen in the 2.0 library include: | `NoSuchPath`, `NoSuchSandbox`, `NoSuchProcess`, … | 1.1.0's codes for paths, Sandboxes and processes | | `Exists` | the name is taken, the same database or bucket is already connected (under any name), or a bucket object was exported from another home | | `InUse` | held: running Sandboxes, processes with the Store open, Stores on a machine; a layout 2 home with any of these, or with a move, removal or create still running, which is not converted yet ([the home](home.md#a-layout-2-home)) | -| `Busy` | an operation runs on the entity: a Store being pushed, pulled or removed, a machine being made or removed, a Sandbox being created or closed; the message names it (`op` id and phase) and its process (#536 §7.1); `detail` is `{entity: {kind, name}, phase, op}`. Also: another project is being added to the same Store | +| `Busy` | an operation runs on the entity: a Store being pushed, pulled or removed, a machine being made or removed, a Sandbox being created or closed; the message names it (`op` id and phase) and its process (#536 §7.1); `detail` is `{entity: {kind, name}, phase, op}`. Also: another project is being added to the same Store; a Store's writable open met SQLite's busy lock before any effect (the open wrote nothing, so the caller may repeat the call) | | `Unresolved` | an earlier operation on the entity ended with its outcome not known; its process is gone and its `op` stays. The message names the call that shows it (`store status`, `machine status`, `sandbox get`) and the one that clears it (`store remove`, `machine remove`, `sandbox close`); `detail` as for `Busy` | | `Unsupported` | not possible here, or not built (see [limitations](limitations.md)); a missing tool is listed in `detail.missing` | | `Unavailable`, `Denied` | a server, bucket, worker or host cannot be reached, or refused the login or secret | diff --git a/engine/AGENTS.md b/engine/AGENTS.md index 82b56a1ace..c363392515 100644 --- a/engine/AGENTS.md +++ b/engine/AGENTS.md @@ -454,8 +454,11 @@ fixtures, examples and tools do not contribute to production LOC. separate main/WAL/SHM descriptor for advisory ownership: closing a main-file descriptor can cancel SQLite's own process POSIX locks. This admission does not gate whole Saves or Commits or replay - a failed SQL attempt. macOS has Session admission only; separate macOS writers - and uncoordinated raw SQLite writers retain exact Busy refusal. Follow the + a failed SQL attempt. macOS holds the same parent-directory lane as Linux, + over the same short writer jobs and its writable opens (#647), so separate + macOS writers queue instead of meeting SQLite's Busy; only a writer that + bypasses the lane (an uncoordinated raw SQLite process) retains the exact + Busy refusal. Follow the [integration contract](https://github.com/Ephemeral-AI-Lab/layerfs/blob/af2ad92bce866b4e3e11365f016b12806828a53b/core/docs/issues/303/06-cluster-one-integration.md) and its [deepest-file plan](https://github.com/Ephemeral-AI-Lab/layerfs/blob/af2ad92bce866b4e3e11365f016b12806828a53b/core/docs/issues/307/SERVERLESS-STORE-PLAN-20261007.md); do not extend the host-mediated SDK runtime, Bridge data codec or daemon upstream. diff --git a/engine/crates/layerfs-bridge/src/process/darwin.rs b/engine/crates/layerfs-bridge/src/process/darwin.rs index 76f9a711e8..69b42c1ee1 100644 --- a/engine/crates/layerfs-bridge/src/process/darwin.rs +++ b/engine/crates/layerfs-bridge/src/process/darwin.rs @@ -7,12 +7,20 @@ use super::Stat; use nix::libc; use std::{ + ffi::CString, io, + os::unix::ffi::OsStrExt, + path::Path, process::{Command, Stdio}, sync::OnceLock, }; const SYSCTL: &str = "/usr/sbin/sysctl"; +/// Wait for volume writeback completion (`SYNC_VOLUME_WAIT = 0x02` in macOS +/// SDK ``; checked against ``, where `MNT_WAIT = 1` is +/// for `getfsstat(2)`/`vfs_sync` while `sync_volume_np(3)` and its flags live +/// in ``). +const SYNC_VOLUME_WAIT: libc::c_int = 0x02; /// One tool's standard output, or its failure with what it said. fn output(program: &str, args: &[&str]) -> io::Result<(bool, String)> { @@ -155,3 +163,22 @@ fn boottime(text: &str) -> Option { }; Some(field("sec")? * 1_000_000 + field("usec")?) } + +/// Flush buffered writes on the volume containing `path` and wait for +/// completion (`sync_volume_np(path, SYNC_VOLUME_WAIT)`). +#[allow(unsafe_code)] +pub fn sync_volume_wait(path: &Path) -> io::Result<()> { + extern "C" { + fn sync_volume_np(path: *const libc::c_char, flags: libc::c_int) -> libc::c_int; + } + let c_path = CString::new(path.as_os_str().as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains interior NUL"))?; + // SAFETY: `c_path` is a valid NUL-terminated C string alive for the call, + // and `SYNC_VOLUME_WAIT` is the documented wait flag for `sync_volume_np`. + let rc = unsafe { sync_volume_np(c_path.as_ptr(), SYNC_VOLUME_WAIT) }; + match rc { + 0 => Ok(()), + 1.. => Err(io::Error::from_raw_os_error(rc)), + _ => Err(io::Error::last_os_error()), + } +} diff --git a/engine/crates/layerfs-bridge/src/process/mod.rs b/engine/crates/layerfs-bridge/src/process/mod.rs index f06582ef97..2d8f629d6c 100644 --- a/engine/crates/layerfs-bridge/src/process/mod.rs +++ b/engine/crates/layerfs-bridge/src/process/mod.rs @@ -10,7 +10,7 @@ mod identity; #[cfg(target_os = "linux")] mod linux; #[cfg(target_os = "macos")] -pub use darwin::{boot, processes, stat}; +pub use darwin::{boot, processes, stat, sync_volume_wait}; pub use identity::{pid, LocalProcess, Stat}; #[cfg(target_os = "linux")] pub use linux::{boot, processes, stat}; diff --git a/engine/crates/layerfs-bridge/tests/process_identity.rs b/engine/crates/layerfs-bridge/tests/process_identity.rs index b425aa88be..49f68e2ca6 100644 --- a/engine/crates/layerfs-bridge/tests/process_identity.rs +++ b/engine/crates/layerfs-bridge/tests/process_identity.rs @@ -45,3 +45,17 @@ fn the_process_table_lists_this_process_in_its_group() { assert_eq!(listed.group, stat.group); assert!(process::pid(u32::MAX).is_err()); } + +#[cfg(target_os = "macos")] +#[test] +fn sync_volume_wait_flushes_existing_directory_and_rejects_missing_path() { + let dir = std::env::temp_dir().join(format!("layerfs-bridge-sync-vol-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + assert!(process::sync_volume_wait(&dir).is_ok()); + std::fs::remove_dir_all(&dir).unwrap(); + + let missing = dir.join("missing-subdir"); + let err = process::sync_volume_wait(&missing).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); +} diff --git a/engine/crates/layerfs-daemon/src/control/export.rs b/engine/crates/layerfs-daemon/src/control/export.rs index c92759b3b7..ead80216dd 100644 --- a/engine/crates/layerfs-daemon/src/control/export.rs +++ b/engine/crates/layerfs-daemon/src/control/export.rs @@ -10,8 +10,9 @@ //! - Ready carries the same receipt record; an NFS receipt is recognized by //! `abi_major == 0`, and its `mount_id` is the export's TCP port. //! - Commit first has the client write back what it buffered (the flush -//! helper, `sync(2)` by default) and closes the export's implicit file -//! descriptors, then runs the unchanged Commit. A Commit that installed +//! helper, else the export volume's own flush: `sync_volume_np(..., +//! SYNC_VOLUME_WAIT)` on macOS, `sync(2)` elsewhere) and closes the +//! export's implicit file descriptors, then runs the unchanged Commit. A Commit that installed //! another root closes the export's descriptors again; the client //! revalidates attributes on its own schedule (close-to-open, `actimeo`). //! - Unmount asks the helper to unmount (a busy answer is reversible and the @@ -77,8 +78,9 @@ pub struct Mounter { pub mount: Helper, pub unmount: Helper, pub force: Helper, - /// Writes back what the client buffered before a Commit; `sync(2)` of - /// the daemon itself when absent. + /// Writes back what the client buffered before a Commit; `sync_volume_np` + /// of the export directory on macOS (its failure refuses the Commit) or + /// `sync(2)` elsewhere when absent. pub flush: Option, } impl Mounter { @@ -583,7 +585,11 @@ impl Service { return Ok(()); }; match &serving.config.mounter.flush { - None => nix::unistd::sync(), + None => { + if let Err(detail) = flush_default(std::path::Path::new(&directory)) { + return Err(kernel_failure(ControlCode::Failed, "commit:flush", detail)); + } + } Some(helper) => { match serving .helpers @@ -932,3 +938,18 @@ pub(super) fn take(kernel: &mut Kernel, phase: MountPhase) -> Result Result<(), String> { + layerfs_bridge::process::sync_volume_wait(directory) + .map_err(|error| format!("sync_volume_np({}): {error}", directory.display())) +} + +#[cfg(not(target_os = "macos"))] +fn flush_default(_directory: &std::path::Path) -> Result<(), String> { + nix::unistd::sync(); + Ok(()) +} diff --git a/engine/crates/layerfs-daemon/tests/support/installed_store.rs b/engine/crates/layerfs-daemon/tests/support/installed_store.rs index a8322b9b04..166b02dc24 100644 --- a/engine/crates/layerfs-daemon/tests/support/installed_store.rs +++ b/engine/crates/layerfs-daemon/tests/support/installed_store.rs @@ -16,7 +16,24 @@ use std::{ pub const BINDING: &[u8] = b"daemon-installed-store"; pub const CURSOR: [u8; 32] = [43; 32]; +/// Creates one directory owner-only (`0700`) as any directory holding a Store +/// file must be: the kernel guard refuses one that grants a group or other bit +/// (#663), and the daemon installs every Store that way. +fn private_directory(path: &std::path::Path) { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path).unwrap(); + } + #[cfg(not(unix))] + std::fs::create_dir(path).unwrap(); +} +/// A sealed Store installed as a consumer sees it. pub struct Fixture { + /// Everything this fixture owns. It stays traversable (`0755`): mounted + /// Workspaces, the overlay template and the kernel source live here, and + /// tests run commands as another uid inside them. Only `store/` below is + /// private. pub directory: PathBuf, pub config: PersistenceConfig, pub branch: BranchId, @@ -37,10 +54,12 @@ impl Fixture { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); std::fs::create_dir(&directory).unwrap(); + let store_directory = directory.join("store"); + private_directory(&store_directory); let source = directory.join("source"); std::fs::create_dir(&source).unwrap(); let count = build(&source); - let sealed = directory.join("sealed.sqlite"); + let sealed = store_directory.join("sealed.sqlite"); let config = PersistenceConfig::sqlite(&sealed) .with_sqlite_profile(SqlitePersistenceProfile::Disposable) .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); @@ -85,7 +104,7 @@ impl Fixture { .unwrap(); drop(storage); let sealed = handles.seal().unwrap(); - let installed = directory.join("store.sqlite"); + let installed = store_directory.join("store.sqlite"); std::fs::copy(sealed.path, &installed).unwrap(); // No source or sealed original remains available to the consumer. std::fs::remove_dir_all(source).unwrap(); diff --git a/engine/crates/layerfs-nfs/tests/mounter.rs b/engine/crates/layerfs-nfs/tests/mounter.rs index 968ae2e0e0..2daffa7624 100644 --- a/engine/crates/layerfs-nfs/tests/mounter.rs +++ b/engine/crates/layerfs-nfs/tests/mounter.rs @@ -1,6 +1,25 @@ -//! The native macOS client must not send atime updates the export ignores. +//! The native macOS client must not send atime updates the export ignores, +//! and a Commit with the default export flush (the export volume's own +//! `sync_volume_np` on macOS) runs when that flush succeeds and is refused, +//! with nothing committed, when it fails. #![cfg(target_os = "macos")] -use layerfs_daemon::control::Mounter; +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +use client::Client; +use layerfs_bridge::control::{CommitControl, ControlCode, Reply, Request, UnmountControl}; +use layerfs_daemon::control::{ExportServing, ExportServingConfig, Failure, Helper, Mounter}; +use layerfs_history::WorkspaceId; +use std::{ + fs, + net::{Ipv4Addr, SocketAddr}, + path::Path, + sync::Arc, + time::Duration, +}; #[test] fn host_mount_disables_access_time_updates() { @@ -15,3 +34,128 @@ fn host_mount_disables_access_time_updates() { // Keep the existing attribute-cache contract; no global noac tradeoff. assert!(options[1].split(',').any(|option| option == "actimeo=60")); } + +#[test] +fn default_flush_commits_with_the_export_volume_and_refuses_without_it() { + // What this observes, and what it cannot: the Workspace is driven through + // this test's own NFS RPC client, so the host's mount client holds no + // writeback for the flush to push and the syscall leaves no trace here. + // The two Commits below therefore check the contract that is observable: + // a flush that succeeds (`sync_volume_np` of the live directory) lets + // the Commit run, and one that fails (`ENOENT` after the directory is + // removed) refuses it as `Failed` at `commit:flush`, before it runs, and + // leaves the Workspace idle. `sync_volume_wait`'s `Ok`/error is covered + // where it lives, by `layerfs-bridge`'s `process_identity` test. + let base = std::env::temp_dir().join(format!("layerfs-nfs-mounter-{}", std::process::id())); + let source = base.join("src"); + let work = base.join("work"); + let mounts = base.join("mounts"); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&source).unwrap(); + fs::write(source.join("README.md"), b"base\n").unwrap(); + fs::create_dir_all(&mounts).unwrap(); + + let rig = rig::Rig::new(&source, &work, Duration::from_secs(60)); + let noop = Helper(vec!["sh".into(), "-c".into(), "true".into()]); + let mounter = Mounter { + mount: noop.clone(), + unmount: noop.clone(), + force: noop, + flush: None, + }; + let export = Arc::new(ExportServing::start(ExportServingConfig::new( + mounts.clone(), + 501, + 20, + mounter, + ))); + let service = rig.service(export); + let workspace = WorkspaceId::from_authority([41; 32]).unwrap(); + let token = match service + .execute_control(&Request::Mount { + workspace, + branch: rig.branch, + }) + .unwrap() + .reply + { + Reply::Bound { token, .. } => token, + other => panic!("{other:?}"), + }; + let ready = match service + .execute_control(&Request::Attach(token)) + .unwrap() + .reply + { + Reply::Ready(ready) => *ready, + other => panic!("{other:?}"), + }; + let port = u16::try_from(ready.receipt.mount_id).unwrap(); + let mut nfs = Client::connect(SocketAddr::from((Ipv4Addr::LOCALHOST, port)), 501, 20); + let root = nfs.mnt("/"); + + // 1. Live export directory: the default flush is the volume's own + // `sync_volume_np`, which succeeds; the Commit runs. + let f1 = nfs.touch(&root, b"one.txt", 0o644).unwrap(); + nfs.write_all(&f1.handle, 0, b"first\n", 6); + assert!(Path::new(&ready.directory).is_dir()); + match service + .execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("first".into()), + })) + .unwrap() + .reply + { + Reply::Committed(_) => {} + other => panic!("{other:?}"), + } + + // 2. Missing export directory: `sync_volume_np` answers `ENOENT`; the + // Commit is refused before it runs instead of skipping the writeback. + let f2 = nfs.touch(&root, b"two.txt", 0o644).unwrap(); + nfs.write_all(&f2.handle, 0, b"second\n", 7); + fs::remove_dir_all(&ready.directory).unwrap(); + assert!(!Path::new(&ready.directory).exists()); + match service.execute_control(&Request::Commit(CommitControl { + token, + force: false, + message: Some("second".into()), + })) { + Err(Failure::Kernel(failure)) => { + assert_eq!( + (failure.code, failure.phase), + (ControlCode::Failed, "commit:flush"), + "{}", + failure.detail + ); + assert!( + failure.detail.contains("sync_volume_np"), + "{}", + failure.detail + ); + } + Ok(outcome) => panic!("the Commit ran without its flush: {:?}", outcome.reply), + Err(other) => panic!("{other:?}"), + } + + fs::create_dir_all(&ready.directory).unwrap(); + drop(nfs); + match service + .execute_control(&Request::Unmount(UnmountControl { + token, + discard: true, + })) + .unwrap() + .reply + { + Reply::Unmounted(unmounted) => assert_eq!(unmounted, token), + other => panic!("{other:?}"), + } + let released = service.export_released(token).unwrap(); + drop(service); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(&base).unwrap(); +} diff --git a/engine/crates/layerfs-persistence/Cargo.toml b/engine/crates/layerfs-persistence/Cargo.toml index 2f0b22459e..090f56a2be 100644 --- a/engine/crates/layerfs-persistence/Cargo.toml +++ b/engine/crates/layerfs-persistence/Cargo.toml @@ -32,7 +32,7 @@ blake3 = { version = "=1.8.5", default-features = false } [target.'cfg(any(target_os = "macos", target_os = "linux"))'.dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs"] } -[target.'cfg(target_os = "linux")'.dev-dependencies] +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dev-dependencies] nix = { version = "=0.31.3", default-features = false, features = ["fs", "pthread", "signal"] } [target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] diff --git a/engine/crates/layerfs-persistence/src/backend/records.rs b/engine/crates/layerfs-persistence/src/backend/records.rs index 8d9d105f5d..9efc76cd55 100644 --- a/engine/crates/layerfs-persistence/src/backend/records.rs +++ b/engine/crates/layerfs-persistence/src/backend/records.rs @@ -114,7 +114,7 @@ pub(crate) enum BackendError { /// database, failed authentication): no session, so nothing ran. #[cfg_attr(not(feature = "postgresql"), allow(dead_code))] Rejected, - /// A signal interrupted the cross-process writer wait (Linux only). + /// A signal interrupted the cross-process writer wait (Linux or macOS). #[cfg_attr(not(target_os = "linux"), allow(dead_code))] Cancelled, ReadOnly, diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs index 8c9e0b32a9..1c31440b0b 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/lane.rs @@ -1,7 +1,7 @@ -//! The Store's publication lane: one connection turn and, on Linux, the -//! cooperative writer lock, held by one thread across a short read-decide-write -//! sequence. Its own SQL work runs as ordinary transactions inside the held -//! turn; no other publication of any process can interleave. +//! The Store's publication lane: one connection turn and the cooperative +//! writer lock (Linux and macOS), held by one thread across a short +//! read-decide-write sequence. Its own SQL work runs as ordinary transactions +//! inside the held turn; no other publication of any process can interleave. use super::writer_admission::Writer; use crate::backend::records::BackendError; use crate::backend::sqlite::session::connection::Session; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs index f9ed78e2d9..2c58af8ecb 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lane/writer_admission.rs @@ -1,7 +1,7 @@ -//! Linux local-volume cooperative ownership before one SQLite write attempt. +//! Cooperative local-volume ownership before one SQLite write attempt. use crate::backend::records::BackendError; use std::path::Path; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use { nix::{ errno::Errno, @@ -11,20 +11,22 @@ use { }; pub(crate) struct Writer { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] lock: Flock, } impl Writer { pub(crate) fn acquire(path: &Path) -> Result { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { // Never open another FD to SQLite's main/WAL/SHM files here: // closing any main-file FD cancels every process POSIX lock on - // that inode, including SQLite's live SHARED locks. An independent - // description of the existing parent directory coordinates short - // writer jobs without interfering with SQLite's descriptors. - // Multiple Store files in one directory share this coarse lane. + // that inode, including SQLite's live SHARED locks, and Darwin + // flock on that file conflicts with SQLite's own fcntl locks. An + // independent description of the existing parent directory + // coordinates short writer jobs without interfering with SQLite's + // descriptors. Multiple Store files in one directory share this + // coarse lane. let directory = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) @@ -47,13 +49,6 @@ impl Writer { Err(_) => Err(BackendError::Integrity), } } - #[cfg(target_os = "macos")] - { - // Darwin's flock conflicts with SQLite's open-file locks. Host Init - // is sole-owner; cross-session contention retains exact SQLite Busy. - let _ = path; - Ok(Self {}) - } #[cfg(not(any(target_os = "linux", target_os = "macos")))] { let _ = path; @@ -67,7 +62,7 @@ impl Writer { /// This is a single attempt, never a retry: no loop, sleep or busy handler /// follows it here, and a caller must not wrap it in one. pub(crate) fn try_acquire(path: &Path) -> Result, BackendError> { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { // The same independent directory description as `acquire`. let directory = path @@ -89,15 +84,15 @@ impl Writer { Err(_) => Err(BackendError::Integrity), } } - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { - // As `acquire`: no advisory lane off Linux. + // As `acquire`: no advisory lane on these platforms. Self::acquire(path).map(Some) } } pub(crate) fn release(self) -> Result<(), Self> { - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { match self.lock.unlock() { Ok(file) => { @@ -107,7 +102,7 @@ impl Writer { Err((lock, _)) => Err(Self { lock }), } } - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "macos")))] { Ok(()) } diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs index e0c9969e4b..e7259fb616 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs @@ -19,7 +19,10 @@ impl Session { if state.quarantined || !state.connection.is_autocommit() { return Err(PersistenceError::Uncertain); } - #[cfg(target_os = "linux")] + // The same advisory lane as a write transaction: a seal is the sole + // owner's offline handoff, so no other opener may write while the + // checkpoint, close and sidecar removal below run. + #[cfg(any(target_os = "linux", target_os = "macos"))] let state = { let mut state = state; state.writer_hold = @@ -46,9 +49,10 @@ impl Session { .map_err(|(_, error)| rows::error(error))?; // SQLite has checked-closed before advisory ownership is released. // Earlier error exits drop State.connection before its retained owner. - if let Some(writer) = state.writer_hold { - writer.release().map_err(|_| PersistenceError::Uncertain)?; - } + // The sidecar check below runs while the lane is still held: an opener + // waiting on it takes the same lane before it may create `-wal`, so + // releasing first would let one land between the close and this check + // and turn a sealed Store into a refused one. for suffix in ["-wal", "-shm", "-journal"] { let mut sidecar = self.path.as_os_str().to_os_string(); sidecar.push(suffix); @@ -66,6 +70,9 @@ impl Session { } } } + if let Some(writer) = state.writer_hold { + writer.release().map_err(|_| PersistenceError::Uncertain)?; + } #[cfg(target_os = "macos")] let bytes = super::seal_allocation::release(&self.path, identity)?; #[cfg(not(target_os = "macos"))] diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs index 8a9fa3ca4d..151d5e3da3 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/checkpoint.rs @@ -12,10 +12,10 @@ impl Session { /// empty (it was, or it is now). `Ok(false)`: it could not complete now /// and is left as it is (its frames stay valid; the last close checkpoints /// them): this thread holds the publication lane, another process holds - /// the writer lane (Linux, tried without waiting), or a reader in another - /// process still reads a snapshot in the log (`busy`). Nothing is retried - /// and no busy handler waits. Short holds: the FIFO turn and the - /// connection lock are held for this one statement only. + /// the writer lane (Linux or macOS, tried without waiting), or a reader + /// in another process still reads a snapshot in the log (`busy`). + /// Nothing is retried and no busy handler waits. Short holds: the FIFO + /// turn and the connection lock are held for this one statement only. pub(crate) fn checkpoint_idle(&self) -> Result { if !self.writable { return Err(BackendError::ReadOnly); diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs index 6744ceba04..3149ee5380 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs @@ -205,6 +205,16 @@ impl Session { if !cfg!(any(target_os = "macos", target_os = "linux")) { return Err(BackendError::Integrity); } + // A writable open runs PRAGMA and schema statements that write, so it + // takes the same local-volume lane a transaction takes: another + // opener waits for it instead of meeting SQLite's Busy (#534, #647). + // A read-only open takes no lane, and the lane is released below + // before this session serves its first call. + let writer = if writable { + Some(crate::backend::sqlite::lane::writer_admission::Writer::acquire(path)?) + } else { + None + }; let flags = if writable { OpenFlags::SQLITE_OPEN_READ_WRITE } else { @@ -310,6 +320,9 @@ impl Session { if !diagnostic { profile::check(&profile)?; } + if let Some(writer) = writer { + writer.release().map_err(|_| BackendError::Unknown)?; + } super::thread_work::add(&work.borrow()); let background = (writable && selected == SqlitePersistenceProfile::Disposable) .then(|| super::background::Checkpointer::start(path)) diff --git a/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs b/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs index cdd38e853a..9f1c0a6c4f 100644 --- a/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs +++ b/engine/crates/layerfs-persistence/tests/idle_checkpoint.rs @@ -160,7 +160,7 @@ fn a_reader_in_the_log_leaves_it_for_the_last_close() { assert_eq!(wal_bytes(&path), 0); } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn another_process_writer_lane_leaves_it_without_waiting() { use nix::fcntl::{Flock, FlockArg}; diff --git a/engine/crates/layerfs-persistence/tests/writer_admission.rs b/engine/crates/layerfs-persistence/tests/writer_admission.rs index eb9fce9fa5..77a90b227d 100644 --- a/engine/crates/layerfs-persistence/tests/writer_admission.rs +++ b/engine/crates/layerfs-persistence/tests/writer_admission.rs @@ -4,7 +4,7 @@ use layerfs_content::{FinalizedObject, ObjectRole}; use layerfs_history::HistoryCatalogConfig; use layerfs_persistence::{PersistenceConfig, SqliteHandles, SqlitePersistenceProfile}; use layerfs_storage::{location::PackInfo, port::*, Storage, StoragePolicy}; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] use std::{ io::{Read, Write}, os::unix::net::{UnixListener, UnixStream}, @@ -26,7 +26,7 @@ fn config(path: &Path) -> PersistenceConfig { fn create(path: &Path) -> SqliteHandles { SqliteHandles::create(config(path), StoragePolicy::frozen_default(), &authority()).unwrap() } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn open(path: &Path) -> SqliteHandles { SqliteHandles::open_writable( config(path), @@ -130,12 +130,12 @@ fn shared_session_reservation_waits_for_an_ordinary_public_read_owner() { assert_eq!(after.rollbacks, before.rollbacks); } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] struct Held { child: Child, socket: UnixStream, } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] impl Held { fn start(path: &Path, socket_path: &Path) -> Self { let listener = UnixListener::bind(socket_path).unwrap(); @@ -169,6 +169,7 @@ impl Held { } }; let mut held = Self { child, socket }; + held.socket.set_nonblocking(false).unwrap(); held.socket .set_read_timeout(Some(Duration::from_secs(3))) .unwrap(); @@ -189,11 +190,11 @@ impl Held { break; } assert!(Instant::now() < deadline, "child exit deadline"); - std::thread::yield_now(); + std::thread::yield_now() } } } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] impl Drop for Held { fn drop(&mut self) { let _ = self.child.kill(); @@ -201,8 +202,115 @@ impl Drop for Held { } } +/// The test children of one concurrency case, owned as one group: the wait +/// carries a single deadline for the whole set, a child that outlives it or +/// exits unsuccessfully fails the case, and dropping the group while any child +/// still runs stops it. A lane deadlock therefore fails the case instead of +/// hanging it or leaving a live child that holds the Store, and the failure +/// names the child that hung together with what the group wrote to stderr. +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct Children(Vec<(String, Child)>); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Children { + fn new() -> Self { + Self(Vec::new()) + } + fn push(&mut self, what: String, child: Child) { + self.0.push((what, child)); + } + /// Waits for every child, `seconds` for the group as a whole; the 60 s + /// ceiling stays under the per-invocation test limit. Any child failure or + /// timeout stops the entire group (`kill` + `wait`) and attaches every + /// child's stderr to the failure panic. + fn wait_all(mut self, seconds: u64) { + let deadline = Instant::now() + Duration::from_secs(seconds); + let mut early_failure = false; + let mut late = Vec::new(); + loop { + let mut any_running = false; + for (what, child) in self.0.iter_mut() { + match child.try_wait().unwrap() { + Some(status) if !status.success() => { + early_failure = true; + } + Some(_) => {} + None => { + any_running = true; + if Instant::now() >= deadline { + late.push(what.clone()); + } + } + } + } + if early_failure || !late.is_empty() || !any_running { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + if early_failure || !late.is_empty() { + self.stop(); + } + let details = self.collect_details(); + assert!( + late.is_empty(), + "the group's deadline passed with {late:?} still running\n{}", + details + .iter() + .map(|(line, _)| line.as_str()) + .collect::>() + .join("\n") + ); + assert!( + !early_failure && details.iter().all(|(_, ok)| *ok), + "opener failed:\n{}", + details + .into_iter() + .map(|(line, _)| line) + .collect::>() + .join("\n") + ); + } + fn collect_details(&mut self) -> Vec<(String, bool)> { + let mut details = Vec::new(); + for (what, child) in std::mem::take(&mut self.0) { + let output = child.wait_with_output().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + let line = format!("{what}: {}\n{}", output.status, stderr.trim_end()); + details.push((line, output.status.success())); + } + details + } + /// Stops every child and panics with `reason` plus each child's stderr. + fn abort(mut self, reason: impl std::fmt::Display) -> ! { + self.stop(); + let details = self + .collect_details() + .into_iter() + .map(|(line, _)| line) + .collect::>(); + panic!("{reason}\n{}", details.join("\n")); + } + /// Stops every child of the group, exited or not. + fn stop(&mut self) { + for (_, child) in self.0.iter_mut() { + let _ = child.kill(); + } + for (_, child) in self.0.iter_mut() { + let _ = child.wait(); + } + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for Children { + fn drop(&mut self) { + self.stop(); + } +} + #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn another_process_admission_precedes_the_single_sqlite_begin() { // Keep the owned AF_UNIX path below Darwin's sockaddr_un capacity. let temp = support::Temp::new("wa"); @@ -243,7 +351,7 @@ fn another_process_admission_precedes_the_single_sqlite_begin() { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn two_opened_handles_admit_eight_distinct_reservations_without_conflict() { let temp = support::Temp::new("writer-admission-two-handles"); let path = temp.join("store.sqlite"); @@ -373,7 +481,7 @@ fn queued_writers_outlive_the_old_window_and_all_succeed() { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn a_cross_process_waiter_outlives_the_old_window_and_succeeds() { let temp = support::Temp::new("wl"); let path = temp.join("store.sqlite"); @@ -497,7 +605,7 @@ fn writers_queue_behind_a_held_publication_lane_without_deadlock() { } #[test] -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn a_holder_process_that_exits_hands_the_lock_to_the_waiter() { let temp = support::Temp::new("wx"); let path = temp.join("store.sqlite"); @@ -606,3 +714,94 @@ fn a_thread_reentering_its_own_turn_is_refused_instead_of_deadlocking() { // The turn was released: the next call is admitted at once. assert!(handles.storage.reserve(reserve()).is_ok()); } + +/// Child half of `three_processes_open_and_reserve_concurrently`: it waits for +/// the parent's go on the inherited socket name, then opens the Store (which +/// takes the open-time lane) and makes twenty reservations there. +#[test] +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn concurrent_open_child() { + let Some(path) = std::env::var_os("LAYERFS_ADMISSION_OPEN_STORE") else { + return; + }; + let mut sock = + UnixStream::connect(std::env::var_os("LAYERFS_ADMISSION_OPEN_SOCK").unwrap()).unwrap(); + sock.set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + sock.set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); + sock.write_all(b"R").unwrap(); + let mut go = [0u8; 1]; + sock.read_exact(&mut go).unwrap(); + let handles = open(Path::new(&path)); + for _ in 0..20 { + handles.storage.reserve(reserve()).unwrap(); + } +} + +/// Three processes open the same Store at once: the open-time lane and the +/// transaction lane queue them instead of meeting SQLite's Busy, so every +/// open and every reservation of all three succeeds (#534, #647). +#[test] +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn three_processes_open_and_reserve_concurrently() { + let temp = support::Temp::new("wo"); + let path = temp.join("store.sqlite"); + create(&path).seal().unwrap(); + let sock_path = temp.join("s"); + let listener = UnixListener::bind(&sock_path).unwrap(); + listener.set_nonblocking(true).unwrap(); + let exe = std::env::current_exe().unwrap(); + let mut children = Children::new(); + for id in 0..3 { + let child = Command::new(&exe) + .args(["--exact", "concurrent_open_child", "--nocapture"]) + .env("LAYERFS_ADMISSION_OPEN_STORE", &path) + .env("LAYERFS_ADMISSION_OPEN_SOCK", &sock_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + children.push(format!("opener-{id}"), child); + } + let mut sockets = Vec::new(); + let deadline = Instant::now() + Duration::from_secs(10); + while sockets.len() < 3 { + match listener.accept() { + Ok((mut stream, _)) => { + stream.set_nonblocking(false).unwrap(); + // The handshake itself is bounded too: a child that connects + // and never writes `R` must fail the case, not wait forever. + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut ready = [0u8; 1]; + if let Err(e) = stream.read_exact(&mut ready) { + children.abort(format!("barrier read: {e}")); + } + if &ready != b"R" { + children.abort(format!("unexpected barrier byte: {ready:?}")); + } + sockets.push(stream); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(1)); + } + Err(e) => children.abort(format!("barrier accept: {e}")), + } + } + for mut stream in sockets { + if let Err(e) = stream.write_all(b"G") { + children.abort(format!("barrier write: {e}")); + } + } + children.wait_all(60); + let handles = open(&path); + let next = handles.storage.reserve(reserve()).unwrap(); + assert_eq!(next.first_pack_id, 61); + assert_eq!(next.first_ordinal, 61); +} diff --git a/engine/crates/layerfs-store-service/src/store/captured.rs b/engine/crates/layerfs-store-service/src/store/captured.rs index 9df08bbf7e..4e7ad30989 100644 --- a/engine/crates/layerfs-store-service/src/store/captured.rs +++ b/engine/crates/layerfs-store-service/src/store/captured.rs @@ -4,7 +4,7 @@ use super::pipeline::Piped; use super::{ BoundWorkspace, CapturedConstruction, CommitError, CommitFailure, CommitOptions, CommitSuccess, - PortError, ReleasedOwner, StoreOperation, + ReleasedOwner, StoreOperation, }; use crate::OwnerError; use layerfs_content::{ @@ -303,15 +303,12 @@ fn classify( (Err(label), _) => label, }; Err(match original { - Some(WorkspaceError::Service(service)) => match service.downcast::() { - Ok(owner) => CommitError::Owner(*owner), - Err(other) => match other.downcast::>() { - Ok(port) => CommitError::Content { - error: label, - provider: Some(*port), - }, - Err(other) => CommitError::Workspace(WorkspaceError::Service(other)), - }, + // The port's own typed cause: an owner job, the Store provider's + // failure, or a History failure that keeps its exact class (a + // definite pre-effect `Busy` is not an unknown outcome, #647). + Some(WorkspaceError::Service(service)) => match super::service_cause(service, label) { + Ok(error) => error, + Err(other) => CommitError::Workspace(WorkspaceError::Service(other)), }, Some(WorkspaceError::Content(error)) => CommitError::Content { error, diff --git a/engine/crates/layerfs-store-service/src/store/commit_types.rs b/engine/crates/layerfs-store-service/src/store/commit_types.rs index ccd0e87a17..014b1ded39 100644 --- a/engine/crates/layerfs-store-service/src/store/commit_types.rs +++ b/engine/crates/layerfs-store-service/src/store/commit_types.rs @@ -237,6 +237,29 @@ impl From for CommitError { Self::Workspace(error) } } +/// The exact original cause a Workspace service port carried, as the product +/// type it is: an owner job, the Store provider's failure, or a typed History +/// failure. A definite pre-effect History refusal (a `Busy` admission, for +/// one) must keep its own class here, so it is never read as an unknown +/// outcome (#647). Anything else stays a Workspace service failure. +pub fn service_cause( + service: Box, + label: ContentError, +) -> Result> { + match service.downcast::() { + Ok(owner) => Ok(CommitError::Owner(*owner)), + Err(other) => match other.downcast::>() { + Ok(port) => Ok(CommitError::Content { + error: label, + provider: Some(*port), + }), + Err(other) => match other.downcast::() { + Ok(history) => Ok(CommitError::History(*history)), + Err(other) => Err(other), + }, + }, + } +} pub fn owner_uncertain(error: &OwnerError) -> bool { match error { OwnerError::Unattempted { .. } => false, diff --git a/engine/crates/layerfs-store-service/src/store/mod.rs b/engine/crates/layerfs-store-service/src/store/mod.rs index 21beeb7bcc..dba2719b9b 100644 --- a/engine/crates/layerfs-store-service/src/store/mod.rs +++ b/engine/crates/layerfs-store-service/src/store/mod.rs @@ -38,3 +38,4 @@ pub use types::{ }; pub use commit_types::owner_uncertain; +pub use commit_types::service_cause; diff --git a/engine/crates/layerfs-store-service/src/store_service.rs b/engine/crates/layerfs-store-service/src/store_service.rs index abcc147fd3..57c4861eb0 100644 --- a/engine/crates/layerfs-store-service/src/store_service.rs +++ b/engine/crates/layerfs-store-service/src/store_service.rs @@ -44,10 +44,23 @@ pub enum OpenRefusal { Unsupported(Unsupported), /// The Store file is absent: no Sandbox installed it into the volume yet. Missing(String), + /// Another writer held the Store at open, before any effect. The call is + /// a definite refusal the caller may repeat; nothing was changed. + Busy(String), /// The Store or the overlay template failed to open. Failed(String), } +/// The refusal for a Store open that failed: a Busy open is a definite +/// pre-effect refusal, every other open failure is not. +fn open_refusal(error: layerfs_storage::StorageError) -> OpenRefusal { + let reason = format!("Store open: {error}"); + match error { + layerfs_storage::StorageError::Busy => OpenRefusal::Busy(reason), + _ => OpenRefusal::Failed(reason), + } +} + /// The Store and overlay owner of this process's Store calls. pub struct StoreService { pub opened: OpenedStore, @@ -101,7 +114,7 @@ impl StoreService { CACHE_BYTES, ReservationBlocks::default(), ) - .map_err(|error| OpenRefusal::Failed(format!("Store open: {error}")))?; + .map_err(open_refusal)?; Self::owning(opened, template) } @@ -124,7 +137,7 @@ impl StoreService { CACHE_BYTES, ReservationBlocks::default(), ) - .map_err(|error| OpenRefusal::Failed(format!("Store open: {error}")))?; + .map_err(open_refusal)?; Self::owning(opened, template) } diff --git a/engine/crates/layerfs-store-service/tests/commit_pipeline.rs b/engine/crates/layerfs-store-service/tests/commit_pipeline.rs index 54a1dfe108..9542cdfbc7 100644 --- a/engine/crates/layerfs-store-service/tests/commit_pipeline.rs +++ b/engine/crates/layerfs-store-service/tests/commit_pipeline.rs @@ -21,7 +21,7 @@ use layerfs_store_service::store::{ CommitPhase, CommitSuccess, }; use layerfs_store_service::telemetry::{mark_phases, Recording}; -use layerfs_workspace::{CapturedNamespace, Operation, Outcome, Position, Time, WorkspaceError}; +use layerfs_workspace::{CapturedNamespace, Operation, Outcome, Position, Time}; use std::{cell::Cell, rc::Rc, sync::Once}; const MIB: usize = 1024 * 1024; @@ -330,14 +330,21 @@ fn construction_failure_abandons_the_save() { &[("a.txt", 0, b"alpha\n"), ("c.bin", 0, &chunked)], ); let failure = failed(&fixture, &workspace); - let CommitError::Workspace(WorkspaceError::Service(cause)) = &failure.error else { - panic!("not the producer's own failure: {failure:?}"); - }; + // The producer's own first cause, kept as the product type it is: a + // definite pre-effect refusal is `History(Busy)`, never a Workspace + // service wrapper read as an unknown outcome (#647). + assert!( + matches!( + failure.error, + CommitError::History(layerfs_history::HistoryError::Busy) + ), + "not the producer's own refusal: {:?}", + failure.error + ); assert!( - cause - .downcast_ref::() - .is_some(), - "{cause:?}" + !failure.error.uncertain(), + "a refusal made before any effect is not unknown: {:?}", + failure.error ); fixture.cleanup(); } diff --git a/engine/crates/layerfs-store-service/tests/commit_service_busy.rs b/engine/crates/layerfs-store-service/tests/commit_service_busy.rs new file mode 100644 index 0000000000..b515109030 --- /dev/null +++ b/engine/crates/layerfs-store-service/tests/commit_service_busy.rs @@ -0,0 +1,47 @@ +//! #647: a definite pre-effect History refusal that reached the Commit through +//! a Workspace service port keeps its own class. The Store's file-digest write +//! meeting SQLite's Busy is `HistoryError::Busy` inside +//! `WorkspaceError::Service` (what `store/digests.rs` returns when +//! `record_file_digests` is refused); read as a `Workspace` service failure it +//! was answered as an unknown outcome and held every owner. +use layerfs_content::ContentError; +use layerfs_history::HistoryError; +use layerfs_store_service::store::{service_cause, CommitError}; + +/// The label the producer's Content result carries; only the provider branch +/// of the classification reads it. +fn label() -> ContentError { + ContentError::OutputRejected +} + +#[test] +fn a_refused_digest_write_classifies_as_history_busy() { + let service: Box = Box::new(HistoryError::Busy); + let classified = service_cause(service, label()).expect("the typed History cause is kept"); + match classified { + CommitError::History(HistoryError::Busy) => {} + other => panic!("{other:?}"), + } + assert!( + !classified.uncertain(), + "a refusal made before any effect is not an unknown outcome" + ); +} + +#[test] +fn a_history_cause_that_is_unknown_stays_unknown() { + let service: Box = Box::new(HistoryError::UnknownOutcome); + let classified = service_cause(service, label()).expect("the typed History cause is kept"); + assert!(matches!( + classified, + CommitError::History(HistoryError::UnknownOutcome) + )); + assert!(classified.uncertain()); +} + +#[test] +fn another_service_cause_is_not_taken_for_a_history_failure() { + let service: Box = + Box::new(std::io::Error::other("not a Store port")); + assert!(service_cause(service, label()).is_err()); +} diff --git a/engine/crates/layerfs-store-service/tests/kernel_guard.rs b/engine/crates/layerfs-store-service/tests/kernel_guard.rs index ab17acbba4..d9f8b9f4ca 100644 --- a/engine/crates/layerfs-store-service/tests/kernel_guard.rs +++ b/engine/crates/layerfs-store-service/tests/kernel_guard.rs @@ -111,6 +111,326 @@ fn direct_access_is_allowed_on_a_local_macos_filesystem() { std::fs::remove_dir_all(directory).unwrap(); } +#[allow(dead_code)] +#[path = "support/installed_store.rs"] +mod installed; + +/// An open that finds the Store held by another writer's unwritten transaction +/// is a definite pre-effect `Busy` the caller may repeat; any other open +/// failure stays `Failed`. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn open_direct_reports_busy_when_locked_and_failed_when_corrupt() { + let fixture = installed::Fixture::new(1, "open-refusal"); + let sql = rusqlite::Connection::open(&fixture.config.path).unwrap(); + sql.busy_timeout(std::time::Duration::ZERO).unwrap(); + sql.execute_batch("BEGIN IMMEDIATE").unwrap(); + let template = fixture.directory.join("overlay.sqlite"); + let busy = StoreService::open_direct( + &fixture.config.path, + installed::BINDING, + installed::CURSOR, + &template, + ); + match busy { + Err(OpenRefusal::Busy(reason)) => { + assert_eq!(reason, "Store open: Store busy before effect"); + } + Err(other) => panic!("expected OpenRefusal::Busy, got {other:?}"), + Ok(_) => panic!("expected OpenRefusal::Busy, Store opened"), + } + sql.execute_batch("ROLLBACK").unwrap(); + drop(sql); + + let corrupt = fixture.store_directory().join("corrupt.sqlite"); + std::fs::write(&corrupt, b"not a valid sqlite store").unwrap(); + let failed = + StoreService::open_direct(&corrupt, installed::BINDING, installed::CURSOR, &template); + match failed { + Err(OpenRefusal::Failed(reason)) => assert!(reason.starts_with("Store open:"), "{reason}"), + Err(other) => panic!("expected OpenRefusal::Failed, got {other:?}"), + Ok(_) => panic!("expected OpenRefusal::Failed, Store opened"), + } + fixture.cleanup(); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn open_concurrent_child() { + let Some(store) = std::env::var_os("LAYERFS_OPEN_CONCURRENT_STORE") else { + return; + }; + use layerfs_bridge::control::{SdkCall, SdkValue}; + use std::io::{Read, Write}; + use std::os::unix::net::UnixStream; + let id = std::env::var("LAYERFS_OPEN_CONCURRENT_ID").unwrap(); + let template = + std::path::PathBuf::from(std::env::var_os("LAYERFS_OPEN_CONCURRENT_TPL").unwrap()); + let mut sock = + UnixStream::connect(std::env::var_os("LAYERFS_OPEN_CONCURRENT_SOCK").unwrap()).unwrap(); + sock.set_read_timeout(Some(std::time::Duration::from_secs(30))) + .unwrap(); + sock.set_write_timeout(Some(std::time::Duration::from_secs(10))) + .unwrap(); + sock.write_all(b"R").unwrap(); + let mut go = [0u8; 1]; + sock.read_exact(&mut go).unwrap(); + let service = StoreService::open_direct( + Path::new(&store), + installed::BINDING, + installed::CURSOR, + &template, + ) + .unwrap_or_else(|e| panic!("agent-{id} open_direct: {e:?}")); + let call = |method: &str, fields: Vec<(&str, SdkValue)>| { + let mut map: Vec<(String, SdkValue)> = + fields.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(); + map.push(("project".to_owned(), SdkValue::text("installed"))); + layerfs_store_service::methods::call( + &service, + &SdkCall { + method: method.to_owned(), + args: SdkValue::Map(map), + }, + ) + .unwrap_or_else(|e| panic!("agent-{id} {method}: {e:?}")) + }; + let branch = format!("agent-{id}"); + call("branch.fork", vec![("name", SdkValue::text(&branch))]); + for step in 0..20 { + call( + "fs.write", + vec![ + ("branch", SdkValue::text(&branch)), + ("path", SdkValue::text(format!("/agent-{id}-{step}.txt"))), + ( + "bytes", + SdkValue::Bytes(format!("step {step}\n").into_bytes()), + ), + ( + "message", + SdkValue::text(format!("agent-{id} commit {step}")), + ), + ], + ); + } + service.close().unwrap(); +} + +/// The test children of one concurrency case, owned as one group: the wait +/// carries a single deadline for the whole set, a child that outlives it or +/// exits unsuccessfully fails the case, and dropping the group while any child +/// still runs stops it. A lane deadlock therefore fails the case instead of +/// hanging it or leaving a live child that holds the Store, and the failure +/// names the child that hung together with what the group wrote to stderr. +#[cfg(any(target_os = "linux", target_os = "macos"))] +struct Children(Vec<(String, std::process::Child)>); + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Children { + fn new() -> Self { + Self(Vec::new()) + } + fn push(&mut self, what: String, child: std::process::Child) { + self.0.push((what, child)); + } + /// Waits for every child, `seconds` for the group as a whole; the 60 s + /// ceiling stays under the per-invocation test limit. Any child failure or + /// timeout stops the entire group (`kill` + `wait`) and attaches every + /// child's stderr to the failure panic. + fn wait_all(mut self, seconds: u64) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(seconds); + let mut early_failure = false; + let mut late = Vec::new(); + loop { + let mut any_running = false; + for (what, child) in self.0.iter_mut() { + match child.try_wait().unwrap() { + Some(status) if !status.success() => { + early_failure = true; + } + Some(_) => {} + None => { + any_running = true; + if std::time::Instant::now() >= deadline { + late.push(what.clone()); + } + } + } + } + if early_failure || !late.is_empty() || !any_running { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + if early_failure || !late.is_empty() { + self.stop(); + } + let details = self.collect_details(); + assert!( + late.is_empty(), + "the group's deadline passed with {late:?} still running\n{}", + details + .iter() + .map(|(line, _)| line.as_str()) + .collect::>() + .join("\n") + ); + assert!( + !early_failure && details.iter().all(|(_, ok)| *ok), + "concurrent open_direct + 20 commits failed:\n{}", + details + .into_iter() + .map(|(line, _)| line) + .collect::>() + .join("\n") + ); + } + fn collect_details(&mut self) -> Vec<(String, bool)> { + let mut details = Vec::new(); + for (what, child) in std::mem::take(&mut self.0) { + let output = child.wait_with_output().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + let line = format!("{what}: {}\n{}", output.status, stderr.trim_end()); + details.push((line, output.status.success())); + } + details + } + /// Stops every child and panics with `reason` plus each child's stderr. + fn abort(mut self, reason: impl std::fmt::Display) -> ! { + self.stop(); + let details = self + .collect_details() + .into_iter() + .map(|(line, _)| line) + .collect::>(); + panic!("{reason}\n{}", details.join("\n")); + } + /// Stops every child of the group, exited or not. + fn stop(&mut self) { + for (_, child) in self.0.iter_mut() { + let _ = child.kill(); + } + for (_, child) in self.0.iter_mut() { + let _ = child.wait(); + } + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +impl Drop for Children { + fn drop(&mut self) { + self.stop(); + } +} + +/// Three processes open the same Store directly and each commits twenty times +/// at once: opening waits on the directory lane instead of failing, and no +/// Commit is lost or doubled. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn three_processes_open_direct_and_commit_concurrently() { + use std::io::{Read, Write}; + use std::os::unix::net::UnixListener; + use std::process::{Command, Stdio}; + use std::time::{Duration, Instant}; + + // A short fixture label and socket name: `sockaddr_un.sun_path` is 104 bytes on + // Darwin (103 of path), and this directory sits under the host's TMPDIR. + let fixture = installed::Fixture::new(2, "co"); + let sock_path = fixture.directory.join("s"); + let listener = UnixListener::bind(&sock_path).unwrap(); + listener.set_nonblocking(true).unwrap(); + + let exe = std::env::current_exe().unwrap(); + let mut children = Children::new(); + for id in 0..3 { + let overlay_dir = fixture.directory.join(format!("overlay-{id}")); + std::fs::create_dir(&overlay_dir).unwrap(); + let tpl = overlay_dir.join("overlay.sqlite"); + let child = Command::new(&exe) + .args(["--exact", "open_concurrent_child", "--nocapture"]) + .env("LAYERFS_OPEN_CONCURRENT_STORE", &fixture.config.path) + .env("LAYERFS_OPEN_CONCURRENT_ID", id.to_string()) + .env("LAYERFS_OPEN_CONCURRENT_TPL", &tpl) + .env("LAYERFS_OPEN_CONCURRENT_SOCK", &sock_path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + children.push(format!("agent-{id}"), child); + } + let mut sockets = Vec::new(); + let deadline = Instant::now() + Duration::from_secs(10); + while sockets.len() < 3 { + match listener.accept() { + Ok((mut stream, _)) => { + stream.set_nonblocking(false).unwrap(); + // The handshake itself is bounded too: a child that connects + // and never writes `R` must fail the case, not wait forever. + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut ready = [0u8; 1]; + if let Err(e) = stream.read_exact(&mut ready) { + children.abort(format!("barrier read: {e}")); + } + if &ready != b"R" { + children.abort(format!("unexpected barrier byte: {ready:?}")); + } + sockets.push(stream); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock && Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(1)); + } + Err(e) => children.abort(format!("barrier accept: {e}")), + } + } + for mut stream in sockets { + if let Err(e) = stream.write_all(b"G") { + children.abort(format!("barrier write: {e}")); + } + } + children.wait_all(60); + let verify_dir = fixture.directory.join("overlay-verify"); + std::fs::create_dir(&verify_dir).unwrap(); + let service = StoreService::open_direct( + &fixture.config.path, + installed::BINDING, + installed::CURSOR, + &verify_dir.join("overlay.sqlite"), + ) + .unwrap(); + for id in 0..3 { + let branch = format!("agent-{id}"); + let list = layerfs_store_service::methods::call( + &service, + &layerfs_bridge::control::SdkCall { + method: "branch.list_commits".to_owned(), + args: layerfs_bridge::control::SdkValue::map([ + ( + "project", + layerfs_bridge::control::SdkValue::text("installed"), + ), + ("branch", layerfs_bridge::control::SdkValue::text(&branch)), + ("pageSize", layerfs_bridge::control::SdkValue::Int(50)), + ]), + }, + ) + .unwrap(); + let Some(layerfs_bridge::control::SdkValue::List(items)) = list.get("items") else { + panic!("missing items: {list:?}"); + }; + assert_eq!(items.len(), 20, "branch {branch} commit count"); + } + service.close().unwrap(); + fixture.cleanup(); +} + #[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] fn direct_access_is_unsupported_off_linux() { diff --git a/engine/crates/layerfs-store-service/tests/support/installed_store.rs b/engine/crates/layerfs-store-service/tests/support/installed_store.rs index 8187f5c8a6..85d13828b7 100644 --- a/engine/crates/layerfs-store-service/tests/support/installed_store.rs +++ b/engine/crates/layerfs-store-service/tests/support/installed_store.rs @@ -16,13 +16,37 @@ use std::{ pub const BINDING: &[u8] = b"daemon-installed-store"; pub const CURSOR: [u8; 32] = [43; 32]; +/// Creates one directory owner-only (`0700`) as any directory holding a Store +/// file must be: the kernel guard refuses one that grants a group or other bit +/// (#663), and the daemon installs every Store that way. +fn private_directory(path: &std::path::Path) { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().mode(0o700).create(path).unwrap(); + } + #[cfg(not(unix))] + std::fs::create_dir(path).unwrap(); +} +/// A sealed Store installed as a consumer sees it. pub struct Fixture { + /// Everything this fixture owns. It stays traversable (`0755`): the kernel + /// source, the overlay template and any mount live here, and tests run + /// commands as another uid inside them. Only `store/` below is private. pub directory: PathBuf, pub config: PersistenceConfig, pub branch: BranchId, pub count: usize, } impl Fixture { + /// `directory/store`, where this fixture keeps its Store files: `0700`, as + /// every directory a daemon opens a Store in must be (#663). + pub fn store_directory(&self) -> &std::path::Path { + self.config + .path + .parent() + .expect("the fixture's Store file stands in its own directory") + } pub fn new(count: usize, label: &str) -> Self { Self::built(label, |source| { for n in 0..count { @@ -45,11 +69,13 @@ impl Fixture { ) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - private_directory(&directory); + std::fs::create_dir(&directory).unwrap(); + let store_directory = directory.join("store"); + private_directory(&store_directory); let source = directory.join("source"); std::fs::create_dir(&source).unwrap(); let count = build(&source); - let sealed = directory.join("sealed.sqlite"); + let sealed = store_directory.join("sealed.sqlite"); let config = PersistenceConfig::sqlite(&sealed) .with_sqlite_profile(SqlitePersistenceProfile::Disposable) .with_sqlite_acquisition(SqliteAcquisitionSchema::Tables); @@ -95,7 +121,7 @@ impl Fixture { .unwrap(); drop(storage); let sealed = handles.seal().unwrap(); - let installed = directory.join("store.sqlite"); + let installed = store_directory.join("store.sqlite"); std::fs::copy(sealed.path, &installed).unwrap(); // No source or sealed original remains available to the consumer. std::fs::remove_dir_all(source).unwrap(); @@ -116,8 +142,10 @@ impl Fixture { pub fn cloned(label: &str, store: &std::path::Path, count: usize) -> Self { let directory = std::env::temp_dir().join(format!("layerfs-installed-{}-{label}", std::process::id())); - private_directory(&directory); - let installed = directory.join("store.sqlite"); + std::fs::create_dir(&directory).unwrap(); + let store_directory = directory.join("store"); + private_directory(&store_directory); + let installed = store_directory.join("store.sqlite"); std::fs::copy(store, &installed).unwrap(); Self { directory, @@ -132,13 +160,6 @@ impl Fixture { std::fs::remove_dir_all(self.directory).unwrap(); } } -/// The fixture's own Store folder: private as every folder a daemon opens, -/// because `kernel_guard` refuses group or other access and an open through -/// `StoreService::open_direct` reaches the Store itself. -fn private_directory(path: &std::path::Path) { - use std::os::unix::fs::DirBuilderExt; - std::fs::DirBuilder::new().mode(0o700).create(path).unwrap(); -} pub fn bytes(n: usize) -> Vec { format!("complete installed file {n:06}\n").into_bytes()