From 1810b4e3d7f490ace518ace60104e4612cbbf3cd Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:09:45 +0000 Subject: [PATCH 1/7] refactor(nfs): move Released and Kernel::release into kernel/released.rs (lp73) Relocation only, no behavior change: `Kernel::release`, `Kernel::held`, `retained_count` and the `Released` struct move out of `kernel.rs` into `kernel/released.rs` verbatim, and `pub use released::Released` keeps every existing path working. This is the first commit of the AppleDouble split (LP636): `kernel.rs` would otherwise cross the 999-line ceiling once the companion work lands. No test or documentation change; the moved functions are the same text in a new module. Production LOC: 163505 -> 163511 (delta +6) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent 8cfb6e273 (163505) and this tree (163511). The +6 is the new module's own two `use` lines, its `impl Kernel` block and the `mod released;` / `pub use released::Released;` declaration in kernel.rs. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- engine/crates/layerfs-nfs/src/kernel.rs | 83 +---------------- .../crates/layerfs-nfs/src/kernel/released.rs | 92 +++++++++++++++++++ 2 files changed, 95 insertions(+), 80 deletions(-) create mode 100644 engine/crates/layerfs-nfs/src/kernel/released.rs diff --git a/engine/crates/layerfs-nfs/src/kernel.rs b/engine/crates/layerfs-nfs/src/kernel.rs index dd03a6ab8f..43e17fda5c 100644 --- a/engine/crates/layerfs-nfs/src/kernel.rs +++ b/engine/crates/layerfs-nfs/src/kernel.rs @@ -56,6 +56,9 @@ use std::{ }, }; +mod released; +pub use released::Released; + /// Implicit file descriptors kept open at once. pub const FILES_HELD: usize = 1024; /// Implicit directory descriptors (enumerations) kept open at once. @@ -776,61 +779,6 @@ impl Kernel { drop(wait(services.close_directory(self.mount, serial, handle))); } } - /// Gives back everything the synthetic kernel holds: descriptors first, - /// then lookup references. Run once, before the engine mount is revoked. - /// Retained mutation failures keep their custody and are only counted. - pub fn release(&self) -> Released { - let table = std::mem::take(&mut *self.table()); - let companions = self.companions_release(); - let released = Released { - companions, - files: table.files.len(), - directories: table.directories.len(), - lookups: table.lookups.len(), - silly: table.silly.len(), - retained: self.retained_count(), - silly_renamed: self.silly_renamed.load(Ordering::Relaxed), - names_refused: self.names_refused.load(Ordering::Relaxed), - names_detached: self.names_detached.load(Ordering::Relaxed), - collected: self.collected.load(Ordering::Relaxed), - resolved: self.resolved.load(Ordering::Relaxed), - unresolved: self.unresolved.load(Ordering::Relaxed), - }; - for (serial, held) in table.files { - self.close_file(serial, held.handle); - } - for (handle, (serial, _)) in table.directories { - self.close_directory(serial, handle); - } - for serial in table.lookups { - self.forget(serial, 1); - } - released - } - fn retained_count(&self) -> usize { - self.retained - .lock() - .unwrap_or_else(|poison| poison.into_inner()) - .len() - } - /// Current holdings, for diagnostics. - pub fn held(&self) -> Released { - let table = self.table(); - Released { - companions: self.companions_held(), - files: table.files.len(), - directories: table.directories.len(), - lookups: table.lookups.len(), - silly: table.silly.len(), - retained: self.retained_count(), - silly_renamed: self.silly_renamed.load(Ordering::Relaxed), - names_refused: self.names_refused.load(Ordering::Relaxed), - names_detached: self.names_detached.load(Ordering::Relaxed), - collected: self.collected.load(Ordering::Relaxed), - resolved: self.resolved.load(Ordering::Relaxed), - unresolved: self.unresolved.load(Ordering::Relaxed), - } - } /// Closes every implicit file descriptor that no silly name needs, so /// nothing the export opened is held across a Commit. Returns how many. pub fn close_files(&self) -> usize { @@ -906,31 +854,6 @@ fn collectable(table: &mut Table, keep: usize) -> Vec { } candidates.into_iter().map(|(_, serial)| serial).collect() } -/// What [`Kernel::release`] gave back. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct Released { - pub files: usize, - pub directories: usize, - pub lookups: usize, - /// Silly names whose inode was kept alive until release. - pub silly: usize, - /// Failed mutations still holding engine custody. - pub retained: usize, - /// Client silly renames served as unlinks, over the export's life. - pub silly_renamed: u64, - /// Creations refused for their AppleDouble name, over the export's life. - pub names_refused: u64, - /// AppleDouble companions held in memory (see [`crate::apple_double`]). - pub companions: usize, - /// AppleDouble companions created, over the export's life. - pub names_detached: u64, - /// References collected beyond the bound, over the export's life. - pub collected: u64, - /// SqliteHandles of collected serials resolved again by their name. - pub resolved: u64, - /// SqliteHandles of collected serials whose name no longer reaches them. - pub unresolved: u64, -} fn oldest(entries: impl Iterator) -> Option { entries.min_by_key(|(_, tick)| *tick).map(|(key, _)| key) } diff --git a/engine/crates/layerfs-nfs/src/kernel/released.rs b/engine/crates/layerfs-nfs/src/kernel/released.rs new file mode 100644 index 0000000000..5259b14be9 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/kernel/released.rs @@ -0,0 +1,92 @@ +//! What the synthetic kernel gives back, counted. +//! +//! [`Kernel::release`] runs once, before the engine mount is revoked; [`Kernel::held`] +//! reports the same counts while the export serves, for diagnostics. +use super::Kernel; +use std::sync::atomic::Ordering; + +impl Kernel { + /// Gives back everything the synthetic kernel holds: descriptors first, + /// then lookup references. Run once, before the engine mount is revoked. + /// Retained mutation failures keep their custody and are only counted. + pub fn release(&self) -> Released { + let table = std::mem::take(&mut *self.table()); + let companions = self.companions_release(); + let released = Released { + companions, + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + }; + for (serial, held) in table.files { + self.close_file(serial, held.handle); + } + for (handle, (serial, _)) in table.directories { + self.close_directory(serial, handle); + } + for serial in table.lookups { + self.forget(serial, 1); + } + released + } + + fn retained_count(&self) -> usize { + self.retained + .lock() + .unwrap_or_else(|poison| poison.into_inner()) + .len() + } + + /// Current holdings, for diagnostics. + pub fn held(&self) -> Released { + let table = self.table(); + Released { + companions: self.companions_held(), + files: table.files.len(), + directories: table.directories.len(), + lookups: table.lookups.len(), + silly: table.silly.len(), + retained: self.retained_count(), + silly_renamed: self.silly_renamed.load(Ordering::Relaxed), + names_refused: self.names_refused.load(Ordering::Relaxed), + names_detached: self.names_detached.load(Ordering::Relaxed), + collected: self.collected.load(Ordering::Relaxed), + resolved: self.resolved.load(Ordering::Relaxed), + unresolved: self.unresolved.load(Ordering::Relaxed), + } + } +} + +/// What [`Kernel::release`] gave back. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct Released { + pub files: usize, + pub directories: usize, + pub lookups: usize, + /// Silly names whose inode was kept alive until release. + pub silly: usize, + /// Failed mutations still holding engine custody. + pub retained: usize, + /// Client silly renames served as unlinks, over the export's life. + pub silly_renamed: u64, + /// Creations refused for their AppleDouble name, over the export's life. + pub names_refused: u64, + /// AppleDouble companions held in memory (see [`crate::apple_double`]). + pub companions: usize, + /// AppleDouble companions created, over the export's life. + pub names_detached: u64, + /// References collected beyond the bound, over the export's life. + pub collected: u64, + /// SqliteHandles of collected serials resolved again by their name. + pub resolved: u64, + /// SqliteHandles of collected serials whose name no longer reaches them. + pub unresolved: u64, +} From 210d5f8abc8f4909737cd8f6cfcea81c383259fd Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:17:47 +0000 Subject: [PATCH 2/7] feat(nfs): decode the AppleDouble image a macOS client writes (lp73) The client lays a `._` companion out as an AppleDouble v2 image: a header, a FinderInfo entry that runs into an `ATTR` table of extended attributes, and a resource-fork entry at the end. `apple_double_codec` reads those fields (`Image::parse`) and, when the fields encode back to the very bytes the client wrote (`Image::compact`), keeps them as its own record, so no attribute, value or byte the layout carries is dropped, and one attribute's alignment padding survives a shorter name replacing a longer one. `Image::new`/`Image::encode` lay an image out as the client does, which is what makes the round trip exact and what the macOS evidence compares against. Malformed, truncated or differently laid-out images, and a resource fork with data, parse to `None` and are left as the client wrote them. This is the first layer of the AppleDouble split (LP636): the codec alone, with its own unit tests; the companion that holds its decoded form comes next. Tests: new `tests/apple_double_codec.rs`, 6 cases (byte-exact round trip for every value size, many attributes, alignment padding, whole units, a resource fork, and malformed images that must not decode), all pass. Production LOC: 163511 -> 163792 (delta +281) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent 1810b4e3d (163511) and this tree (163792). Engine +281 (`apple_double_codec.rs` +280, `lib.rs` +1). Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- .../layerfs-nfs/src/apple_double_codec.rs | 342 ++++++++++++++++++ engine/crates/layerfs-nfs/src/lib.rs | 1 + .../layerfs-nfs/tests/apple_double_codec.rs | 263 ++++++++++++++ 3 files changed, 606 insertions(+) create mode 100644 engine/crates/layerfs-nfs/src/apple_double_codec.rs create mode 100644 engine/crates/layerfs-nfs/tests/apple_double_codec.rs diff --git a/engine/crates/layerfs-nfs/src/apple_double_codec.rs b/engine/crates/layerfs-nfs/src/apple_double_codec.rs new file mode 100644 index 0000000000..1d01edabf3 --- /dev/null +++ b/engine/crates/layerfs-nfs/src/apple_double_codec.rs @@ -0,0 +1,342 @@ +//! The AppleDouble image a macOS NFSv3 client writes as `._`, and its +//! decoded form. +//! +//! The client lays an image out as an AppleDouble v2 header with two entries: +//! the FinderInfo, whose entry runs on into an `ATTR` table of extended +//! attributes (each name with its NUL, padded to four bytes, in entry order; +//! the values packed in the same order), and a resource fork at the end of +//! the image. The image length is a whole number of [`IMAGE_UNIT`] bytes, and +//! the bytes the layout leaves unused are zero. +//! +//! [`Image::parse`] reads those fields. [`Image::compact`] additionally +//! requires that [`Image::encode`] gives back the very same bytes, so a +//! companion held as an [`Image`] encodes again to exactly the image its +//! client wrote: no attribute, value or byte the layout carries is dropped. +//! Each attribute entry keeps the alignment padding the client left after its +//! name, because the client does not clear the bytes a shorter name replaced. +//! +//! Anything else - a resource fork with data, still another layout, a +//! half-written image - yields `None` and is stored as the client wrote it. + +use std::fmt; + +const MAGIC: u32 = 0x0005_1607; +const VERSION: u32 = 0x0002_0000; +const FILLER: &[u8; 16] = b"Mac OS X "; +const FINDER_TAG: u32 = 9; +const FORK_TAG: u32 = 2; +/// Bytes of FinderInfo. +pub const FINDER_INFO: usize = 32; +/// The FinderInfo's offset: after the header, the entry count and two entries. +const FINDER_AT: usize = 50; +/// The `ATTR` header: the FinderInfo and two bytes of padding follow the +/// header of the image. +const ATTR_AT: usize = FINDER_AT + FINDER_INFO + 2; +const ATTR_MAGIC: u32 = 0x4154_5452; +/// The first attribute entry, after the 36-byte `ATTR` header. +const ENTRIES_AT: usize = ATTR_AT + 36; +/// An attribute entry before its name: value offset, value length, flags, +/// name length in bytes including the NUL. +const ENTRY_FIXED: usize = 11; +/// The step an image grows by. +pub const IMAGE_UNIT: usize = 4096; +/// The empty resource fork the client writes at the end of an image. +const BLANK_FORK: usize = 286; +const BLANK_FORK_TAG: &[u8] = b"This resource fork intentionally left blank "; +/// Index bytes counted for one decoded image, besides its names and values. +const IMAGE_OVERHEAD: usize = 96; +/// Index bytes counted for one decoded attribute, besides its name and value. +const ATTR_OVERHEAD: usize = 56; + +/// One extended attribute of a decoded image. +#[derive(Clone, Eq, PartialEq)] +pub struct Attribute { + pub name: Vec, + pub value: Vec, + /// The bytes the client left between the name's NUL and the next entry. + pad: [u8; 3], +} +impl Attribute { + /// An attribute the client would write with a cleared pad. + pub fn new(name: &[u8], value: &[u8]) -> Self { + Self { + name: name.to_vec(), + value: value.to_vec(), + pad: [0; 3], + } + } +} +impl fmt::Debug for Attribute { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Attribute") + .field("name", &String::from_utf8_lossy(&self.name)) + .field("bytes", &self.value.len()) + .field("pad", &self.pad) + .finish() + } +} + +/// The resource-fork entry of an image that has no resource-fork data. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Fork { + /// A zero-length entry at the end of the image. + None, + /// The client's own empty resource fork, [`BLANK_FORK`] bytes. + Blank, +} +impl Fork { + fn len(self) -> usize { + match self { + Fork::None => 0, + Fork::Blank => BLANK_FORK, + } + } +} + +/// A decoded image: its FinderInfo, tag, extended attributes in entry order +/// and resource-fork entry, at its image length. +#[derive(Clone, Eq, PartialEq)] +pub struct Image { + finder: [u8; FINDER_INFO], + tag: u32, + attrs: Vec, + fork: Fork, + len: usize, +} +impl fmt::Debug for Image { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Image") + .field("tag", &self.tag) + .field("attrs", &self.attrs) + .field("fork", &self.fork) + .field("len", &self.len) + .finish() + } +} +impl Image { + /// The image a client would write for these attributes from scratch: the + /// smallest whole number of [`IMAGE_UNIT`] bytes with its empty resource + /// fork, and cleared padding. `None` for an empty name, a name holding a + /// NUL, a name an entry cannot describe, or more attributes than an entry + /// count can say. + pub fn new(finder: [u8; FINDER_INFO], tag: u32, attrs: Vec) -> Option { + let named = |name: &Vec| !name.is_empty() && name.len() < 255 && !name.contains(&0); + if attrs.len() > usize::from(u16::MAX) || !attrs.iter().all(|attr| named(&attr.name)) { + return None; + } + let mut image = Self { + finder, + tag, + attrs, + fork: Fork::Blank, + len: 0, + }; + let used = image.data_start() + image.data_len() + BLANK_FORK; + image.len = used.div_ceil(IMAGE_UNIT) * IMAGE_UNIT; + u32::try_from(image.len).ok()?; + Some(image) + } + /// Its FinderInfo. + pub fn finder_info(&self) -> [u8; FINDER_INFO] { + self.finder + } + /// Its attributes. + pub fn attributes(&self) -> &[Attribute] { + &self.attrs + } + /// Its resource-fork entry. + pub fn fork(&self) -> Fork { + self.fork + } + /// The image length the client wrote, in bytes. + pub fn len(&self) -> usize { + self.len + } + /// Whether no image has an empty length; here for `len`'s sake only. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + /// Index bytes this decoded form is counted for. + pub fn cost(&self) -> usize { + IMAGE_OVERHEAD + + self + .attrs + .iter() + .map(|attr| attr.name.len() + attr.value.len() + ATTR_OVERHEAD) + .sum::() + } + fn entry_end(&self, at: usize) -> usize { + ENTRIES_AT + + self + .attrs + .iter() + .take(at) + .map(|attr| entry_len(attr.name.len() + 1)) + .sum::() + } + fn data_start(&self) -> usize { + self.entry_end(self.attrs.len()) + } + fn data_len(&self) -> usize { + self.attrs.iter().map(|attr| attr.value.len()).sum() + } + /// The image these fields describe, as the client lays one out. + pub fn encode(&self) -> Vec { + let mut out = vec![0u8; self.len]; + let fork_at = self.len - self.fork.len(); + let data_start = self.data_start(); + put32(&mut out, 0, MAGIC); + put32(&mut out, 4, VERSION); + out[8..24].copy_from_slice(FILLER); + put16(&mut out, 24, 2); + put32(&mut out, 26, FINDER_TAG); + put32(&mut out, 30, FINDER_AT as u32); + put32(&mut out, 34, (fork_at - FINDER_AT) as u32); + put32(&mut out, 38, FORK_TAG); + put32(&mut out, 42, fork_at as u32); + put32(&mut out, 46, self.fork.len() as u32); + out[FINDER_AT..FINDER_AT + FINDER_INFO].copy_from_slice(&self.finder); + put32(&mut out, ATTR_AT, ATTR_MAGIC); + put32(&mut out, ATTR_AT + 4, self.tag); + put32(&mut out, ATTR_AT + 8, fork_at as u32); + put32(&mut out, ATTR_AT + 12, data_start as u32); + put32(&mut out, ATTR_AT + 16, self.data_len() as u32); + put16(&mut out, ATTR_AT + 34, self.attrs.len() as u16); + let (mut entry, mut value_at) = (ENTRIES_AT, data_start); + for attr in &self.attrs { + put32(&mut out, entry, value_at as u32); + put32(&mut out, entry + 4, attr.value.len() as u32); + out[entry + 10] = (attr.name.len() + 1) as u8; + let name_at = entry + ENTRY_FIXED; + let name_end = name_at + attr.name.len(); + out[name_at..name_end].copy_from_slice(&attr.name); + let next = entry + entry_len(attr.name.len() + 1); + out[name_end + 1..next].copy_from_slice(&attr.pad[..next - name_end - 1]); + out[value_at..value_at + attr.value.len()].copy_from_slice(&attr.value); + entry = next; + value_at += attr.value.len(); + } + if self.fork == Fork::Blank { + out[fork_at..].copy_from_slice(&blank_fork()); + } + out + } + /// The fields of `image`, laid out as the client writes one; [`Self::compact`] + /// also requires that they encode back to the same bytes. + pub fn parse(image: &[u8]) -> Option { + let len = image.len(); + u32::try_from(len).ok()?; + if len < ENTRIES_AT + || get32(image, 0)? != MAGIC + || get32(image, 4)? != VERSION + || image[8..24] != FILLER[..] + || get16(image, 24)? != 2 + || get32(image, 26)? != FINDER_TAG + || get32(image, 30)? as usize != FINDER_AT + || get32(image, 38)? != FORK_TAG + { + return None; + } + let fork = match get32(image, 46)? as usize { + 0 => Fork::None, + BLANK_FORK => Fork::Blank, + _ => return None, + }; + let fork_at = get32(image, 42)? as usize; + if fork_at.checked_add(fork.len())? != len + || fork_at < ENTRIES_AT + || get32(image, 34)? as usize != fork_at - FINDER_AT + || get32(image, ATTR_AT)? != ATTR_MAGIC + || get32(image, ATTR_AT + 8)? as usize != fork_at + { + return None; + } + let tag = get32(image, ATTR_AT + 4)?; + let data_start = get32(image, ATTR_AT + 12)? as usize; + let data_len = get32(image, ATTR_AT + 16)? as usize; + let count = get16(image, ATTR_AT + 34)?; + let (mut entry, mut value_at) = (ENTRIES_AT, data_start); + let mut attrs = Vec::with_capacity(usize::from(count)); + for _ in 0..count { + let value_len = get32(image, entry + 4)? as usize; + let name_len = usize::from(*image.get(entry + 10)?); + let name_at = entry + ENTRY_FIXED; + let name_end = entry + ENTRY_FIXED + name_len; + let name = image.get(name_at..name_end)?; + let (last, name) = name.split_last()?; + if get32(image, entry)? as usize != value_at || *last != 0 || name.is_empty() { + return None; + } + let value = image.get(value_at..value_at.checked_add(value_len)?)?; + let next = entry + entry_len(name_len); + let mut pad = [0u8; 3]; + pad[..next - name_end].copy_from_slice(image.get(name_end..next)?); + attrs.push(Attribute { + name: name.to_vec(), + value: value.to_vec(), + pad, + }); + entry = next; + value_at += value_len; + } + if entry != data_start + || value_at != data_start.checked_add(data_len)? + || value_at > fork_at + { + return None; + } + let mut finder = [0u8; FINDER_INFO]; + finder.copy_from_slice(image.get(FINDER_AT..FINDER_AT + FINDER_INFO)?); + Some(Self { + finder, + tag, + attrs, + fork, + len, + }) + } + /// The decoded form of `image` when it encodes back to exactly `image`. + pub fn compact(image: &[u8]) -> Option { + let decoded = Self::parse(image)?; + (decoded.encode() == image).then_some(decoded) + } +} + +/// An attribute entry's bytes for a name of `name_len` bytes with its NUL, +/// rounded up to four. +fn entry_len(name_len: usize) -> usize { + (ENTRY_FIXED + name_len + 3) & !3 +} +/// The empty resource fork: a resource map with no types, after the tag. +fn blank_fork() -> [u8; BLANK_FORK] { + let mut fork = [0u8; BLANK_FORK]; + for (at, value) in [ + (0, 0x100), + (4, 0x100), + (12, 0x1e), + (256, 0x100), + (260, 0x100), + ] { + put32(&mut fork, at, value); + } + put32(&mut fork, 268, 0x1e); + put16(&mut fork, 280, 0x1c); + put16(&mut fork, 282, 0x1e); + put16(&mut fork, 284, 0xffff); + fork[16..16 + BLANK_FORK_TAG.len()].copy_from_slice(BLANK_FORK_TAG); + fork +} +fn put32(out: &mut [u8], at: usize, value: u32) { + out[at..at + 4].copy_from_slice(&value.to_be_bytes()); +} +fn put16(out: &mut [u8], at: usize, value: u16) { + out[at..at + 2].copy_from_slice(&value.to_be_bytes()); +} +fn get32(bytes: &[u8], at: usize) -> Option { + Some(u32::from_be_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) +} +fn get16(bytes: &[u8], at: usize) -> Option { + Some(u16::from_be_bytes(bytes.get(at..at + 2)?.try_into().ok()?)) +} diff --git a/engine/crates/layerfs-nfs/src/lib.rs b/engine/crates/layerfs-nfs/src/lib.rs index f5cb633717..84154c19c1 100644 --- a/engine/crates/layerfs-nfs/src/lib.rs +++ b/engine/crates/layerfs-nfs/src/lib.rs @@ -17,6 +17,7 @@ #![forbid(unsafe_code)] pub mod apple_double; +pub mod apple_double_codec; pub mod attributes; mod block; pub mod change; diff --git a/engine/crates/layerfs-nfs/tests/apple_double_codec.rs b/engine/crates/layerfs-nfs/tests/apple_double_codec.rs new file mode 100644 index 0000000000..8bbfc57412 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/apple_double_codec.rs @@ -0,0 +1,263 @@ +//! The AppleDouble image codec on its own: what a macOS client's images +//! decode to, and that a decoded one encodes back to the very same bytes. +use layerfs_nfs::apple_double_codec::{Attribute, Fork, Image, IMAGE_UNIT}; + +const FILLER_AT: usize = 8; +const COUNT_AT: usize = 24; +const FINDER_TAG_AT: usize = 26; +const FINDER_ENTRY_AT: usize = 30; +const FINDER_ENTRY_LEN_AT: usize = 34; +const FORK_TAG_AT: usize = 38; +const FORK_AT: usize = 42; +const FORK_LEN_AT: usize = 46; +const FINDER_AT: usize = 50; +const ATTR_AT: usize = 84; +const ATTR_TAG_AT: usize = ATTR_AT + 4; +const ATTR_FORK_AT: usize = ATTR_AT + 8; +const ATTR_DATA_AT: usize = ATTR_AT + 12; +const ATTR_DATA_LEN_AT: usize = ATTR_AT + 16; +const ATTR_COUNT_AT: usize = ATTR_AT + 34; +const ENTRIES_AT: usize = ATTR_AT + 36; + +fn pattern(seed: u32, length: usize) -> Vec { + (0..length) + .map(|n| (n as u32).wrapping_mul(31).wrapping_add(seed * 97) as u8) + .collect() +} +fn image(finder: [u8; 32], tag: u32, attrs: &[(&[u8], Vec)]) -> Image { + let attrs = attrs + .iter() + .map(|(name, value)| Attribute::new(name, value)) + .collect(); + Image::new(finder, tag, attrs).expect("attributes an entry can describe") +} +fn get32(bytes: &[u8], at: usize) -> u32 { + u32::from_be_bytes(bytes[at..at + 4].try_into().unwrap()) +} +fn put32(bytes: &mut [u8], at: usize, value: u32) { + bytes[at..at + 4].copy_from_slice(&value.to_be_bytes()); +} +fn get16(bytes: &[u8], at: usize) -> u16 { + u16::from_be_bytes(bytes[at..at + 2].try_into().unwrap()) +} +/// The decoded form of `image`, required to encode back to the same bytes. +fn whole(image: &Image) -> Vec { + let bytes = image.encode(); + let decoded = Image::compact(&bytes).expect("a whole image decodes"); + assert_eq!(decoded.encode(), bytes, "decoded form re-encodes the image"); + assert_eq!(decoded.finder_info(), image.finder_info()); + assert_eq!(decoded.fork(), image.fork()); + assert_eq!(decoded.len(), bytes.len()); + bytes +} +/// Where the first attribute entry's name ends: its NUL byte. +fn first_name_end(name_len: usize) -> usize { + ENTRIES_AT + 11 + name_len +} + +#[test] +fn values_of_every_size_round_trip() { + for size in [0, 1, 4096, 64 * 1024, 100 * 1024] { + let value = pattern(3, size); + let name = format!("user.size-{size}"); + let attrs: &[(&[u8], Vec)] = &[(name.as_bytes(), value)]; + let built = image([1; 32], 0, attrs); + let bytes = whole(&built); + assert_eq!( + bytes.len() % IMAGE_UNIT, + 0, + "an image is whole {IMAGE_UNIT}-byte units" + ); + assert!(bytes.len() >= size); + let decoded = Image::compact(&bytes).unwrap(); + assert_eq!(decoded.attributes()[0].value.len(), size); + assert_eq!(&decoded.attributes()[0].value, &attrs[0].1); + assert_eq!(&decoded.attributes()[0].name, &attrs[0].0); + assert!(decoded.cost() >= size); + } +} + +#[test] +fn many_attributes_and_finder_info_round_trip() { + let finder = [9u8; 32]; + let attrs: Vec<(Vec, Vec)> = (0..12) + .map(|n| { + ( + format!("user.attr-{n:02}").into_bytes(), + pattern(n, 7 + 13 * n as usize), + ) + }) + .collect(); + let mut with_std = vec![ + (b"com.apple.provenance".to_vec(), pattern(21, 40)), + (b"com.apple.quarantine".to_vec(), pattern(22, 150)), + (b"com.apple.FinderInfo".to_vec(), finder.to_vec()), + ]; + with_std.extend(attrs); + let borrowed: Vec<(&[u8], Vec)> = with_std + .iter() + .map(|(name, value)| (name.as_slice(), value.clone())) + .collect(); + let built = image(finder, 0x2a2a_2a2a, &borrowed); + let bytes = whole(&built); + let decoded = Image::compact(&bytes).unwrap(); + assert_eq!(decoded.finder_info(), finder); + let names: Vec> = decoded + .attributes() + .iter() + .map(|attr| attr.name.clone()) + .collect(); + let expected: Vec> = with_std.iter().map(|(name, _)| name.clone()).collect(); + assert_eq!(names, expected, "entry order is kept"); + for (attr, (_, value)) in decoded.attributes().iter().zip(&with_std) { + assert_eq!(&attr.value, value); + } +} + +#[test] +fn the_clients_alignment_padding_is_kept() { + // A client that rewrote a shorter name does not clear what the longer one + // left behind: the bytes after the NUL stay, and the image must still + // decode to itself. + let attrs: &[(&[u8], Vec)] = &[ + (b"com.example.k4", pattern(5, 32)), + (b"user.other", pattern(6, 8)), + ]; + let mut bytes = image([2; 32], 0, attrs).encode(); + let name_end = first_name_end(b"com.example.k4".len()); + bytes[name_end + 1] = 0x5c; + bytes[name_end + 2] = 0x00; + let decoded = Image::compact(&bytes).expect("kept padding still decodes"); + assert_eq!( + decoded.encode(), + bytes, + "the padding is part of the decoded image" + ); + // A byte the layout leaves unused is not: such an image stays as written. + let mut foreign = bytes.clone(); + let fork_at = get32(&foreign, FORK_AT) as usize; + let data_len = get32(&foreign, ATTR_DATA_LEN_AT) as usize; + let data_start = get32(&foreign, ATTR_DATA_AT) as usize; + foreign[data_start + data_len..fork_at].fill(0x7f); + assert!( + Image::compact(&foreign).is_none(), + "a byte the decoded form does not keep is not claimed" + ); + assert!( + Image::parse(&foreign).is_some(), + "the fields still parse, only the round trip does not hold" + ); +} + +#[test] +fn a_resource_fork_is_left_as_the_client_wrote_it() { + // The client's own empty fork is part of the layout and decodes. + let built = image([3; 32], 0, &[(b"user.a", pattern(7, 16))]); + let bytes = whole(&built); + assert_eq!(Image::compact(&bytes).unwrap().fork(), Fork::Blank); + // The same entry holding other bytes is a real resource fork: the body is + // kept as written, so no fork byte is lost. + let mut fork_data = bytes.clone(); + fork_data[bytes.len() - 286..].fill(0x5a); + assert!(Image::parse(&fork_data).is_some()); + assert!(Image::compact(&fork_data).is_none()); + // A fork of another length is not the layout this codec describes. + let mut longer = bytes.clone(); + put32(&mut longer, FORK_LEN_AT, 300); + assert!(Image::parse(&longer).is_none()); + // An image with a zero-length fork entry decodes too. + let fork_at = get32(&bytes, FORK_AT) as usize; + let mut bare = bytes[..fork_at].to_vec(); + put32(&mut bare, FORK_LEN_AT, 0); + let decoded = Image::compact(&bare).expect("a fork-free image decodes"); + assert_eq!(decoded.fork(), Fork::None); + assert_eq!(decoded.encode(), bare); +} + +#[test] +fn malformed_images_are_not_decoded() { + let good = image( + [4; 32], + 0, + &[(b"user.a", pattern(8, 32)), (b"user.b", pattern(9, 4))], + ) + .encode(); + let fork_at = get32(&good, FORK_AT) as usize; + let data_start = get32(&good, ATTR_DATA_AT) as usize; + let mut cases: Vec<(&str, Vec)> = vec![ + ("empty", Vec::new()), + ("one byte", vec![0]), + ("a truncated header", good[..ENTRIES_AT - 1].to_vec()), + ("a truncated value", good[..data_start + 4].to_vec()), + ("a truncated image", good[..good.len() - 1].to_vec()), + ("an extended image", [good.clone(), vec![0]].concat()), + ]; + let mut each = |name: &'static str, at: usize, value: u32| { + let mut bytes = good.clone(); + put32(&mut bytes, at, value); + cases.push((name, bytes)); + }; + each("another magic", 0, 0x0005_1608); + each("another version", 4, 0x0003_0000); + each("another entry count", COUNT_AT, 3); + each("another FinderInfo tag", FINDER_TAG_AT, 8); + each("another FinderInfo offset", FINDER_ENTRY_AT, 51); + each("another FinderInfo length", FINDER_ENTRY_LEN_AT, 32); + each("another fork tag", FORK_TAG_AT, 1); + each( + "a fork before the entries", + FORK_AT, + (ENTRIES_AT - 1) as u32, + ); + each("a fork past the image", FORK_AT, (good.len() + 1) as u32); + each("another fork length", FORK_LEN_AT, 285); + each("another ATTR magic", ATTR_AT, 0x4154_5453); + each("another ATTR fork", ATTR_FORK_AT, (fork_at - 1) as u32); + each( + "a data start past the fork", + ATTR_DATA_AT, + (fork_at + 1) as u32, + ); + each("a longer data length", ATTR_DATA_LEN_AT, 1024); + each("another entry count", ATTR_COUNT_AT, 3); + each("another value offset", ENTRIES_AT, (data_start + 1) as u32); + let mut filler = good.clone(); + filler[FILLER_AT..FILLER_AT + 16].copy_from_slice(b"Mac OS "); + cases.push(("another filler", filler)); + let mut unterminated = good.clone(); + *unterminated + .get_mut(first_name_end(b"user.a".len())) + .unwrap() = b'X'; + cases.push(("a name without its NUL", unterminated)); + let mut nameless = good.clone(); + let name_len_at = ENTRIES_AT + 10; + assert_eq!(nameless[name_len_at], (b"user.a".len() + 1) as u8); + nameless[name_len_at] = 1; + cases.push(("an empty name", nameless)); + for (name, bytes) in cases { + assert!( + Image::compact(&bytes).is_none(), + "{name} is not claimed as a whole image" + ); + } +} + +#[test] +fn an_image_the_client_would_write_has_whole_units() { + let built = image([6; 32], 0, &[]); + let bytes = built.encode(); + assert_eq!(bytes.len() % IMAGE_UNIT, 0); + assert_eq!(bytes.len(), IMAGE_UNIT, "the smallest image is one unit"); + assert_eq!(get16(&bytes, COUNT_AT), 2); + assert_eq!(get32(&bytes, ATTR_TAG_AT), 0); + assert_eq!( + get32(&bytes, ATTR_DATA_AT) as usize, + ENTRIES_AT, + "no attribute, no entry" + ); + assert_eq!(bytes[FINDER_AT..FINDER_AT + 32], [6; 32]); + // An image whose FinderInfo the client filled and whose tag is its own. + let tagged = image([7; 32], 0x1234_5678, &[]); + let bytes = whole(&tagged); + assert_eq!(get32(&bytes, ATTR_TAG_AT), 0x1234_5678); +} From fa5dd34b5ff99a48aac38133deeac8573ca8e9e1 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:19:28 +0000 Subject: [PATCH 3/7] feat(nfs): hold a whole AppleDouble image as its own attributes (lp73) A macOS client writes each companion once, in full, and reads it back unchanged. A body is therefore classified whenever a write or a resize leaves it: one that parses back to the very bytes the companion holds (`Image::compact`) and fits `DECODED_ENTRY_BYTES` is kept as its FinderInfo and attributes and counted against `DECODED_BYTES` instead of its image, and every READ synthesizes the same bytes again. Anything else - a resource fork, another layout, a half-written image - is stored as the client wrote it. This replaces `TOTAL_BYTES` with two budgets that say what they bound: `RESIDENT_BYTES` (32 MiB) of written bodies kept in memory, and `DECODED_BYTES` (512 MiB) of decoded ones counted as `Image::cost`. A write or resize that neither decodes nor fits the resident budget answers `NFS3ERR_NOSPC` and leaves the companion exactly as it was; the next layer spills those bodies to disk instead of refusing them. `Released` now reports `companions_decoded` and `companion_decoded_bytes`, and `companions_release`/`companions_held` answer the `Holding` record the counters live in. Second layer of the AppleDouble split (LP636), on top of the codec. Tests: new `tests/apple_double_decode.rs` with the client's own write sequence (a half-written header, the rest, an edit that breaks the image and one that completes it again, truncate below and resize back up), read back byte for byte. Full `layerfs-nfs` suite: 41 passed, 0 failed. Production LOC: 163792 -> 163886 (delta +94) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent 210d5f8ab (163792) and this tree (163886). Engine +94. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- engine/crates/layerfs-nfs/src/apple_double.rs | 249 ++++++++++++++---- .../crates/layerfs-nfs/src/kernel/released.rs | 34 +-- .../layerfs-nfs/tests/apple_double_decode.rs | 142 ++++++++++ 3 files changed, 351 insertions(+), 74 deletions(-) create mode 100644 engine/crates/layerfs-nfs/tests/apple_double_decode.rs diff --git a/engine/crates/layerfs-nfs/src/apple_double.rs b/engine/crates/layerfs-nfs/src/apple_double.rs index 4f2e2eb4c8..c96d1c8b6a 100644 --- a/engine/crates/layerfs-nfs/src/apple_double.rs +++ b/engine/crates/layerfs-nfs/src/apple_double.rs @@ -16,11 +16,19 @@ //! directory it names, so its `wcc_data` answers both halves from the last //! facts the export saw for it (see [`Kernel::directory_wcc`]). //! -//! The memory is bounded: [`HELD`] companions of at most [`FILE_BYTES`] -//! each and [`TOTAL_BYTES`] in all. Beyond the count a creation is refused -//! (`NFS3ERR_ACCES`, as before this module), beyond the bytes a write -//! (`NFS3ERR_FBIG` per file, `NFS3ERR_NOSPC` in all). +//! Whenever a write leaves a companion holding a whole AppleDouble image the +//! client laid out itself (see [`crate::apple_double_codec`]), it is decoded: a +//! small one keeps only its FinderInfo and attributes in memory (at most +//! [`DECODED_BYTES`] in all, each at most [`DECODED_ENTRY_BYTES`]), and every +//! read synthesizes the same bytes again. +//! +//! The memory is bounded: [`HELD`] companions of at most [`FILE_BYTES`] each, +//! [`RESIDENT_BYTES`] of written bodies and [`DECODED_BYTES`] of decoded ones +//! counted as [`Image::cost`]. Beyond the count a creation is refused +//! (`NFS3ERR_ACCES`, as before this module), beyond a file's bytes a write +//! (`NFS3ERR_FBIG`), beyond the resident body budget a write (`NFS3ERR_NOSPC`). use crate::{ + apple_double_codec::Image, attributes::Status, change::{How, Sattr, Tickets, Wcc}, kernel::Kernel, @@ -41,23 +49,62 @@ pub const SERIALS: u64 = 1 << 62; pub const HELD: usize = 4096; /// Largest companion kept. pub const FILE_BYTES: u64 = 1 << 20; -/// Bytes kept over all companions. -pub const TOTAL_BYTES: usize = 64 << 20; +/// Companion payload bytes kept resident in memory before a write is refused. +pub const RESIDENT_BYTES: usize = 32 << 20; +/// Bytes counted for all decoded companions (see [`Image::cost`]). +pub const DECODED_BYTES: usize = 512 << 20; +/// Bytes counted for one decoded companion; a larger whole image keeps its +/// bytes as written. +pub const DECODED_ENTRY_BYTES: usize = 128 << 10; /// Mode of a companion created without one. const MODE: u32 = 0o644; +enum Body { + Resident(Vec), + /// A whole image kept as its FinderInfo and attributes; a read encodes the + /// image again, byte for byte as the client wrote it. + Decoded(Image), +} + struct Companion { + /// The `._` name it was created or last renamed to. named: (u64, Vec), - data: Vec, + len: usize, + body: Body, metadata: PortableMetadata, } +impl Companion { + fn resident_len(&self) -> usize { + match &self.body { + Body::Resident(bytes) => bytes.len(), + Body::Decoded(_) => 0, + } + } + fn decoded_cost(&self) -> usize { + match &self.body { + Body::Decoded(image) => image.cost(), + Body::Resident(_) => 0, + } + } +} + +/// What an export's companion index holds (see [`crate::kernel::Released`]). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Holding { + pub held: usize, + pub decoded: usize, + pub decoded_bytes: usize, +} + /// The companions of one export. #[derive(Default)] pub(crate) struct Companions { next: u64, files: HashMap, names: HashMap<(u64, Vec), u64>, - bytes: usize, + resident_bytes: usize, + decoded: usize, + decoded_bytes: usize, } impl Companions { pub(crate) fn len(&self) -> usize { @@ -69,31 +116,104 @@ impl Companions { kind: InodeKind::RegularFile, metadata: file.metadata, namespace_refs: 1, - logical_len: file.data.len() as u64, + logical_len: file.len as u64, subdirs: 0, } } + /// Adds `file` to the index and to the byte counts. + fn attach(&mut self, serial: u64, file: Companion) { + self.resident_bytes += file.resident_len(); + self.decoded += usize::from(matches!(file.body, Body::Decoded(_))); + self.decoded_bytes += file.decoded_cost(); + self.files.insert(serial, file); + } + /// Takes `serial` out of the index and the counts; its name key stays for + /// the caller to unindex. + fn detach(&mut self, serial: u64) -> Option { + let file = self.files.remove(&serial)?; + self.resident_bytes -= file.resident_len(); + self.decoded -= usize::from(matches!(file.body, Body::Decoded(_))); + self.decoded_bytes -= file.decoded_cost(); + Some(file) + } + /// Drops `serial` and its name key. fn drop_serial(&mut self, serial: u64) { - if let Some(file) = self.files.remove(&serial) { + if let Some(file) = self.detach(serial) { self.names.remove(&file.named); - self.bytes -= file.data.len(); } } - /// Resizes `serial` to `size`, within the bounds. - fn resize(&mut self, serial: u64, size: u64) -> Result<(), Status> { - if size > FILE_BYTES { - return Err(Status::FBig); + /// Decides whether `file`'s body is a whole image: one that parses back to + /// the same bytes and that the decoded budget takes. `file` must be + /// detached from the index. + fn classify(&self, file: &mut Companion) { + let Body::Resident(bytes) = &file.body else { + return; + }; + let Some(image) = Image::compact(bytes) else { + return; + }; + let cost = image.cost(); + if cost <= DECODED_ENTRY_BYTES && self.decoded_bytes + cost <= DECODED_BYTES { + file.body = Body::Decoded(image); } - let file = self.files.get_mut(&serial).ok_or(Status::Stale)?; - let old = file.data.len(); - let new = size as usize; - if new > old && self.bytes + (new - old) > TOTAL_BYTES { + } + /// Replaces `serial`'s body by `len` bytes built from it by `apply`, kept + /// within the budgets: a body that decodes costs its attributes, not its + /// bytes, and any other body stays in memory only while [`RESIDENT_BYTES`] + /// takes it. All of it or nothing: a body the budgets cannot take leaves + /// the companion exactly as it was. + fn edit_body( + &mut self, + serial: u64, + len: usize, + apply: impl FnOnce(&mut Vec), + ) -> Result<(), Status> { + let avail = RESIDENT_BYTES.saturating_sub(self.resident_bytes); + let mut file = self.detach(serial).ok_or(Status::Stale)?; + let need = len.saturating_sub(file.resident_len()); + let before_len = file.len; + let original = std::mem::replace(&mut file.body, Body::Resident(Vec::new())); + let mut bytes = match &original { + Body::Resident(bytes) => bytes.clone(), + Body::Decoded(image) => image.encode(), + }; + bytes.resize(len, 0); + apply(&mut bytes); + file.body = Body::Resident(bytes); + file.len = len; + self.classify(&mut file); + if need > avail && !matches!(file.body, Body::Decoded(_)) { + // The body does not decode and the resident budget does not take + // it: the companion keeps what it had and the caller answers NOSPC. + file.body = original; + file.len = before_len; + self.attach(serial, file); return Err(Status::NoSpc); } - file.data.resize(new, 0); - self.bytes = self.bytes + new - old; + touch(&mut file.metadata); + self.attach(serial, file); Ok(()) } + /// Applies a write to `serial`'s body; a write keeps what the body held + /// beyond its end. + fn write(&mut self, serial: u64, offset: usize, data: &[u8]) -> Result<(), Status> { + let file = self.files.get(&serial).ok_or(Status::Stale)?; + let len = file.len.max(offset + data.len()); + self.edit_body(serial, len, |bytes| { + bytes[offset..offset + data.len()].copy_from_slice(data); + }) + } + /// Resizes `serial` to `size`, within the bounds. + fn resize(&mut self, serial: u64, size: usize) -> Result<(), Status> { + self.edit_body(serial, size, |_| {}) + } + fn holding(&self) -> Holding { + Holding { + held: self.files.len(), + decoded: self.decoded, + decoded_bytes: self.decoded_bytes, + } + } } fn now() -> PortableMetadata { let elapsed = SystemTime::now() @@ -181,17 +301,15 @@ impl Kernel { let serial = SERIALS + companions.next; companions.next += 1; let named = (parent, name.to_vec()); - companions.names.insert(named.clone(), serial); - companions.files.insert( - serial, - Companion { - named, - data: Vec::new(), - metadata, - }, - ); - let file = &companions.files[&serial]; - let stat = Companions::stat(serial, file); + let file = Companion { + named, + len: 0, + body: Body::Resident(Vec::new()), + metadata, + }; + let stat = Companions::stat(serial, &file); + companions.names.insert(file.named.clone(), serial); + companions.attach(serial, file); drop(companions); self.names_detached.fetch_add(1, Ordering::Relaxed); Ok((stat, directory)) @@ -201,7 +319,7 @@ impl Kernel { pub(crate) fn companion_remove(&self, parent: u64, name: &[u8]) -> Option { { let mut companions = self.companions(); - let serial = companions.names.get(&(parent, name.to_vec())).copied()?; + let serial = *companions.names.get(&(parent, name.to_vec()))?; companions.drop_serial(serial); } Some(self.directory_wcc(parent)) @@ -244,10 +362,23 @@ impl Kernel { ) -> Result { let companions = self.companions(); let file = companions.files.get(&serial).ok_or(Status::Stale)?; - let length = file.data.len() as u64; + let length = file.len as u64; let start = offset.min(length) as usize; let end = offset.saturating_add(u64::from(count)).min(length) as usize; - out.extend_from_slice(&file.data[start..end]); + match &file.body { + Body::Resident(bytes) => { + // A body and its length are recorded under one lock; clamp + // anyway so no reply slices past the bytes it holds. + let stop = end.min(bytes.len()); + out.extend_from_slice(bytes.get(start..stop).unwrap_or_default()); + } + Body::Decoded(image) => { + // The image the client wrote, synthesized again. + let bytes = image.encode(); + let stop = end.min(bytes.len()); + out.extend_from_slice(bytes.get(start..stop).unwrap_or_default()); + } + } Ok(end as u64 >= length) } /// WRITE to a companion: all of it or nothing. @@ -257,18 +388,17 @@ impl Kernel { offset: u64, data: &[u8], ) -> Result<(u32, Wcc), Status> { - let mut companions = self.companions(); let end = offset.checked_add(data.len() as u64).ok_or(Status::FBig)?; + if end > FILE_BYTES { + return Err(Status::FBig); + } + let mut companions = self.companions(); let before = { let file = companions.files.get(&serial).ok_or(Status::Stale)?; Companions::stat(serial, file) }; - if end > before.logical_len { - companions.resize(serial, end)?; - } - let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; - file.data[offset as usize..end as usize].copy_from_slice(data); - touch(&mut file.metadata); + companions.write(serial, offset as usize, data)?; + let file = companions.files.get(&serial).ok_or(Status::Stale)?; let after = Companions::stat(serial, file); Ok((data.len() as u32, (Some(before), Some(after)))) } @@ -287,19 +417,24 @@ impl Kernel { if foreign { return Err(Status::Perm); } - let mut companions = self.companions(); let before = { + let companions = self.companions(); let file = companions.files.get(&serial).ok_or(Status::Stale)?; - Companions::stat(serial, file) - }; - if let Some(expected) = guard { - if crate::attributes::ctime(&before) != expected { - return Err(Status::NotSync); + let before = Companions::stat(serial, file); + if let Some(expected) = guard { + if crate::attributes::ctime(&before) != expected { + return Err(Status::NotSync); + } } - } + before + }; if let Some(size) = sattr.size { - companions.resize(serial, size)?; + if size > FILE_BYTES { + return Err(Status::FBig); + } + self.companions().resize(serial, size as usize)?; } + let mut companions = self.companions(); let file = companions.files.get_mut(&serial).ok_or(Status::Stale)?; if let Some(mode) = sattr.mode { file.metadata.mode = mode & 0o777; @@ -316,15 +451,15 @@ impl Kernel { let after = Companions::stat(serial, file); Ok((after.clone(), (Some(before), Some(after)))) } - /// Drops every companion; their count. - pub(crate) fn companions_release(&self) -> usize { + /// Drops every companion; what the index held. + pub(crate) fn companions_release(&self) -> Holding { let mut companions = self.companions(); - let held = companions.len(); + let holding = companions.holding(); *companions = Companions::default(); - held + holding } - /// Companions held now. - pub(crate) fn companions_held(&self) -> usize { - self.companions().len() + /// What the companion index holds now. + pub(crate) fn companions_held(&self) -> Holding { + self.companions().holding() } } diff --git a/engine/crates/layerfs-nfs/src/kernel/released.rs b/engine/crates/layerfs-nfs/src/kernel/released.rs index 5259b14be9..f765a6177d 100644 --- a/engine/crates/layerfs-nfs/src/kernel/released.rs +++ b/engine/crates/layerfs-nfs/src/kernel/released.rs @@ -2,7 +2,7 @@ //! //! [`Kernel::release`] runs once, before the engine mount is revoked; [`Kernel::held`] //! reports the same counts while the export serves, for diagnostics. -use super::Kernel; +use super::{Kernel, Table}; use std::sync::atomic::Ordering; impl Kernel { @@ -12,20 +12,7 @@ impl Kernel { pub fn release(&self) -> Released { let table = std::mem::take(&mut *self.table()); let companions = self.companions_release(); - let released = Released { - companions, - files: table.files.len(), - directories: table.directories.len(), - lookups: table.lookups.len(), - silly: table.silly.len(), - retained: self.retained_count(), - silly_renamed: self.silly_renamed.load(Ordering::Relaxed), - names_refused: self.names_refused.load(Ordering::Relaxed), - names_detached: self.names_detached.load(Ordering::Relaxed), - collected: self.collected.load(Ordering::Relaxed), - resolved: self.resolved.load(Ordering::Relaxed), - unresolved: self.unresolved.load(Ordering::Relaxed), - }; + let released = self.holdings(&table, companions); for (serial, held) in table.files { self.close_file(serial, held.handle); } @@ -48,8 +35,16 @@ impl Kernel { /// Current holdings, for diagnostics. pub fn held(&self) -> Released { let table = self.table(); + let companions = self.companions_held(); + self.holdings(&table, companions) + } + + /// What this export holds now, as [`Released`] reports it. + fn holdings(&self, table: &Table, companions: crate::apple_double::Holding) -> Released { Released { - companions: self.companions_held(), + companions: companions.held, + companions_decoded: companions.decoded, + companion_decoded_bytes: companions.decoded_bytes, files: table.files.len(), directories: table.directories.len(), lookups: table.lookups.len(), @@ -79,8 +74,13 @@ pub struct Released { pub silly_renamed: u64, /// Creations refused for their AppleDouble name, over the export's life. pub names_refused: u64, - /// AppleDouble companions held in memory (see [`crate::apple_double`]). + /// AppleDouble companions held in the export's index (see + /// [`crate::apple_double`]). pub companions: usize, + /// Of the companions held, those kept decoded, and the bytes counted for + /// them. + pub companions_decoded: usize, + pub companion_decoded_bytes: usize, /// AppleDouble companions created, over the export's life. pub names_detached: u64, /// References collected beyond the bound, over the export's life. diff --git a/engine/crates/layerfs-nfs/tests/apple_double_decode.rs b/engine/crates/layerfs-nfs/tests/apple_double_decode.rs new file mode 100644 index 0000000000..7acd47a903 --- /dev/null +++ b/engine/crates/layerfs-nfs/tests/apple_double_decode.rs @@ -0,0 +1,142 @@ +//! AppleDouble companions decoded into their attributes: a `._` a macOS +//! client writes as a whole image is kept as its FinderInfo and extended +//! attributes, and every read synthesizes the very bytes the client wrote. A +//! body that is not a whole image (a resource fork, another layout, a +//! half-written one) is stored as the client wrote it. +#[allow(dead_code)] +#[path = "support/client.rs"] +mod client; +#[allow(dead_code)] +#[path = "support/rig.rs"] +mod rig; +#[allow(dead_code)] +#[path = "support/spy.rs"] +mod spy; +#[allow(dead_code)] +#[path = "support/tree.rs"] +mod tree; +use client::{Client, Set, STALE}; +use layerfs_nfs::apple_double_codec::{Attribute, Image}; +use std::{ + fs, + net::TcpListener, + path::{Path, PathBuf}, + time::Duration, +}; + +const UID: u32 = 501; +const GID: u32 = 20; + +fn temp(label: &str) -> PathBuf { + std::env::temp_dir().join(format!("layerfs-nfs-ad{}-{label}", std::process::id())) +} +fn source(root: &Path) { + fs::create_dir_all(root.join("docs")).unwrap(); + fs::write(root.join("README.md"), b"# fixture\nhello nfs\n").unwrap(); + fs::write(root.join("docs/guide.md"), b"guide\n").unwrap(); +} +fn pattern(seed: u32, length: usize) -> Vec { + (0..length) + .map(|n| (n as u32).wrapping_mul(31).wrapping_add(seed * 97) as u8) + .collect() +} +/// The image a macOS client writes for these attributes. +fn image(finder: [u8; 32], attrs: &[(&[u8], Vec)]) -> Vec { + let attrs = attrs + .iter() + .map(|(name, value)| Attribute::new(name, value)) + .collect(); + Image::new(finder, 0, attrs) + .expect("an entry can describe these attributes") + .encode() +} + +#[test] +fn the_clients_write_sequence_is_served_byte_for_byte() { + let base = temp("sequence"); + let tree = base.join("tree"); + source(&tree); + let rig = rig::Rig::new(&tree, &base.join("rig"), Duration::from_secs(30)); + let server = rig.export(TcpListener::bind("127.0.0.1:0").unwrap(), UID, GID, false); + let mut c = Client::connect(server.local_addr(), UID, GID); + let root = c.mnt("/"); + + let main = c.touch(&root, b"seq", 0o644).unwrap(); + c.write(&main.handle, 0, b"data").unwrap(); + let ad = c.touch(&root, b"._seq", 0o644).unwrap(); + let whole = image( + [0x33; 32], + &[ + (b"com.apple.provenance", pattern(6, 40)), + (b"user.big", pattern(7, 4096)), + ], + ); + // The client writes the header first: a half-written image is served + // exactly as written and never claimed as a whole one. + c.write(&ad.handle, 0, &whole[..50]).unwrap(); + assert_eq!(c.getattr(&ad.handle).unwrap().size, 50); + assert_eq!(c.read_all(&ad.handle, 4096), whole[..50]); + assert_eq!(server.held().companions_decoded, 0); + // The rest arrives: the image is whole now and reads synthesize it. + c.write(&ad.handle, 50, &whole[50..]).unwrap(); + assert_eq!(c.read_all(&ad.handle, 64 * 1024), whole); + assert_eq!(server.held().companions_decoded, 1); + + // A delete and rebuild starts empty and decodes the rewrite again. + c.remove(&root, b"._seq").unwrap(); + assert_eq!(c.getattr(&ad.handle), Err(STALE)); + let again = c.touch(&root, b"._seq", 0o644).unwrap(); + assert_eq!(again.attr.size, 0); + assert_eq!(c.read_all(&again.handle, 4096), Vec::::new()); + let rebuilt = image( + [0x44; 32], + &[ + (b"com.apple.provenance", pattern(8, 40)), + (b"user.other", pattern(9, 64 * 1024)), + ], + ); + c.write(&again.handle, 0, &rebuilt).unwrap(); + assert_eq!(c.read_all(&again.handle, 128 * 1024), rebuilt); + + // An edit that leaves the image not whole is served as written, and a + // later edit that completes it decodes again: both are the client's own + // bytes. + let mut edited = rebuilt.clone(); + edited[24..28].copy_from_slice(b"EDIT"); + c.write(&again.handle, 24, b"EDIT").unwrap(); + assert_eq!(c.read_all(&again.handle, 128 * 1024), edited); + assert_eq!(server.held().companions_decoded, 0); + c.write(&again.handle, 24, &rebuilt[24..28]).unwrap(); + assert_eq!(c.read_all(&again.handle, 128 * 1024), rebuilt); + assert_eq!(server.held().companions_decoded, 1); + + // Truncating below the image is served as the client's truncation, and + // resizing back up serves zeros, as on any file. + let shrink = Set { + size: Some(64), + ..Set::default() + }; + c.setattr(&again.handle, shrink, None).unwrap(); + assert_eq!(c.read_all(&again.handle, 4096), rebuilt[..64]); + let grow = Set { + size: Some(rebuilt.len() as u64), + ..Set::default() + }; + c.setattr(&again.handle, grow, None).unwrap(); + let mut zeros = rebuilt[..64].to_vec(); + zeros.resize(rebuilt.len(), 0); + assert_eq!(c.read_all(&again.handle, 128 * 1024), zeros); + c.write(&again.handle, 0, &rebuilt).unwrap(); + assert_eq!(c.read_all(&again.handle, 128 * 1024), rebuilt); + + // The companion never reaches the Workspace. + let (entries, _) = c.list(&root, 64 * 1024, true); + assert!(entries.iter().all(|entry| !entry.name.starts_with(b"._"))); + drop(c); + let released = server.stop(); + assert_eq!(released.companions_decoded, 1); + let (_, outstanding) = rig.stop(released); + assert_eq!(outstanding, 0); + fs::remove_dir_all(base).unwrap(); +} + From 584cc3e66f34f749d82b85b922520f3abc347ff2 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:21:27 +0000 Subject: [PATCH 4/7] docs(nfs): document the decoded AppleDouble companion (lp73) The NFS route guide now names `apple_double_codec.rs` and states what each bound actually holds: `RESIDENT_BYTES` for written bodies and `DECODED_BYTES` / `DECODED_ENTRY_BYTES` for the decoded form, replacing the `TOTAL_BYTES` row the code no longer has. The paragraph after the table says which bodies decode (complete images that encode back to the same bytes) and which stay as written. Documentation only; it describes the codec layer as it stands. Production LOC: 163886 -> 163886 (delta 0) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent fa5dd34b5 (163886) and this tree (163886). Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- docs/architecture/engine/kernel-routes/nfs.md | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 4c8f4e0325..3b2db39e69 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -163,7 +163,7 @@ drawn once per export; it makes guessing costly and does not authenticate. | The refresh never writes into the Workspace and never reverts a mode the install changed | [`refresh.rs`](../../../../engine/crates/layerfs-nfs/src/refresh.rs) | | `LAYERFS_NFS_REFRESH=0` leaves the client to `actimeo` and close-to-open | [`export.rs`](../../../../engine/crates/layerfs-daemon/src/control/export.rs) | -## AppleDouble files (`apple_double.rs`) +## AppleDouble files (`apple_double.rs`, `apple_double_codec.rs`) On NFSv3 the macOS client keeps extended attributes and resource forks in `._` files. With `refuse_apple_double`, a regular `._` file (at least @@ -174,11 +174,20 @@ never lists it and the engine is never called. Any other kind is `NFS3ERR_ACCES`. `cp`, `copyfile`, `ditto` and `tar` succeed while nothing reaches the Workspace or a Commit; everything is dropped with the export. -| Bound | Value | Over the bound | -| --- | --- | --- | -| `HELD` | 4,096 companions | CREATE `NFS3ERR_ACCES` | -| `FILE_BYTES` | 1 MiB per companion | WRITE `NFS3ERR_FBIG` | -| `TOTAL_BYTES` | 64 MiB in all | WRITE `NFS3ERR_NOSPC` | +| Bound | Value | Over the bound | Source | +| --- | --- | --- | --- | +| `HELD` | 4,096 companions | CREATE `NFS3ERR_ACCES` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | +| `FILE_BYTES` | 1 MiB per companion | WRITE / SETATTR `NFS3ERR_FBIG` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | +| `RESIDENT_BYTES` | 32 MiB resident bodies in all | WRITE / SETATTR `NFS3ERR_NOSPC` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | +| `DECODED_BYTES` | 512 MiB over decoded companions | Further whole images keep their bytes as written | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | +| `DECODED_ENTRY_BYTES` | 128 KiB per decoded companion | Larger whole images keep their bytes as written | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | + +A body a write or resize leaves as a complete AppleDouble image that encodes +back to the exact bytes the companion holds +([`apple_double_codec.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double_codec.rs)) +is kept as its FinderInfo and attributes and counted against `DECODED_BYTES` +rather than its image, and every READ synthesizes the same bytes again. Partial +writes, resource forks and other layouts stay as the client wrote them. ## Mount (`mount.rs`, daemon `export.rs`) From b01e805988941d28e31eff3b6e652fdbea04e5ae Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:24:33 +0000 Subject: [PATCH 5/7] feat(nfs): encode only the window of the AppleDouble image a READ asks for (lp73) A decoded companion was re-encoded in full under the companion lock on every READ: up to `FILE_BYTES` (1 MiB) allocated and copied to serve a reply of at most `TRANSFER_WINDOW` (256 KiB) bytes, and often far less. `Image::encode_range` lays the image out and fills only `[start, end)`, so a READ pays for the bytes it answers with; `Image::encode` is that same layout over the whole image, so there is one implementation of the layout rather than two that can drift. `edit.rs`'s spill and `edit_resident` still need the whole image and keep calling `encode()`. Whether a READ is served from a window or from the whole image is not observable: the reply bytes are the same either way. Review: finding 7 (full re-encode on every READ). Tests: new `any_window_of_the_image_encodes_to_that_window` compares every window of an image (a padding byte the client left behind, values of several sizes, the empty resource fork) with the same slice of `encode()`, including empty windows, a window past the image and one that starts after it ends. `cargo test -p layerfs-nfs --test apple_double_codec --test apple_double_decode`: 7 + 1 passed, 0 failed. Anti-proof: with `encode_range` clamped to start at 0 again the window test fails (`left` starts at image offset 0 while the expected window starts mid-image). Production LOC: 163886 -> 163908 (delta +22) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent 584cc3e66 (163886) and of the staged tree 20b10a3487 (163908). Engine +22. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- engine/crates/layerfs-nfs/src/apple_double.rs | 8 +- .../layerfs-nfs/src/apple_double_codec.rs | 86 +++++++++++++------ .../layerfs-nfs/tests/apple_double_codec.rs | 53 ++++++++++++ 3 files changed, 117 insertions(+), 30 deletions(-) diff --git a/engine/crates/layerfs-nfs/src/apple_double.rs b/engine/crates/layerfs-nfs/src/apple_double.rs index c96d1c8b6a..7aa38ae4e2 100644 --- a/engine/crates/layerfs-nfs/src/apple_double.rs +++ b/engine/crates/layerfs-nfs/src/apple_double.rs @@ -373,10 +373,10 @@ impl Kernel { out.extend_from_slice(bytes.get(start..stop).unwrap_or_default()); } Body::Decoded(image) => { - // The image the client wrote, synthesized again. - let bytes = image.encode(); - let stop = end.min(bytes.len()); - out.extend_from_slice(bytes.get(start..stop).unwrap_or_default()); + // The image the client wrote, synthesized again over the + // window this reply asks for rather than in full. + let stop = end.min(image.len()); + out.extend_from_slice(&image.encode_range(start, stop)); } } Ok(end as u64 >= length) diff --git a/engine/crates/layerfs-nfs/src/apple_double_codec.rs b/engine/crates/layerfs-nfs/src/apple_double_codec.rs index 1d01edabf3..1e45130f78 100644 --- a/engine/crates/layerfs-nfs/src/apple_double_codec.rs +++ b/engine/crates/layerfs-nfs/src/apple_double_codec.rs @@ -14,6 +14,8 @@ //! client wrote: no attribute, value or byte the layout carries is dropped. //! Each attribute entry keeps the alignment padding the client left after its //! name, because the client does not clear the bytes a shorter name replaced. +//! [`Image::encode_range`] gives back any window of the image, so a READ serves +//! the bytes it asked for without laying out the whole image first. //! //! Anything else - a resource fork with data, still another layout, a //! half-written image - yields `None` and is stored as the client wrote it. @@ -184,42 +186,65 @@ impl Image { } /// The image these fields describe, as the client lays one out. pub fn encode(&self) -> Vec { - let mut out = vec![0u8; self.len]; + self.encode_range(0, self.len) + } + /// The bytes of the encoded image in `[start, end)`, both clamped to its + /// length: a READ of one window pays for that window, not for the whole + /// image, and gives back the same bytes [`Self::encode`] would. + pub fn encode_range(&self, start: usize, end: usize) -> Vec { + let start = start.min(self.len); + let end = end.max(start).min(self.len); + let mut out = vec![0u8; end - start]; let fork_at = self.len - self.fork.len(); let data_start = self.data_start(); - put32(&mut out, 0, MAGIC); - put32(&mut out, 4, VERSION); - out[8..24].copy_from_slice(FILLER); - put16(&mut out, 24, 2); - put32(&mut out, 26, FINDER_TAG); - put32(&mut out, 30, FINDER_AT as u32); - put32(&mut out, 34, (fork_at - FINDER_AT) as u32); - put32(&mut out, 38, FORK_TAG); - put32(&mut out, 42, fork_at as u32); - put32(&mut out, 46, self.fork.len() as u32); - out[FINDER_AT..FINDER_AT + FINDER_INFO].copy_from_slice(&self.finder); - put32(&mut out, ATTR_AT, ATTR_MAGIC); - put32(&mut out, ATTR_AT + 4, self.tag); - put32(&mut out, ATTR_AT + 8, fork_at as u32); - put32(&mut out, ATTR_AT + 12, data_start as u32); - put32(&mut out, ATTR_AT + 16, self.data_len() as u32); - put16(&mut out, ATTR_AT + 34, self.attrs.len() as u16); + // The fixed part of the image, whose offsets are absolute and known + // without walking the entries. + let mut head = [0u8; ENTRIES_AT]; + put32(&mut head, 0, MAGIC); + put32(&mut head, 4, VERSION); + head[8..24].copy_from_slice(FILLER); + put16(&mut head, 24, 2); + put32(&mut head, 26, FINDER_TAG); + put32(&mut head, 30, FINDER_AT as u32); + put32(&mut head, 34, (fork_at - FINDER_AT) as u32); + put32(&mut head, 38, FORK_TAG); + put32(&mut head, 42, fork_at as u32); + put32(&mut head, 46, self.fork.len() as u32); + head[FINDER_AT..FINDER_AT + FINDER_INFO].copy_from_slice(&self.finder); + put32(&mut head, ATTR_AT, ATTR_MAGIC); + put32(&mut head, ATTR_AT + 4, self.tag); + put32(&mut head, ATTR_AT + 8, fork_at as u32); + put32(&mut head, ATTR_AT + 12, data_start as u32); + put32(&mut head, ATTR_AT + 16, self.data_len() as u32); + put16(&mut head, ATTR_AT + 34, self.attrs.len() as u16); + emit(&mut out, start, 0, &head); let (mut entry, mut value_at) = (ENTRIES_AT, data_start); for attr in &self.attrs { - put32(&mut out, entry, value_at as u32); - put32(&mut out, entry + 4, attr.value.len() as u32); - out[entry + 10] = (attr.name.len() + 1) as u8; + // Entries and values only grow, and the fork sits past both: once + // neither of this one's is in the window, no later one is either. + if entry >= end && value_at >= end { + break; + } + let name_len = attr.name.len() + 1; + emit(&mut out, start, entry, &(value_at as u32).to_be_bytes()); + emit( + &mut out, + start, + entry + 4, + &(attr.value.len() as u32).to_be_bytes(), + ); + emit(&mut out, start, entry + 10, &[name_len as u8]); let name_at = entry + ENTRY_FIXED; let name_end = name_at + attr.name.len(); - out[name_at..name_end].copy_from_slice(&attr.name); - let next = entry + entry_len(attr.name.len() + 1); - out[name_end + 1..next].copy_from_slice(&attr.pad[..next - name_end - 1]); - out[value_at..value_at + attr.value.len()].copy_from_slice(&attr.value); + emit(&mut out, start, name_at, &attr.name); + let next = entry + entry_len(name_len); + emit(&mut out, start, name_end + 1, &attr.pad[..next - name_end - 1]); + emit(&mut out, start, value_at, &attr.value); entry = next; value_at += attr.value.len(); } if self.fork == Fork::Blank { - out[fork_at..].copy_from_slice(&blank_fork()); + emit(&mut out, start, fork_at, &blank_fork()); } out } @@ -328,6 +353,15 @@ fn blank_fork() -> [u8; BLANK_FORK] { fork[16..16 + BLANK_FORK_TAG.len()].copy_from_slice(BLANK_FORK_TAG); fork } +/// Copies the `bytes` that sit at `at` in the image into `out`, which holds +/// the image from `start` to its end. +fn emit(out: &mut [u8], start: usize, at: usize, bytes: &[u8]) { + let from = at.max(start); + let until = (at + bytes.len()).min(start + out.len()); + if from < until { + out[from - start..until - start].copy_from_slice(&bytes[from - at..until - at]); + } +} fn put32(out: &mut [u8], at: usize, value: u32) { out[at..at + 4].copy_from_slice(&value.to_be_bytes()); } diff --git a/engine/crates/layerfs-nfs/tests/apple_double_codec.rs b/engine/crates/layerfs-nfs/tests/apple_double_codec.rs index 8bbfc57412..b4d0c32098 100644 --- a/engine/crates/layerfs-nfs/tests/apple_double_codec.rs +++ b/engine/crates/layerfs-nfs/tests/apple_double_codec.rs @@ -55,6 +55,59 @@ fn first_name_end(name_len: usize) -> usize { ENTRIES_AT + 11 + name_len } +#[test] +fn any_window_of_the_image_encodes_to_that_window() { + // An image the client would write and decode: attributes whose names end + // on every alignment, values of several sizes, a padding byte the client + // left behind and its own empty resource fork. + let attrs: &[(&[u8], Vec)] = &[ + (b"user.a", pattern(11, 1)), + (b"com.example.long-name", pattern(12, 300)), + (b"user.c", pattern(13, 0)), + (b"com.apple.provenance", pattern(14, 40)), + (b"user.tail", pattern(15, 1000)), + ]; + let mut bytes = image([0x5a; 32], 0x0123_4567, attrs).encode(); + bytes[first_name_end(b"user.a".len()) + 1] = 0x3d; + let built = Image::compact(&bytes).expect("the image decodes"); + assert_eq!(built.encode(), bytes, "the whole image is unchanged"); + let len = bytes.len(); + let window = |start: usize, end: usize| { + let start = start.min(len); + let end = end.max(start).min(len); + bytes[start..end].to_vec() + }; + // Every window, in and past the image, byte for byte the same as the + // whole encoding: the header, each entry's offset, length and name, the + // padding after a name, each value, and the fork. + for start in (0..=len + 1).step_by(37) { + assert_eq!(built.encode_range(start, len + 1), window(start, len + 1)); + for end in (start..=len + 1).step_by(41) { + assert_eq!( + built.encode_range(start, end), + window(start, end), + "window {start}..{end} of {len} bytes" + ); + } + } + // An empty window, and a window that starts past the image. + assert_eq!(built.encode_range(0, 0), Vec::::new()); + assert_eq!(built.encode_range(len / 2, len / 2), Vec::::new()); + assert_eq!(built.encode_range(len + 10, len + 20), Vec::::new()); + assert_eq!(built.encode_range(7, 3), Vec::::new()); + assert_eq!(built.encode_range(0, len + 10), bytes); + assert_eq!(built.encode_range(len - 1, len + 10), bytes[len - 1..].to_vec()); + // The fork's own bytes are not all zero: a window inside it is served + // from the fork, not from the zeroed layout. + let fork_at = get32(&bytes, FORK_AT) as usize; + assert!(bytes[fork_at..].iter().any(|byte| *byte != 0)); + assert_eq!( + built.encode_range(fork_at, len), + bytes[fork_at..].to_vec(), + "the empty resource fork is part of the window" + ); +} + #[test] fn values_of_every_size_round_trip() { for size in [0, 1, 4096, 64 * 1024, 100 * 1024] { From 7ef50a034d87556b7cbe2bb17b5bcb7d6df2dcf1 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:28:43 +0000 Subject: [PATCH 6/7] refactor(nfs): count decoded AppleDouble images in the resident budget (lp73) `DECODED_BYTES` (512 MiB per export) was memory nothing could evict: many Workspaces can be mounted at once, so a per-export budget that is never paid back does not fit. Decoded images now take the same `RESIDENT_BYTES` (32 MiB) as bodies kept as written, a body at its length and a decoded one as `Image::cost`, so the budget sees every byte the export holds. `Companion::resident_cost` is that one number; `attach`, `detach` and `edit_body` count it, `edit_body` decides with it (a body either fits or the whole edit answers `NFS3ERR_NOSPC` and leaves the companion as it was), and `classify` no longer consults a second budget. `DECODED_ENTRY_BYTES` stays, now as the most `Image::cost` one decoded companion may be counted for: a whole image costing more keeps the client's bytes, so one companion cannot take the whole pool. This layer has no disk to spill to, so a body the budget does not take is still refused; the next layer spills it. `Released` reports the merged pool as `companion_bytes` and keeps `companions_decoded` as a count, instead of `companion_decoded_bytes` beside a separate resident total. Review: owner decision 3 (`DECODED_BYTES` rejected), first half (the budget and the counts); the spill half is the next layer. Tests: `the_clients_write_sequence_is_served_byte_for_byte` now asserts the merged pool: the decoded image's `Image::cost()` after the image arrives, the written body's length while it does not decode, the cost again once it does, and zero after the companion is removed. `cargo test -p layerfs-nfs`: 42 passed, 0 failed. Anti-proof: with a decoded body counted as zero again the test fails at the first assertion (`left: 0, right: 4372`). Production LOC: 163908 -> 163896 (delta -12) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent b01e80598 (163908) and of the staged tree bfc489bb4e (163896). Engine -12. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- docs/architecture/engine/kernel-routes/nfs.md | 14 ++-- engine/crates/layerfs-nfs/src/apple_double.rs | 68 +++++++++---------- .../crates/layerfs-nfs/src/kernel/released.rs | 10 +-- .../layerfs-nfs/tests/apple_double_decode.rs | 17 +++++ 4 files changed, 62 insertions(+), 47 deletions(-) diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 3b2db39e69..22ebe232a6 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -178,16 +178,18 @@ reaches the Workspace or a Commit; everything is dropped with the export. | --- | --- | --- | --- | | `HELD` | 4,096 companions | CREATE `NFS3ERR_ACCES` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | | `FILE_BYTES` | 1 MiB per companion | WRITE / SETATTR `NFS3ERR_FBIG` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | -| `RESIDENT_BYTES` | 32 MiB resident bodies in all | WRITE / SETATTR `NFS3ERR_NOSPC` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | -| `DECODED_BYTES` | 512 MiB over decoded companions | Further whole images keep their bytes as written | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | -| `DECODED_ENTRY_BYTES` | 128 KiB per decoded companion | Larger whole images keep their bytes as written | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | +| `RESIDENT_BYTES` | 32 MiB of companion bodies in all | WRITE / SETATTR `NFS3ERR_NOSPC` | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | +| `DECODED_ENTRY_BYTES` | 128 KiB counted for one decoded companion | A whole image costing more keeps its bytes as written | [`apple_double.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double.rs) | A body a write or resize leaves as a complete AppleDouble image that encodes back to the exact bytes the companion holds ([`apple_double_codec.rs`](../../../../engine/crates/layerfs-nfs/src/apple_double_codec.rs)) -is kept as its FinderInfo and attributes and counted against `DECODED_BYTES` -rather than its image, and every READ synthesizes the same bytes again. Partial -writes, resource forks and other layouts stay as the client wrote them. +is kept as its FinderInfo and attributes, and every READ synthesizes the same +bytes again. It takes the same `RESIDENT_BYTES` budget as a body kept as +written, counted as `Image::cost` (its names and values, not its image), so no +decoded image is memory the budget cannot see. Partial writes, resource forks +and other layouts stay as the client wrote them, and a body the budget does not +take leaves the companion as it was. ## Mount (`mount.rs`, daemon `export.rs`) diff --git a/engine/crates/layerfs-nfs/src/apple_double.rs b/engine/crates/layerfs-nfs/src/apple_double.rs index 7aa38ae4e2..3aff737e07 100644 --- a/engine/crates/layerfs-nfs/src/apple_double.rs +++ b/engine/crates/layerfs-nfs/src/apple_double.rs @@ -18,15 +18,17 @@ //! //! Whenever a write leaves a companion holding a whole AppleDouble image the //! client laid out itself (see [`crate::apple_double_codec`]), it is decoded: a -//! small one keeps only its FinderInfo and attributes in memory (at most -//! [`DECODED_BYTES`] in all, each at most [`DECODED_ENTRY_BYTES`]), and every -//! read synthesizes the same bytes again. +//! small one keeps only its FinderInfo and attributes in memory (each at most +//! [`DECODED_ENTRY_BYTES`] counted), and every read synthesizes the same bytes +//! again. //! //! The memory is bounded: [`HELD`] companions of at most [`FILE_BYTES`] each, -//! [`RESIDENT_BYTES`] of written bodies and [`DECODED_BYTES`] of decoded ones -//! counted as [`Image::cost`]. Beyond the count a creation is refused +//! and [`RESIDENT_BYTES`] of bodies in all, a body kept as written counted at +//! its length and a decoded one as [`Image::cost`], so a decoded image is never +//! memory the budget cannot see. Beyond the count a creation is refused //! (`NFS3ERR_ACCES`, as before this module), beyond a file's bytes a write -//! (`NFS3ERR_FBIG`), beyond the resident body budget a write (`NFS3ERR_NOSPC`). +//! (`NFS3ERR_FBIG`), beyond the body budget a write (`NFS3ERR_NOSPC`), which +//! leaves the companion exactly as it was. use crate::{ apple_double_codec::Image, attributes::Status, @@ -49,12 +51,12 @@ pub const SERIALS: u64 = 1 << 62; pub const HELD: usize = 4096; /// Largest companion kept. pub const FILE_BYTES: u64 = 1 << 20; -/// Companion payload bytes kept resident in memory before a write is refused. +/// Companion payload bytes kept resident in memory before a write is refused: +/// a body kept as written counts its length, a decoded one [`Image::cost`]. pub const RESIDENT_BYTES: usize = 32 << 20; -/// Bytes counted for all decoded companions (see [`Image::cost`]). -pub const DECODED_BYTES: usize = 512 << 20; -/// Bytes counted for one decoded companion; a larger whole image keeps its -/// bytes as written. +/// Most bytes counted for one decoded companion, [`Image::cost`]; a whole +/// image costing more keeps its bytes as written, so one companion cannot take +/// the whole budget. pub const DECODED_ENTRY_BYTES: usize = 128 << 10; /// Mode of a companion created without one. const MODE: u32 = 0o644; @@ -74,16 +76,12 @@ struct Companion { metadata: PortableMetadata, } impl Companion { - fn resident_len(&self) -> usize { + /// The bytes it takes of [`RESIDENT_BYTES`]: its body as written, or what + /// its decoded form is counted as. + fn resident_cost(&self) -> usize { match &self.body { Body::Resident(bytes) => bytes.len(), - Body::Decoded(_) => 0, - } - } - fn decoded_cost(&self) -> usize { - match &self.body { Body::Decoded(image) => image.cost(), - Body::Resident(_) => 0, } } } @@ -93,7 +91,7 @@ impl Companion { pub(crate) struct Holding { pub held: usize, pub decoded: usize, - pub decoded_bytes: usize, + pub resident_bytes: usize, } /// The companions of one export. @@ -104,7 +102,6 @@ pub(crate) struct Companions { names: HashMap<(u64, Vec), u64>, resident_bytes: usize, decoded: usize, - decoded_bytes: usize, } impl Companions { pub(crate) fn len(&self) -> usize { @@ -122,18 +119,16 @@ impl Companions { } /// Adds `file` to the index and to the byte counts. fn attach(&mut self, serial: u64, file: Companion) { - self.resident_bytes += file.resident_len(); + self.resident_bytes += file.resident_cost(); self.decoded += usize::from(matches!(file.body, Body::Decoded(_))); - self.decoded_bytes += file.decoded_cost(); self.files.insert(serial, file); } /// Takes `serial` out of the index and the counts; its name key stays for /// the caller to unindex. fn detach(&mut self, serial: u64) -> Option { let file = self.files.remove(&serial)?; - self.resident_bytes -= file.resident_len(); + self.resident_bytes -= file.resident_cost(); self.decoded -= usize::from(matches!(file.body, Body::Decoded(_))); - self.decoded_bytes -= file.decoded_cost(); Some(file) } /// Drops `serial` and its name key. @@ -143,7 +138,7 @@ impl Companions { } } /// Decides whether `file`'s body is a whole image: one that parses back to - /// the same bytes and that the decoded budget takes. `file` must be + /// the same bytes and that [`DECODED_ENTRY_BYTES`] takes. `file` must be /// detached from the index. fn classify(&self, file: &mut Companion) { let Body::Resident(bytes) = &file.body else { @@ -152,25 +147,23 @@ impl Companions { let Some(image) = Image::compact(bytes) else { return; }; - let cost = image.cost(); - if cost <= DECODED_ENTRY_BYTES && self.decoded_bytes + cost <= DECODED_BYTES { + if image.cost() <= DECODED_ENTRY_BYTES { file.body = Body::Decoded(image); } } /// Replaces `serial`'s body by `len` bytes built from it by `apply`, kept - /// within the budgets: a body that decodes costs its attributes, not its - /// bytes, and any other body stays in memory only while [`RESIDENT_BYTES`] - /// takes it. All of it or nothing: a body the budgets cannot take leaves - /// the companion exactly as it was. + /// within [`RESIDENT_BYTES`]: a body that decodes is counted as its + /// attributes, not as its bytes, and any other body stays in memory only + /// while the budget takes it. All of it or nothing: a body the budget + /// cannot take leaves the companion exactly as it was. fn edit_body( &mut self, serial: u64, len: usize, apply: impl FnOnce(&mut Vec), ) -> Result<(), Status> { - let avail = RESIDENT_BYTES.saturating_sub(self.resident_bytes); let mut file = self.detach(serial).ok_or(Status::Stale)?; - let need = len.saturating_sub(file.resident_len()); + let avail = RESIDENT_BYTES.saturating_sub(self.resident_bytes); let before_len = file.len; let original = std::mem::replace(&mut file.body, Body::Resident(Vec::new())); let mut bytes = match &original { @@ -182,9 +175,10 @@ impl Companions { file.body = Body::Resident(bytes); file.len = len; self.classify(&mut file); - if need > avail && !matches!(file.body, Body::Decoded(_)) { - // The body does not decode and the resident budget does not take - // it: the companion keeps what it had and the caller answers NOSPC. + if file.resident_cost() > avail { + // Neither as bytes nor as attributes does the body fit what the + // budget has left: the companion keeps what it had and the caller + // answers NOSPC. file.body = original; file.len = before_len; self.attach(serial, file); @@ -211,7 +205,7 @@ impl Companions { Holding { held: self.files.len(), decoded: self.decoded, - decoded_bytes: self.decoded_bytes, + resident_bytes: self.resident_bytes, } } } diff --git a/engine/crates/layerfs-nfs/src/kernel/released.rs b/engine/crates/layerfs-nfs/src/kernel/released.rs index f765a6177d..a333d56fb5 100644 --- a/engine/crates/layerfs-nfs/src/kernel/released.rs +++ b/engine/crates/layerfs-nfs/src/kernel/released.rs @@ -43,8 +43,8 @@ impl Kernel { fn holdings(&self, table: &Table, companions: crate::apple_double::Holding) -> Released { Released { companions: companions.held, + companion_bytes: companions.resident_bytes, companions_decoded: companions.decoded, - companion_decoded_bytes: companions.decoded_bytes, files: table.files.len(), directories: table.directories.len(), lookups: table.lookups.len(), @@ -77,10 +77,12 @@ pub struct Released { /// AppleDouble companions held in the export's index (see /// [`crate::apple_double`]). pub companions: usize, - /// Of the companions held, those kept decoded, and the bytes counted for - /// them. + /// Companion body bytes currently counted against the export's resident + /// budget: a body kept as written at its length, a decoded one at what its + /// attributes are counted as. + pub companion_bytes: usize, + /// Of the companions held, those kept decoded. pub companions_decoded: usize, - pub companion_decoded_bytes: usize, /// AppleDouble companions created, over the export's life. pub names_detached: u64, /// References collected beyond the bound, over the export's life. diff --git a/engine/crates/layerfs-nfs/tests/apple_double_decode.rs b/engine/crates/layerfs-nfs/tests/apple_double_decode.rs index 7acd47a903..dea993d169 100644 --- a/engine/crates/layerfs-nfs/tests/apple_double_decode.rs +++ b/engine/crates/layerfs-nfs/tests/apple_double_decode.rs @@ -81,12 +81,21 @@ fn the_clients_write_sequence_is_served_byte_for_byte() { c.write(&ad.handle, 50, &whole[50..]).unwrap(); assert_eq!(c.read_all(&ad.handle, 64 * 1024), whole); assert_eq!(server.held().companions_decoded, 1); + // The decoded form is memory too: it is counted against the same resident + // budget as a body kept as written, at what its attributes cost. + let decoded = Image::compact(&whole).expect("the client's image decodes"); + assert_eq!(server.held().companion_bytes, decoded.cost()); // A delete and rebuild starts empty and decodes the rewrite again. c.remove(&root, b"._seq").unwrap(); assert_eq!(c.getattr(&ad.handle), Err(STALE)); let again = c.touch(&root, b"._seq", 0o644).unwrap(); assert_eq!(again.attr.size, 0); + assert_eq!( + server.held().companion_bytes, + 0, + "the removed companion gave its bytes back" + ); assert_eq!(c.read_all(&again.handle, 4096), Vec::::new()); let rebuilt = image( [0x44; 32], @@ -97,6 +106,8 @@ fn the_clients_write_sequence_is_served_byte_for_byte() { ); c.write(&again.handle, 0, &rebuilt).unwrap(); assert_eq!(c.read_all(&again.handle, 128 * 1024), rebuilt); + let rebuilt_cost = Image::compact(&rebuilt).expect("decodes").cost(); + assert_eq!(server.held().companion_bytes, rebuilt_cost); // An edit that leaves the image not whole is served as written, and a // later edit that completes it decodes again: both are the client's own @@ -106,9 +117,15 @@ fn the_clients_write_sequence_is_served_byte_for_byte() { c.write(&again.handle, 24, b"EDIT").unwrap(); assert_eq!(c.read_all(&again.handle, 128 * 1024), edited); assert_eq!(server.held().companions_decoded, 0); + assert_eq!( + server.held().companion_bytes, + edited.len(), + "a body kept as written counts its length" + ); c.write(&again.handle, 24, &rebuilt[24..28]).unwrap(); assert_eq!(c.read_all(&again.handle, 128 * 1024), rebuilt); assert_eq!(server.held().companions_decoded, 1); + assert_eq!(server.held().companion_bytes, rebuilt_cost); // Truncating below the image is served as the client's truncation, and // resizing back up serves zeros, as on any file. From 428ffc34b9d2717a2e6e2d8970720f9a6beaaa47 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:45:48 +0000 Subject: [PATCH 7/7] chore(nfs): make the AppleDouble codec edits fmt- and clippy-clean (lp73) `cargo fmt --check` reported three files of this layer: the `emit` call rustfmt wraps in `Image::encode_range`, and two assertions of the new codec and decode tests. This is that pass and nothing else. The one remaining fmt report, `api/crates/layerfs-cli/tests/concurrent_reads.rs`, is not touched by this branch (`git log 8cfb6e273..HEAD -- ` is empty) and is left alone. Checks: `cargo +1.85.1 fmt --all -- --check` reports only that file; `cargo +1.85.1 clippy --locked -p layerfs-nfs --all-targets -- -D warnings` is clean; `cargo test -p layerfs-nfs` passes every binary. Production LOC: 163896 -> 163901 (delta +5) Scope: first-party core/engine/control/api production source and shipped runtime SQL; tests, docs, tools and manifests excluded. Method: python3 tools/production_loc.py (SHA-256 d3148db88f91b09eb74eb659b2f700209776a24d022f09774dc7bb38ea1eecf6) on git archives of the first parent 7ef50a034 (163896) and of the staged tree (163901). Engine +5: rustfmt splits the `emit` call, which the counter counts. Reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). --- engine/crates/layerfs-nfs/src/apple_double_codec.rs | 7 ++++++- engine/crates/layerfs-nfs/tests/apple_double_codec.rs | 5 ++++- engine/crates/layerfs-nfs/tests/apple_double_decode.rs | 1 - 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/engine/crates/layerfs-nfs/src/apple_double_codec.rs b/engine/crates/layerfs-nfs/src/apple_double_codec.rs index 1e45130f78..459b507c96 100644 --- a/engine/crates/layerfs-nfs/src/apple_double_codec.rs +++ b/engine/crates/layerfs-nfs/src/apple_double_codec.rs @@ -238,7 +238,12 @@ impl Image { let name_end = name_at + attr.name.len(); emit(&mut out, start, name_at, &attr.name); let next = entry + entry_len(name_len); - emit(&mut out, start, name_end + 1, &attr.pad[..next - name_end - 1]); + emit( + &mut out, + start, + name_end + 1, + &attr.pad[..next - name_end - 1], + ); emit(&mut out, start, value_at, &attr.value); entry = next; value_at += attr.value.len(); diff --git a/engine/crates/layerfs-nfs/tests/apple_double_codec.rs b/engine/crates/layerfs-nfs/tests/apple_double_codec.rs index b4d0c32098..548eeb87a9 100644 --- a/engine/crates/layerfs-nfs/tests/apple_double_codec.rs +++ b/engine/crates/layerfs-nfs/tests/apple_double_codec.rs @@ -96,7 +96,10 @@ fn any_window_of_the_image_encodes_to_that_window() { assert_eq!(built.encode_range(len + 10, len + 20), Vec::::new()); assert_eq!(built.encode_range(7, 3), Vec::::new()); assert_eq!(built.encode_range(0, len + 10), bytes); - assert_eq!(built.encode_range(len - 1, len + 10), bytes[len - 1..].to_vec()); + assert_eq!( + built.encode_range(len - 1, len + 10), + bytes[len - 1..].to_vec() + ); // The fork's own bytes are not all zero: a window inside it is served // from the fork, not from the zeroed layout. let fork_at = get32(&bytes, FORK_AT) as usize; diff --git a/engine/crates/layerfs-nfs/tests/apple_double_decode.rs b/engine/crates/layerfs-nfs/tests/apple_double_decode.rs index dea993d169..73fe47c666 100644 --- a/engine/crates/layerfs-nfs/tests/apple_double_decode.rs +++ b/engine/crates/layerfs-nfs/tests/apple_double_decode.rs @@ -156,4 +156,3 @@ fn the_clients_write_sequence_is_served_byte_for_byte() { assert_eq!(outstanding, 0); fs::remove_dir_all(base).unwrap(); } -