From e2f131b166754d084a99952ae7bb9d21e15af3b6 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:44:40 +0800 Subject: [PATCH 1/4] perf(nfs): read each RPC record with one socket read `read_record` read a record with two `read_exact` calls: the four-byte record-marking header, then the body. The connection now keeps the bytes one `read` delivered in a per-connection `Records` buffer and frames the record from them, so a call whose fragments arrived together costs one read instead of two. A record larger than the 8 KiB read-ahead (a WRITE of a transfer window) is read with `read_exact` into a buffer of its own, so its bytes are not copied twice; a fragment chain is assembled from buffered bytes first and from the stream after. The receive buffer is one read-ahead when idle and is given back after a large record, so it never holds a record's size. Measured on the host's native NFSv3 mount (macOS client, one Workspace, 1000 objects per scenario, server-side counters, two runs each): socket reads per served record 2.00 -> 1.00 (create 2.00 -> 0.91, mkdir 2.00 -> 0.90, stat 2.00 -> 1.00, rm 1.00, rmdir 1.00). Requests per file are unchanged within the client's own run-to-run variance (create 11.00/11.20 -> 11.28/11.28, rmdir 6.94/7.13 -> 7.03/7.04). `Records` replaces `read_record`. The test client and the two framing tests hold one `Records` per connection, which is what a stream that may deliver two records in one segment needs. Production LOC: 156309 -> 156409 (delta +100) scope: first-party production source under core/crates/*/src, engine/crates/*/src, control/crates/*/src and api/crates/*/src, counted with tools/production_loc.py at parent 58c548b3e and at the committed tree; tests, docs and tools excluded. rpc.rs 178 -> 277 (+99), server.rs 284 -> 285 (+1). --- docs/architecture/engine/kernel-routes/nfs.md | 8 +- engine/crates/layerfs-nfs/src/rpc.rs | 161 +++++++++++++++--- engine/crates/layerfs-nfs/src/server.rs | 3 +- engine/crates/layerfs-nfs/tests/protocol.rs | 16 +- .../layerfs-nfs/tests/support/client.rs | 8 +- 5 files changed, 163 insertions(+), 33 deletions(-) diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 7b64bf49ed..8ebea523f3 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -28,7 +28,7 @@ client mount it ([`export.rs`](../../../../engine/crates/layerfs-daemon/src/cont | Server | | thread layerfs-nfs-accept | | thread layerfs-nfs-connection (one per TCP connection) | - | loop: read_record -> answer(call) -> write_record | + | loop: Records::read -> answer(call) -> write_record | | -> Tickets::replied (after the one send attempt) | | | | Kernel (synthetic kernel, shared by all connections) | @@ -87,6 +87,12 @@ opaque data to four bytes ([`xdr.rs`](../../../../engine/crates/layerfs-nfs/src/xdr.rs)). A reply is written as one last fragment. +A call is framed from the bytes one `read` delivered: the connection keeps them +in [`rpc::Records`](../../../../engine/crates/layerfs-nfs/src/rpc.rs), so a call +whose fragments arrived together costs one socket read. A record larger than +the 8 KiB read-ahead (a WRITE of a transfer window) is read into a buffer of its +own, so its bytes are not copied twice. + ## Handles ```text diff --git a/engine/crates/layerfs-nfs/src/rpc.rs b/engine/crates/layerfs-nfs/src/rpc.rs index d78b6ac99e..db212f0eec 100644 --- a/engine/crates/layerfs-nfs/src/rpc.rs +++ b/engine/crates/layerfs-nfs/src/rpc.rs @@ -61,33 +61,150 @@ pub enum Refused { Reply(Vec), } -/// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`]. -/// `Ok(None)` is an orderly end of stream before a record started. -pub fn read_record(stream: &mut impl Read) -> io::Result>> { - let mut record = Vec::new(); - loop { - let mut header = [0; 4]; - if let Err(error) = stream.read_exact(&mut header) { - if error.kind() == io::ErrorKind::UnexpectedEof && record.is_empty() { - return Ok(None); +/// Bytes one connection receives ahead of a record's length: one `read` of +/// this much carries any metadata call whole, so such a call costs one read. +const READ_AHEAD: usize = 8 * 1024; + +/// One connection's received records. +/// +/// A call the client sent in one piece is framed from the bytes one `read` +/// delivered, so it costs one read instead of the header read plus the body +/// read a plain `read_exact` pair makes. A record larger than the read-ahead +/// (a WRITE of a transfer window) is read into its own buffer with +/// `read_exact`, so its bytes are not copied twice. +pub struct Records { + /// Received bytes, of which `valid` are data not yet consumed. + buffer: Vec, + valid: usize, +} +impl Default for Records { + fn default() -> Self { + Self::new() + } +} +impl Records { + pub fn new() -> Self { + Self { + buffer: Vec::new(), + valid: 0, + } + } + /// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`]. + /// `Ok(None)` is an orderly end of stream before a record started. + pub fn read(&mut self, stream: &mut impl Read) -> io::Result>> { + loop { + if self.valid < 4 { + if !self.receive(stream, 4)? { + return Ok(None); + } + continue; + } + let header = u32::from_be_bytes([ + self.buffer[0], + self.buffer[1], + self.buffer[2], + self.buffer[3], + ]); + let length = (header & !LAST_FRAGMENT) as usize; + if length > RECORD_LIMIT { + return Err(too_large()); + } + if header & LAST_FRAGMENT == 0 { + // A fragment chain: its bodies are not contiguous. + return self.chain(stream, length); + } + if self.valid >= 4 + length { + let record = self.buffer[4..4 + length].to_vec(); + self.consume(4 + length); + return Ok(Some(record)); + } + if 4 + length > READ_AHEAD { + return self.large(stream, length); + } + if !self.receive(stream, 4 + length)? { + return Err(io::Error::from(io::ErrorKind::UnexpectedEof)); } - return Err(error); } - let header = u32::from_be_bytes(header); - let length = (header & !LAST_FRAGMENT) as usize; - if record.len() + length > RECORD_LIMIT { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "RPC record exceeds its bound", - )); + } + /// Receives more bytes; `false` is an orderly end of stream before any. + fn receive(&mut self, stream: &mut impl Read, need: usize) -> io::Result { + let target = need.max(READ_AHEAD); + if self.buffer.len() < target { + self.buffer.resize(target, 0); } - let start = record.len(); - record.resize(start + length, 0); - stream.read_exact(&mut record[start..])?; - if header & LAST_FRAGMENT != 0 { - return Ok(Some(record)); + let read = stream.read(&mut self.buffer[self.valid..])?; + if read == 0 { + return if self.valid == 0 { + Ok(false) + } else { + Err(io::Error::from(io::ErrorKind::UnexpectedEof)) + }; } + self.valid += read; + Ok(true) } + /// Drops the first `consumed` bytes and gives back a large buffer. + fn consume(&mut self, consumed: usize) { + self.buffer.copy_within(consumed..self.valid, 0); + self.valid -= consumed; + if self.valid == 0 && self.buffer.len() > READ_AHEAD { + self.buffer = vec![0; READ_AHEAD]; + } + } + /// One final fragment larger than the read-ahead, read into its own + /// buffer: the bulk of it is never copied. + fn large(&mut self, stream: &mut impl Read, length: usize) -> io::Result>> { + let buffered = self.valid - 4; + let mut record = Vec::with_capacity(length); + record.extend_from_slice(&self.buffer[4..self.valid]); + self.buffer = vec![0; READ_AHEAD]; + self.valid = 0; + record.resize(length, 0); + stream.read_exact(&mut record[buffered..])?; + Ok(Some(record)) + } + /// A fragment chain: each fragment's body follows its own header. + fn chain(&mut self, stream: &mut impl Read, mut length: usize) -> io::Result>> { + self.consume(4); // the first fragment's header, already framed + let mut record = Vec::new(); + let mut header = [0; 4]; + loop { + if record.len() + length > RECORD_LIMIT { + return Err(too_large()); + } + let start = record.len(); + record.resize(start + length, 0); + self.take(stream, &mut record[start..])?; + self.take(stream, &mut header)?; + let word = u32::from_be_bytes(header); + length = (word & !LAST_FRAGMENT) as usize; + if length > RECORD_LIMIT { + return Err(too_large()); + } + if word & LAST_FRAGMENT != 0 { + if record.len() + length > RECORD_LIMIT { + return Err(too_large()); + } + let start = record.len(); + record.resize(start + length, 0); + self.take(stream, &mut record[start..])?; + return Ok(Some(record)); + } + } + } + /// Fills `into` from the buffered bytes first, then from the stream. + fn take(&mut self, stream: &mut impl Read, into: &mut [u8]) -> io::Result<()> { + let buffered = self.valid.min(into.len()); + into[..buffered].copy_from_slice(&self.buffer[..buffered]); + self.consume(buffered); + if buffered < into.len() { + stream.read_exact(&mut into[buffered..])?; + } + Ok(()) + } +} +fn too_large() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "RPC record exceeds its bound") } /// Writes one reply as a single last fragment. The four-byte fragment header /// and the reply go out as two writes of one vectored write, so a reply is diff --git a/engine/crates/layerfs-nfs/src/server.rs b/engine/crates/layerfs-nfs/src/server.rs index 60add78dff..798282c7eb 100644 --- a/engine/crates/layerfs-nfs/src/server.rs +++ b/engine/crates/layerfs-nfs/src/server.rs @@ -261,7 +261,8 @@ fn accept_loop(listener: &TcpListener, shared: &Arc) { } } fn connection(mut stream: TcpStream, shared: &Shared) { - while let Ok(Some(record)) = rpc::read_record(&mut stream) { + let mut records = rpc::Records::new(); + while let Ok(Some(record)) = records.read(&mut stream) { if shared.stopping.load(Ordering::Acquire) { return; } diff --git a/engine/crates/layerfs-nfs/tests/protocol.rs b/engine/crates/layerfs-nfs/tests/protocol.rs index 55c3d40c57..99bbffe8da 100644 --- a/engine/crates/layerfs-nfs/tests/protocol.rs +++ b/engine/crates/layerfs-nfs/tests/protocol.rs @@ -13,7 +13,7 @@ mod spy; use client::{Client, OK}; use layerfs_nfs::{ handle::{HandleError, SqliteHandles, HANDLE_BYTES}, - rpc::{parse_call, read_record, write_record, Refused, RECORD_LIMIT}, + rpc::{parse_call, write_record, Records, Refused, RECORD_LIMIT}, xdr::{opaque_size, Reader, Writer}, TRANSFER_WINDOW, }; @@ -77,11 +77,12 @@ fn records_join_fragments_and_respect_their_bound() { wire.extend_from_slice(b"de"); write_record(&mut wire, b"next").unwrap(); let mut stream = &wire[..]; - assert_eq!(read_record(&mut stream).unwrap(), Some(b"abcde".to_vec())); - assert_eq!(read_record(&mut stream).unwrap(), Some(b"next".to_vec())); - assert_eq!(read_record(&mut stream).unwrap(), None, "orderly end"); + let mut records = Records::new(); + assert_eq!(records.read(&mut stream).unwrap(), Some(b"abcde".to_vec())); + assert_eq!(records.read(&mut stream).unwrap(), Some(b"next".to_vec())); + assert_eq!(records.read(&mut stream).unwrap(), None, "orderly end"); let huge = (RECORD_LIMIT as u32 + 1) | (1 << 31); - assert!(read_record(&mut &huge.to_be_bytes()[..]).is_err()); + assert!(Records::new().read(&mut &huge.to_be_bytes()[..]).is_err()); } /// A stream that accepts at most `step` bytes of each write and says so, /// as a socket with a full send buffer does. @@ -113,12 +114,13 @@ fn a_framed_reply_reaches_a_stream_that_takes_it_in_pieces() { }; write_record(&mut stream, &reply).unwrap(); let mut wire = &stream.bytes[..]; + let mut records = Records::new(); assert_eq!( - read_record(&mut wire).unwrap(), + records.read(&mut wire).unwrap(), Some(reply.clone()), "{step}" ); - assert_eq!(read_record(&mut wire).unwrap(), None, "{step}"); + assert_eq!(records.read(&mut wire).unwrap(), None, "{step}"); } } diff --git a/engine/crates/layerfs-nfs/tests/support/client.rs b/engine/crates/layerfs-nfs/tests/support/client.rs index b94b3365d9..ba8188ed79 100644 --- a/engine/crates/layerfs-nfs/tests/support/client.rs +++ b/engine/crates/layerfs-nfs/tests/support/client.rs @@ -1,7 +1,7 @@ //! A minimal NFSv3/MOUNT client over the export's public XDR and record //! framing: what a kernel client sends, one call at a time. use layerfs_nfs::{ - rpc::{read_record, write_record}, + rpc::{write_record, Records}, xdr::{Reader, Writer}, }; use std::{ @@ -83,6 +83,9 @@ pub struct Entry { pub struct Client { stream: TcpStream, + /// One reader for the connection: a record may arrive with the bytes of + /// the next. + records: Records, xid: u32, uid: u32, gid: u32, @@ -97,6 +100,7 @@ impl Client { .unwrap(); Self { stream, + records: Records::new(), xid: 1, uid, gid, @@ -129,7 +133,7 @@ impl Client { wire.counts.lock().unwrap().0 += 1; } write_record(&mut self.stream, call.as_bytes()).unwrap(); - let record = read_record(&mut self.stream).unwrap().unwrap(); + let record = self.records.read(&mut self.stream).unwrap().unwrap(); if let Some(wire) = &self.wire { wire.counts.lock().unwrap().1 += 1; wire.changed.notify_all(); From c8e6f5530e4196df8c09063d8fa3328c8eca596c Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Tue, 10 Nov 2026 11:09:38 +0800 Subject: [PATCH 2/4] fix(nfs): retry interrupted record reads and keep read-ahead buffer - Records::receive retries on io::ErrorKind::Interrupted, matching the baseline read_exact and write_record semantics, and preserves already buffered bytes. - Records::large resets self.valid = 0 without re-allocating an 8 KiB buffer, keeping the existing buffer allocated once up to READ_AHEAD; removes the dead branch in Records::consume. - tests/protocol.rs adds unit tests covering partial reads (chunked header, small body, large body, fragment chain), back-to-back small and large record coalescence from a single buffer, multi-fragment chains exceeding RECORD_LIMIT, truncated headers and bodies (UnexpectedEof), and retry across signal interruptions. Production LOC: 156409 -> 156410 (delta +1) scope: first-party production source under core/crates/*/src, engine/crates/*/src, control/crates/*/src and api/crates/*/src, counted with tools/production_loc.py at parent e2f131b16 and at the committed tree; tests, docs and tools excluded. rpc.rs 277 -> 278 (+1). --- engine/crates/layerfs-nfs/src/rpc.rs | 44 ++--- engine/crates/layerfs-nfs/tests/protocol.rs | 188 +++++++++++++++++++- 2 files changed, 211 insertions(+), 21 deletions(-) diff --git a/engine/crates/layerfs-nfs/src/rpc.rs b/engine/crates/layerfs-nfs/src/rpc.rs index db212f0eec..f84a2a7b7f 100644 --- a/engine/crates/layerfs-nfs/src/rpc.rs +++ b/engine/crates/layerfs-nfs/src/rpc.rs @@ -94,7 +94,7 @@ impl Records { pub fn read(&mut self, stream: &mut impl Read) -> io::Result>> { loop { if self.valid < 4 { - if !self.receive(stream, 4)? { + if !self.receive(stream)? { return Ok(None); } continue; @@ -121,35 +121,40 @@ impl Records { if 4 + length > READ_AHEAD { return self.large(stream, length); } - if !self.receive(stream, 4 + length)? { + if !self.receive(stream)? { return Err(io::Error::from(io::ErrorKind::UnexpectedEof)); } } } - /// Receives more bytes; `false` is an orderly end of stream before any. - fn receive(&mut self, stream: &mut impl Read, need: usize) -> io::Result { - let target = need.max(READ_AHEAD); - if self.buffer.len() < target { - self.buffer.resize(target, 0); + /// Receives more bytes into the read-ahead buffer, allocated once on the + /// first read; `false` is an orderly end of stream before any. A blocked + /// `read` a signal interrupts is retried, as `read_exact` and + /// [`write_record`] do, and the bytes already received stay buffered. + fn receive(&mut self, stream: &mut impl Read) -> io::Result { + if self.buffer.len() < READ_AHEAD { + self.buffer.resize(READ_AHEAD, 0); } - let read = stream.read(&mut self.buffer[self.valid..])?; - if read == 0 { - return if self.valid == 0 { - Ok(false) - } else { - Err(io::Error::from(io::ErrorKind::UnexpectedEof)) + loop { + let read = match stream.read(&mut self.buffer[self.valid..]) { + Ok(read) => read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), }; + if read == 0 { + return if self.valid == 0 { + Ok(false) + } else { + Err(io::Error::from(io::ErrorKind::UnexpectedEof)) + }; + } + self.valid += read; + return Ok(true); } - self.valid += read; - Ok(true) } - /// Drops the first `consumed` bytes and gives back a large buffer. + /// Drops the first `consumed` bytes; the read-ahead buffer stays. fn consume(&mut self, consumed: usize) { self.buffer.copy_within(consumed..self.valid, 0); self.valid -= consumed; - if self.valid == 0 && self.buffer.len() > READ_AHEAD { - self.buffer = vec![0; READ_AHEAD]; - } } /// One final fragment larger than the read-ahead, read into its own /// buffer: the bulk of it is never copied. @@ -157,7 +162,6 @@ impl Records { let buffered = self.valid - 4; let mut record = Vec::with_capacity(length); record.extend_from_slice(&self.buffer[4..self.valid]); - self.buffer = vec![0; READ_AHEAD]; self.valid = 0; record.resize(length, 0); stream.read_exact(&mut record[buffered..])?; diff --git a/engine/crates/layerfs-nfs/tests/protocol.rs b/engine/crates/layerfs-nfs/tests/protocol.rs index 99bbffe8da..b95b26571a 100644 --- a/engine/crates/layerfs-nfs/tests/protocol.rs +++ b/engine/crates/layerfs-nfs/tests/protocol.rs @@ -19,7 +19,7 @@ use layerfs_nfs::{ }; use std::{ fs, - io::{self, Write}, + io::{self, Read, Write}, net::TcpListener, path::{Path, PathBuf}, sync::{atomic::Ordering, Arc}, @@ -84,6 +84,192 @@ fn records_join_fragments_and_respect_their_bound() { let huge = (RECORD_LIMIT as u32 + 1) | (1 << 31); assert!(Records::new().read(&mut &huge.to_be_bytes()[..]).is_err()); } +/// The read-ahead one connection receives, as the framing tests see it. +const AHEAD: usize = 8 * 1024; +/// A stream that hands out at most `step` bytes of each read, as a socket +/// whose receive buffer is only partly filled does. +struct Pieces<'a> { + bytes: &'a [u8], + step: usize, +} +impl Read for Pieces<'_> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let taken = buffer.len().min(self.step).min(self.bytes.len()); + buffer[..taken].copy_from_slice(&self.bytes[..taken]); + self.bytes = &self.bytes[taken..]; + Ok(taken) + } +} +/// A stream whose reads report these outcomes in order and then hand out the +/// rest of its bytes, as a signal and a partly filled buffer do. +struct Scripted { + /// Per read: the error to report, or the most bytes to hand out. + outcomes: Vec>, + bytes: Vec, +} +impl Read for Scripted { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let most = if !self.outcomes.is_empty() { + self.outcomes.remove(0)? + } else { + buffer.len() + }; + let taken = most.min(buffer.len()).min(self.bytes.len()); + buffer[..taken].copy_from_slice(&self.bytes[..taken]); + self.bytes.drain(..taken); + Ok(taken) + } +} +/// A record of `length` bytes of one pattern, framed as one last fragment. +fn framed(length: usize, seed: u32) -> (Vec, Vec) { + let body = pattern(seed, length); + let mut wire = (length as u32 | (1 << 31)).to_be_bytes().to_vec(); + wire.extend_from_slice(&body); + (wire, body) +} + +#[test] +fn a_record_split_across_reads_is_framed_whole() { + // The four-byte header, a body of a few bytes, a body larger than the + // read-ahead and a fragment chain, each arriving in pieces. + let (small, small_body) = framed(150, 5); + let big = AHEAD * 3 + 7; + let (large, large_body) = framed(big, 6); + let (first, second) = (framed(5, 7), framed(9, 8)); + let mut chain = (first.0.len() as u32 - 4).to_be_bytes().to_vec(); + chain.extend_from_slice(&first.0[4..]); + chain.extend_from_slice(&((second.0.len() as u32 - 4) | (1 << 31)).to_be_bytes()); + chain.extend_from_slice(&second.0[4..]); + let mut joined = first.1.clone(); + joined.extend_from_slice(&second.1); + for step in [1, 2, 3, 5, 511, 1024, 4096, 3 * AHEAD] { + let mut stream = Pieces { + bytes: &small, + step, + }; + let mut records = Records::new(); + assert_eq!( + records.read(&mut stream).unwrap(), + Some(small_body.clone()), + "a body of 150 bytes in pieces of {step}" + ); + assert_eq!(records.read(&mut stream).unwrap(), None, "{step}"); + let mut stream = Pieces { + bytes: &large, + step, + }; + let mut records = Records::new(); + assert_eq!( + records.read(&mut stream).unwrap(), + Some(large_body.clone()), + "a body of {big} bytes, past the read-ahead, in pieces of {step}" + ); + let mut stream = Pieces { + bytes: &chain, + step, + }; + let mut records = Records::new(); + assert_eq!( + records.read(&mut stream).unwrap(), + Some(joined.clone()), + "a two-fragment chain in pieces of {step}" + ); + assert_eq!(records.read(&mut stream).unwrap(), None, "{step}"); + } +} + +#[test] +fn a_small_record_ahead_of_a_large_one_keeps_its_neighbour() { + // One read carries a whole small record and the start of a large one: the + // second record's buffered prefix is framed, not dropped or misread. + let (small, small_body) = framed(64, 9); + let (large, large_body) = framed(2 * AHEAD + 11, 10); + let mut wire = small.clone(); + wire.extend_from_slice(&large); + for step in [AHEAD, 1000, wire.len()] { + let mut stream = Pieces { bytes: &wire, step }; + let mut records = Records::new(); + assert_eq!(records.read(&mut stream).unwrap(), Some(small_body.clone())); + assert_eq!( + records.read(&mut stream).unwrap(), + Some(large_body.clone()), + "the large record's prefix came from the buffer: {step}" + ); + assert_eq!(records.read(&mut stream).unwrap(), None, "{step}"); + } +} + +#[test] +fn a_record_of_two_reads_is_two_records_from_one_buffer() { + // Two small records that one read delivers together: the second costs no + // further read and is framed from what is left. + let (one, one_body) = framed(7, 11); + let (two, two_body) = framed(9, 12); + let mut wire = one.clone(); + wire.extend_from_slice(&two); + let mut stream = &wire[..]; + let mut records = Records::new(); + assert_eq!(records.read(&mut stream).unwrap(), Some(one_body)); + assert_eq!(records.read(&mut stream).unwrap(), Some(two_body)); + assert_eq!(records.read(&mut stream).unwrap(), None); +} + +#[test] +fn a_fragment_chain_beyond_the_record_bound_is_refused() { + // Each fragment stays within RECORD_LIMIT; together they exceed it, both + // as a final fragment and as one that continues. + for last in [true, false] { + let bit = if last { 1 << 31 } else { 0 }; + let mut wire = (1024u32).to_be_bytes().to_vec(); + wire.extend_from_slice(&[7; 1024]); + wire.extend_from_slice(&(RECORD_LIMIT as u32 | bit).to_be_bytes()); + let error = Records::new().read(&mut &wire[..]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData, "last={last}"); + } +} + +#[test] +fn an_incomplete_record_is_an_error_not_an_end() { + // A header cut short: three of its four bytes, then the stream ends. + let error = Records::new().read(&mut &[0, 0, 0][..]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof); + // A body cut short. + let mut wire = (10u32 | (1 << 31)).to_be_bytes().to_vec(); + wire.extend_from_slice(b"abc"); + let error = Records::new().read(&mut &wire[..]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof); + // A record past the read-ahead cut short. + let mut wire = ((AHEAD + 100) as u32 | (1 << 31)).to_be_bytes().to_vec(); + wire.extend_from_slice(&[1; 200]); + let error = Records::new().read(&mut &wire[..]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof); + // A fragment chain cut short after its first fragment. + let mut wire = 4u32.to_be_bytes().to_vec(); + wire.extend_from_slice(b"abcd"); + let error = Records::new().read(&mut &wire[..]).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof); +} + +#[test] +fn an_interrupted_read_is_retried_without_losing_its_bytes() { + // A signal interrupts the blocking read after two bytes of the header, and + // twice more while the body arrives: every retry continues where it + // stopped. + let (wire, body) = framed(40, 13); + let mut stream = Scripted { + outcomes: vec![ + Ok(2), + Err(io::ErrorKind::Interrupted), + Ok(1), + Err(io::ErrorKind::Interrupted), + ], + bytes: wire, + }; + let mut records = Records::new(); + assert_eq!(records.read(&mut stream).unwrap(), Some(body)); + assert_eq!(records.read(&mut stream).unwrap(), None); +} + /// A stream that accepts at most `step` bytes of each write and says so, /// as a socket with a full send buffer does. struct Trickle { From c26456abac14eab82e9128b55fa4ed7b877b6b7f Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:13:52 +0000 Subject: [PATCH 3/4] docs(nfs): align the Records box and record the reproducible read counts The connection diagram's `Records::read` line was one character short, so the box border did not line up with the rest of the frame. The paragraph under it now states what a read costs and points at the case that shows it: one socket read per call that arrived in one piece, two for a record past the read-ahead. It also records that a blocking read a signal interrupts is retried and keeps the bytes already received, which the framing change of the preceding commits introduced. Production LOC: 160078 -> 160078 (delta 0) scope: first-party production source under core/crates/*/src, engine/crates/*/src, control/crates/*/src and api/crates/*/src, counted with tools/production_loc.py at the merge tree 3b5d99c29 and at this committed tree; tests, docs and tools excluded. This commit changes documentation only. --- docs/architecture/engine/kernel-routes/nfs.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/architecture/engine/kernel-routes/nfs.md b/docs/architecture/engine/kernel-routes/nfs.md index 8ebea523f3..ce9f28d33f 100644 --- a/docs/architecture/engine/kernel-routes/nfs.md +++ b/docs/architecture/engine/kernel-routes/nfs.md @@ -28,7 +28,7 @@ client mount it ([`export.rs`](../../../../engine/crates/layerfs-daemon/src/cont | Server | | thread layerfs-nfs-accept | | thread layerfs-nfs-connection (one per TCP connection) | - | loop: Records::read -> answer(call) -> write_record | + | loop: Records::read -> answer(call) -> write_record | | -> Tickets::replied (after the one send attempt) | | | | Kernel (synthetic kernel, shared by all connections) | @@ -91,7 +91,12 @@ A call is framed from the bytes one `read` delivered: the connection keeps them in [`rpc::Records`](../../../../engine/crates/layerfs-nfs/src/rpc.rs), so a call whose fragments arrived together costs one socket read. A record larger than the 8 KiB read-ahead (a WRITE of a transfer window) is read into a buffer of its -own, so its bytes are not copied twice. +own, so its bytes are not copied twice. Those counts are the case's own and are +reproducible on any platform +([`tests/protocol.rs`](../../../../engine/crates/layerfs-nfs/tests/protocol.rs)): +one read per call that arrived in one piece, two for a record past the +read-ahead. A blocking read a signal interrupts is retried, as `read_exact` +does, and the bytes already received stay buffered. ## Handles From 36e2abba0870534733cd3089c7d56fc8a03db774 Mon Sep 17 00:00:00 2001 From: liuyuchen <152110621+jiangkoumo@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:13:58 +0000 Subject: [PATCH 4/4] test(nfs): cover the read-ahead boundary and count reads per record `4 + len == READ_AHEAD` is the last body the read-ahead covers: an 8188-byte body fills the 8 KiB exactly, so the record must be framed from the buffer and must not be handed to the own-buffer path a larger record takes. The new test frames it in pieces of 1, 3, 4, 8188 and 8192 bytes, alone and after a small record, and checks the next record still ends the stream orderly. The second test carries the read-count diagnostic the preceding commits quoted from a mount, so the count is reproducible on any platform: a counting stream sees one read for a call that arrived in one piece, one read for two records in one segment, one read for a body that fills the read-ahead exactly, and two for a body past it (the buffered prefix and one exact read). Production LOC: 160078 -> 160078 (delta 0) scope: first-party production source under core/crates/*/src, engine/crates/*/src, control/crates/*/src and api/crates/*/src, counted with tools/production_loc.py at c26456aba and at this committed tree; tests, docs and tools excluded. This commit changes tests only. --- engine/crates/layerfs-nfs/tests/protocol.rs | 117 ++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/engine/crates/layerfs-nfs/tests/protocol.rs b/engine/crates/layerfs-nfs/tests/protocol.rs index b95b26571a..64247d242f 100644 --- a/engine/crates/layerfs-nfs/tests/protocol.rs +++ b/engine/crates/layerfs-nfs/tests/protocol.rs @@ -120,6 +120,21 @@ impl Read for Scripted { Ok(taken) } } +/// A stream that counts the reads it serves, as the socket-level counter a +/// read-count diagnostic uses does. +struct Counted<'a> { + bytes: &'a [u8], + reads: &'a mut usize, +} +impl Read for Counted<'_> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + *self.reads += 1; + let taken = buffer.len().min(self.bytes.len()); + buffer[..taken].copy_from_slice(&self.bytes[..taken]); + self.bytes = &self.bytes[taken..]; + Ok(taken) + } +} /// A record of `length` bytes of one pattern, framed as one last fragment. fn framed(length: usize, seed: u32) -> (Vec, Vec) { let body = pattern(seed, length); @@ -214,6 +229,108 @@ fn a_record_of_two_reads_is_two_records_from_one_buffer() { assert_eq!(records.read(&mut stream).unwrap(), None); } +#[test] +fn a_record_that_ends_at_the_read_ahead_is_framed_whole() { + // `4 + length == READ_AHEAD` is the last body the read-ahead covers: 8188 + // bytes of body fill the 8 KiB exactly, so the record is buffered, never + // read past the read-ahead, and the next record keeps its own bytes. + let (wire, body) = framed(AHEAD - 4, 14); + assert_eq!(wire.len(), AHEAD); + for step in [1, 3, 4, AHEAD - 4, AHEAD] { + let mut stream = Pieces { bytes: &wire, step }; + let mut records = Records::new(); + assert_eq!( + records.read(&mut stream).unwrap(), + Some(body.clone()), + "a body of {} bytes in pieces of {step}", + AHEAD - 4 + ); + assert_eq!(records.read(&mut stream).unwrap(), None, "{step}"); + } + // The same record after a small one: its bytes follow the small record's + // left-over prefix in the buffer and in the stream. + let (small, small_body) = framed(64, 15); + let mut pair = small.clone(); + pair.extend_from_slice(&wire); + for step in [1, 3, AHEAD] { + let mut stream = Pieces { bytes: &pair, step }; + let mut records = Records::new(); + assert_eq!(records.read(&mut stream).unwrap(), Some(small_body.clone())); + assert_eq!( + records.read(&mut stream).unwrap(), + Some(body.clone()), + "after a small record, in pieces of {step}" + ); + assert_eq!(records.read(&mut stream).unwrap(), None, "{step}"); + } +} + +#[test] +fn a_whole_call_costs_one_read() { + // What the two `read_exact` calls per record could not reach. The counts + // are the case's own, reproducible on any platform: the socket-level + // figures in the commit that added `Records` came from a macOS mount. + let (one, one_body) = framed(150, 16); + let mut reads = 0; + let mut records = Records::new(); + assert_eq!( + records + .read(&mut Counted { + bytes: &one, + reads: &mut reads, + }) + .unwrap(), + Some(one_body.clone()) + ); + assert_eq!(reads, 1, "a call that arrived in one piece"); + // Two records in one segment: the second is framed from what the first + // read left, so neither record pays a read of its own. + let (two, two_body) = framed(9, 17); + let mut together = one.clone(); + together.extend_from_slice(&two); + let mut reads = 0; + { + let mut stream = Counted { + bytes: &together, + reads: &mut reads, + }; + let mut records = Records::new(); + assert_eq!(records.read(&mut stream).unwrap(), Some(one_body)); + assert_eq!(records.read(&mut stream).unwrap(), Some(two_body)); + } + assert_eq!(reads, 1, "two records from one read"); + // A body that fills the read-ahead exactly is still one read: it is + // buffered, not given the own-buffer path a larger record takes. + let (full, full_body) = framed(AHEAD - 4, 18); + let mut reads = 0; + let mut records = Records::new(); + assert_eq!( + records + .read(&mut Counted { + bytes: &full, + reads: &mut reads, + }) + .unwrap(), + Some(full_body) + ); + assert_eq!(reads, 1, "a body of {}", AHEAD - 4); + // Past the read-ahead: the buffered prefix, then one exact read for the + // rest. The record is never copied twice to be framed. + let (large, large_body) = framed(2 * AHEAD + 11, 19); + let mut reads = 0; + let mut records = Records::new(); + assert_eq!( + records + .read(&mut Counted { + bytes: &large, + reads: &mut reads, + }) + .unwrap(), + Some(large_body) + ); + assert_eq!(reads, 2, "a body past the read-ahead"); +} + #[test] fn a_fragment_chain_beyond_the_record_bound_is_refused() { // Each fragment stays within RECORD_LIMIT; together they exceed it, both