Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion docs/architecture/engine/kernel-routes/nfs.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ The daemon starts it and has the host's own NFS client mount it.
| Server |
| thread layerfs-nfs-accept |
| thread layerfs-nfs-connection (one per TCP connection) |
| read_record -> answer(call) -> write_record -> Tickets::replied
| Records::read -> answer(call) -> write_record -> Tickets::replied
| Kernel (synthetic kernel, shared by all connections) |
| lookups, parents, files, directories, silly names, |
| verifiers, names, refreshing; Companions (AppleDouble) |
Expand Down Expand Up @@ -65,6 +65,17 @@ read-only export answers each changing procedure `NFS3ERR_ROFS`
`Reader` bounds-checks every read and declared length; `Writer` pads opaque
data to four bytes. A reply is written as one last fragment.

A call is framed from the bytes one `read` delivered: the connection keeps them
in [`rpc::Records`](../../../../engine/crates/layerfs-nfs/src/rpc.rs), so a call
whose fragments arrived together costs one socket read. A record larger than
the 8 KiB read-ahead (a WRITE of a transfer window) is read into a buffer of its
own, so its bytes are not copied twice. Those counts are the case's own and are
reproducible on any platform
([`tests/protocol.rs`](../../../../engine/crates/layerfs-nfs/tests/protocol.rs)):
one read per call that arrived in one piece, two for a record past the
read-ahead. A blocking read a signal interrupts is retried, as `read_exact`
does, and the bytes already received stay buffered.

## Handles (`handle.rs`)

```text
Expand Down
165 changes: 143 additions & 22 deletions engine/crates/layerfs-nfs/src/rpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,33 +61,154 @@ pub enum Refused {
Reply(Vec<u8>),
}

/// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`].
/// `Ok(None)` is an orderly end of stream before a record started.
pub fn read_record(stream: &mut impl Read) -> io::Result<Option<Vec<u8>>> {
let mut record = Vec::new();
loop {
let mut header = [0; 4];
if let Err(error) = stream.read_exact(&mut header) {
if error.kind() == io::ErrorKind::UnexpectedEof && record.is_empty() {
return Ok(None);
/// Bytes one connection receives ahead of a record's length: one `read` of
/// this much carries any metadata call whole, so such a call costs one read.
const READ_AHEAD: usize = 8 * 1024;

/// One connection's received records.
///
/// A call the client sent in one piece is framed from the bytes one `read`
/// delivered, so it costs one read instead of the header read plus the body
/// read a plain `read_exact` pair makes. A record larger than the read-ahead
/// (a WRITE of a transfer window) is read into its own buffer with
/// `read_exact`, so its bytes are not copied twice.
pub struct Records {
/// Received bytes, of which `valid` are data not yet consumed.
buffer: Vec<u8>,
valid: usize,
}
impl Default for Records {
fn default() -> Self {
Self::new()
}
}
impl Records {
pub fn new() -> Self {
Self {
buffer: Vec::new(),
valid: 0,
}
}
/// Reads one whole record (all fragments), bounded by [`RECORD_LIMIT`].
/// `Ok(None)` is an orderly end of stream before a record started.
pub fn read(&mut self, stream: &mut impl Read) -> io::Result<Option<Vec<u8>>> {
loop {
if self.valid < 4 {
if !self.receive(stream)? {
return Ok(None);
}
continue;
}
let header = u32::from_be_bytes([
self.buffer[0],
self.buffer[1],
self.buffer[2],
self.buffer[3],
]);
let length = (header & !LAST_FRAGMENT) as usize;
if length > RECORD_LIMIT {
return Err(too_large());
}
if header & LAST_FRAGMENT == 0 {
// A fragment chain: its bodies are not contiguous.
return self.chain(stream, length);
}
if self.valid >= 4 + length {
let record = self.buffer[4..4 + length].to_vec();
self.consume(4 + length);
return Ok(Some(record));
}
if 4 + length > READ_AHEAD {
return self.large(stream, length);
}
if !self.receive(stream)? {
return Err(io::Error::from(io::ErrorKind::UnexpectedEof));
}
return Err(error);
}
let header = u32::from_be_bytes(header);
let length = (header & !LAST_FRAGMENT) as usize;
if record.len() + length > RECORD_LIMIT {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"RPC record exceeds its bound",
));
}
/// Receives more bytes into the read-ahead buffer, allocated once on the
/// first read; `false` is an orderly end of stream before any. A blocked
/// `read` a signal interrupts is retried, as `read_exact` and
/// [`write_record`] do, and the bytes already received stay buffered.
fn receive(&mut self, stream: &mut impl Read) -> io::Result<bool> {
if self.buffer.len() < READ_AHEAD {
self.buffer.resize(READ_AHEAD, 0);
}
let start = record.len();
record.resize(start + length, 0);
stream.read_exact(&mut record[start..])?;
if header & LAST_FRAGMENT != 0 {
return Ok(Some(record));
loop {
let read = match stream.read(&mut self.buffer[self.valid..]) {
Ok(read) => read,
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
Err(error) => return Err(error),
};
if read == 0 {
return if self.valid == 0 {
Ok(false)
} else {
Err(io::Error::from(io::ErrorKind::UnexpectedEof))
};
}
self.valid += read;
return Ok(true);
}
}
/// Drops the first `consumed` bytes; the read-ahead buffer stays.
fn consume(&mut self, consumed: usize) {
self.buffer.copy_within(consumed..self.valid, 0);
self.valid -= consumed;
}
/// One final fragment larger than the read-ahead, read into its own
/// buffer: the bulk of it is never copied.
fn large(&mut self, stream: &mut impl Read, length: usize) -> io::Result<Option<Vec<u8>>> {
let buffered = self.valid - 4;
let mut record = Vec::with_capacity(length);
record.extend_from_slice(&self.buffer[4..self.valid]);
self.valid = 0;
record.resize(length, 0);
stream.read_exact(&mut record[buffered..])?;
Ok(Some(record))
}
/// A fragment chain: each fragment's body follows its own header.
fn chain(&mut self, stream: &mut impl Read, mut length: usize) -> io::Result<Option<Vec<u8>>> {
self.consume(4); // the first fragment's header, already framed
let mut record = Vec::new();
let mut header = [0; 4];
loop {
if record.len() + length > RECORD_LIMIT {
return Err(too_large());
}
let start = record.len();
record.resize(start + length, 0);
self.take(stream, &mut record[start..])?;
self.take(stream, &mut header)?;
let word = u32::from_be_bytes(header);
length = (word & !LAST_FRAGMENT) as usize;
if length > RECORD_LIMIT {
return Err(too_large());
}
if word & LAST_FRAGMENT != 0 {
if record.len() + length > RECORD_LIMIT {
return Err(too_large());
}
let start = record.len();
record.resize(start + length, 0);
self.take(stream, &mut record[start..])?;
return Ok(Some(record));
}
}
}
/// Fills `into` from the buffered bytes first, then from the stream.
fn take(&mut self, stream: &mut impl Read, into: &mut [u8]) -> io::Result<()> {
let buffered = self.valid.min(into.len());
into[..buffered].copy_from_slice(&self.buffer[..buffered]);
self.consume(buffered);
if buffered < into.len() {
stream.read_exact(&mut into[buffered..])?;
}
Ok(())
}
}
fn too_large() -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, "RPC record exceeds its bound")
}
/// Writes one reply as a single last fragment. The four-byte fragment header
/// and the reply go out as two writes of one vectored write, so a reply is
Expand Down
3 changes: 2 additions & 1 deletion engine/crates/layerfs-nfs/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,8 @@ fn accept_loop(listener: &TcpListener, shared: &Arc<Shared>) {
}
}
fn connection(mut stream: TcpStream, shared: &Shared) {
while let Ok(Some(record)) = rpc::read_record(&mut stream) {
let mut records = rpc::Records::new();
while let Ok(Some(record)) = records.read(&mut stream) {
if shared.stopping.load(Ordering::Acquire) {
return;
}
Expand Down
Loading