diff --git a/core/crates/layerfs-storage/src/encoding/delta/edge.rs b/core/crates/layerfs-storage/src/encoding/delta/edge.rs new file mode 100644 index 0000000000..663d84f9ea --- /dev/null +++ b/core/crates/layerfs-storage/src/encoding/delta/edge.rs @@ -0,0 +1,161 @@ +//! A payload record's dependency edge from bounded reads of its pack. +//! +//! [`super::read::ChainBases::edge`] borrows the record's complete encoded +//! group, so a walk that wants only edges pays for every payload byte it +//! passes. An uncompressed group keeps everything an edge needs at offsets its +//! own framing names: the record count and directory at the group's head, then +//! the record's first bytes. [`sparse_directory`] and [`sparse_edge`] read +//! those from whatever the caller already holds and name the next range it +//! lacks, so the caller decides how to batch the reads. No payload byte is +//! read, and so none is authenticated here: a consumer that reconstructs the +//! record still does that. +//! +//! A group's directory is validated and parsed once into a +//! [`RecordDirectory`]; each record's range is then one step. Asking the +//! framing reader for every record of a group instead scans the directory once +//! per record. +use super::read::ChainEdge; +use super::record; +use crate::error::{StorageError, StorageResult}; +use crate::location::ObjectLocation; +use crate::pack::layout::{record_range, GroupCodec, GroupView, PackLane}; +use crate::policy::RECORD_COUNT_LIMIT; +use crate::port::PackRange; + +/// Bytes at a group's head a walk asks for first: the record count, the +/// directory of a group of up to 100 records, and usually its first header. +pub const HEAD_BYTES: usize = 512; + +/// One uncompressed group's record directory, validated whole exactly as +/// [`crate::encoding::framed_record`] validates it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RecordDirectory { + /// Record `n` spans `bounds[n]..bounds[n + 1]` of the group. + bounds: Vec, +} + +impl RecordDirectory { + /// The directory of a complete uncompressed group body. + pub fn of(group: &[u8]) -> StorageResult { + let count = count_of(group.get(..4), group.len())?; + let ends = group + .get(4..4 + 4 * count) + .ok_or(StorageError::Integrity("group record directory"))?; + Self::parse(count, ends, group.len()) + } + + fn parse(count: usize, ends: &[u8], group_length: usize) -> StorageResult { + // The one validator of a directory: counts, ordering and the exact + // record area. What follows only copies offsets it accepted. + record_range(count, ends, group_length, 0)?; + let area = 4 + ends.len(); + let mut bounds = Vec::with_capacity(count + 1); + bounds.push(area); + for end in ends.chunks_exact(4) { + let mut field = [0; 4]; + field.copy_from_slice(end); + bounds.push(area + u32::from_le_bytes(field) as usize); + } + Ok(Self { bounds }) + } + + /// The records the group holds. + pub fn count(&self) -> usize { + self.bounds.len() - 1 + } + + /// The group-relative range of record `number`. + pub fn range(&self, number: usize) -> StorageResult<(usize, usize)> { + match (self.bounds.get(number), self.bounds.get(number + 1)) { + (Some(start), Some(end)) => Ok((*start, *end)), + _ => Err(StorageError::Integrity("group record directory")), + } + } +} + +fn count_of(field: Option<&[u8]>, group_length: usize) -> StorageResult { + let mut count = [0; 4]; + count.copy_from_slice(field.ok_or(StorageError::Integrity("group framing"))?); + let count = u32::from_le_bytes(count) as usize; + if !(1..=RECORD_COUNT_LIMIT).contains(&count) || 4 + 4 * count > group_length { + return Err(StorageError::Integrity("group record directory")); + } + Ok(count) +} + +/// What the bytes a caller holds answer. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Sparse { + /// The answer. + Ready(T), + /// The pack range to acquire before asking again. + Need(PackRange), +} + +/// Whether a record of the group `view` of a `lane` pack has an edge these +/// readers can report. A compressed group and the role-dependent `Ordinary` +/// lane take the decoding reader. +pub fn has_sparse_edges(lane: PackLane, view: GroupView) -> bool { + lane != PackLane::Ordinary && view.codec == GroupCodec::Raw +} + +/// The record directory of the uncompressed group `view`, from the pack bytes +/// `held` returns. +/// +/// `held(offset, length)` returns exactly `length` bytes at pack offset +/// `offset`, or `None` when the caller does not hold all of them. +pub fn sparse_directory<'a>( + view: GroupView, + held: impl Fn(usize, usize) -> Option<&'a [u8]>, +) -> StorageResult> { + let length = view + .end + .checked_sub(view.start) + .filter(|length| *length >= 4) + .ok_or(StorageError::Integrity("group framing"))?; + let need = |length: usize| { + Sparse::Need(PackRange { + offset: view.start, + length, + }) + }; + let Some(count) = held(view.start, 4) else { + return Ok(need(length.min(HEAD_BYTES))); + }; + let count = count_of(Some(count), length)?; + let Some(ends) = held(view.start + 4, 4 * count) else { + return Ok(need((4 + 4 * count + record::HEADER_BYTES).min(length))); + }; + RecordDirectory::parse(count, ends, length).map(Sparse::Ready) +} + +/// The edge of the record `location` names in the group `view` of a `lane` +/// pack, exactly as [`super::read::ChainBases::edge`] reports it, from the +/// group's `directory` and the pack bytes `held` returns. +pub fn sparse_edge<'a>( + lane: PackLane, + view: GroupView, + directory: &RecordDirectory, + location: &ObjectLocation, + held: impl Fn(usize, usize) -> Option<&'a [u8]>, +) -> StorageResult> { + if !has_sparse_edges(lane, view) { + return Err(StorageError::Integrity("encoded record edge")); + } + if lane == PackLane::Singleton && location.record_number != 0 { + return Err(StorageError::Integrity("singleton record ordinal")); + } + let (start, end) = directory.range(location.record_number)?; + let width = end - start; + let header = width.min(record::HEADER_BYTES); + let Some(bytes) = held(view.start + start, header) else { + return Ok(Sparse::Need(PackRange { + offset: view.start + start, + length: header, + })); + }; + Ok(Sparse::Ready(ChainEdge { + base: record::header_base(lane, bytes, width, location.canonical_length)?, + width: width as u64, + })) +} diff --git a/core/crates/layerfs-storage/src/encoding/delta/mod.rs b/core/crates/layerfs-storage/src/encoding/delta/mod.rs index 0cf400c9e1..4514a764ef 100644 --- a/core/crates/layerfs-storage/src/encoding/delta/mod.rs +++ b/core/crates/layerfs-storage/src/encoding/delta/mod.rs @@ -4,6 +4,7 @@ pub mod branch; pub mod candidates; +pub mod edge; pub mod read; pub mod record; pub mod select; diff --git a/core/crates/layerfs-storage/src/encoding/delta/read.rs b/core/crates/layerfs-storage/src/encoding/delta/read.rs index 2850201b81..071b31ec3e 100644 --- a/core/crates/layerfs-storage/src/encoding/delta/read.rs +++ b/core/crates/layerfs-storage/src/encoding/delta/read.rs @@ -440,15 +440,45 @@ impl<'a> ChainBases<'a> { workspace: &mut DecompressionWorkspace, location: &ObjectLocation, ) -> StorageResult { - let base = self.base_of(connection, workspace, location)?; - let (selected, _) = group_of(self.packs, connection, location)?; - Ok(ChainEdge { - base, - width: selected_record_width(selected, location)?, - }) + stored_edge( + connection, + self.packs, + &mut self.groups, + workspace, + &mut self.group_decodes, + location, + ) } } +/// [`ChainBases::edge`] over caches the caller owns. +/// +/// A walk that also decodes passes its own decoded-group cache, so a +/// compressed group it already decoded, or whose other records it asks about, +/// is decompressed once and not once per record. +pub fn stored_edge( + connection: &dyn Source, + packs: &mut crate::encoding::PackCache, + groups: &mut GroupCache, + workspace: &mut DecompressionWorkspace, + group_decodes: &mut u64, + location: &ObjectLocation, +) -> StorageResult { + let base = stored_base( + connection, + packs, + groups, + workspace, + group_decodes, + location, + )?; + let (selected, _) = group_of(packs, connection, location)?; + Ok(ChainEdge { + base, + width: selected_record_width(selected, location)?, + }) +} + /// One stored record of a chain, as a cost walk meets it. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ChainEdge { diff --git a/core/crates/layerfs-storage/src/encoding/delta/record.rs b/core/crates/layerfs-storage/src/encoding/delta/record.rs index f9255077e6..b7989bff2e 100644 --- a/core/crates/layerfs-storage/src/encoding/delta/record.rs +++ b/core/crates/layerfs-storage/src/encoding/delta/record.rs @@ -36,6 +36,10 @@ pub const fn is_stored(tag: u8) -> bool { /// Width of one canonical identity. const ID_BYTES: usize = 32; +/// The widest record prefix that carries a payload lane's tag, length fields +/// and base identity: the framed lane's nine bytes and one identity. +pub const HEADER_BYTES: usize = 9 + ID_BYTES; + /// One parsed lane record. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ParsedRecord<'a> { @@ -358,3 +362,59 @@ pub fn stored_base( } Ok(parse(lane, record, canonical_length)?.base) } + +/// The direct base a payload-lane record names, read from its first bytes. +/// +/// `header` is the record's first `min(record_length, HEADER_BYTES)` bytes and +/// `record_length` its complete stored width, which the group directory gives. +/// This accepts and refuses exactly the records [`stored_base`] does on the +/// complete record: the tag, the base identity, the length fields and the +/// non-empty frame are all decided by the header and the width, so only the +/// frame's own bytes go unread. The `Ordinary` lane has no fixed header (its +/// groups are compressed and its grammar depends on the role) and is refused. +pub fn header_base( + lane: PackLane, + header: &[u8], + record_length: usize, + canonical_length: usize, +) -> StorageResult> { + let fixed = match lane { + PackLane::WholeFile => 1, + PackLane::Chunk => 5, + PackLane::Singleton => 9, + PackLane::Ordinary => return Err(StorageError::Integrity("ordinary record header")), + }; + if header.len() != record_length.min(HEADER_BYTES) || header.len() < fixed { + return Err(StorageError::Integrity("record header width")); + } + let field = |at: usize| { + let mut bytes = [0; 4]; + bytes.copy_from_slice(&header[at..at + 4]); + u32::from_le_bytes(bytes) as usize + }; + let (base, start) = match header[0] { + FULL_TAG | STORED_TAG => (None, fixed), + PREFIX_TAG => ( + Some(ObjectId::from_bytes( + header + .get(fixed..fixed + ID_BYTES) + .ok_or(StorageError::Integrity("record base identity"))?, + )?), + fixed + ID_BYTES, + ), + _ => return Err(StorageError::Integrity("record tag")), + }; + let frame = record_length + .checked_sub(start) + .filter(|frame| *frame > 0) + .ok_or(StorageError::Integrity("record frame"))?; + let raw_known = match lane { + // The compact lane drops its length fields; the locator carries them. + PackLane::WholeFile => canonical_length > crate::policy::WHOLE_FILE_CANONICAL_OVERHEAD, + _ => field(1) != 0, + }; + if !raw_known || (lane == PackLane::Singleton && field(5) != frame) { + return Err(StorageError::Integrity("record framing")); + } + Ok(base) +} diff --git a/core/crates/layerfs-storage/tests/sparse_edges.rs b/core/crates/layerfs-storage/tests/sparse_edges.rs new file mode 100644 index 0000000000..270b5d1927 --- /dev/null +++ b/core/crates/layerfs-storage/tests/sparse_edges.rs @@ -0,0 +1,334 @@ +//! A payload record's edge read from bounded ranges agrees with the reader +//! that borrows the complete record. +//! +//! `header_base` must accept and refuse exactly the records `stored_base` +//! does, and `sparse_edge` must report the base and width the complete group +//! gives, whatever pieces of the pack the caller happens to hold. A group's +//! directory is parsed once, so asking for every record of a group costs one +//! pass over it and not one pass per record. +use layerfs_content::{ObjectId, ObjectRole}; +use layerfs_storage::encoding::delta::edge::{ + has_sparse_edges, sparse_directory, sparse_edge, RecordDirectory, Sparse, HEAD_BYTES, +}; +use layerfs_storage::encoding::delta::read::ChainEdge; +use layerfs_storage::encoding::delta::record::{ + header_base, stored_base, FULL_TAG, HEADER_BYTES, PREFIX_TAG, STORED_TAG, +}; +use layerfs_storage::encoding::framed_record; +use layerfs_storage::location::ObjectLocation; +use layerfs_storage::pack::layout::{GroupCodec, GroupView, PackLane}; +use layerfs_storage::policy::WHOLE_FILE_CANONICAL_OVERHEAD; + +const PAYLOAD_LANES: [PackLane; 3] = [PackLane::WholeFile, PackLane::Chunk, PackLane::Singleton]; + +fn base() -> ObjectId { + ObjectId::for_bytes(b"sparse-edges/base") +} + +/// One record of `lane` as the encoder frames it, with every field free so a +/// case can break exactly one of them. +fn record(lane: PackLane, tag: u8, raw: u32, declared_frame: u32, frame: &[u8]) -> Vec { + let mut out = vec![tag]; + if lane != PackLane::WholeFile { + out.extend_from_slice(&raw.to_le_bytes()); + } + if lane == PackLane::Singleton { + out.extend_from_slice(&declared_frame.to_le_bytes()); + } + if tag == PREFIX_TAG { + out.extend_from_slice(&base().to_bytes()); + } + out.extend_from_slice(frame); + out +} + +fn header_of(record: &[u8]) -> &[u8] { + &record[..record.len().min(HEADER_BYTES)] +} + +#[test] +fn the_header_reader_accepts_and_refuses_what_the_record_reader_does() { + let mut accepted = 0; + let mut refused = 0; + for lane in PAYLOAD_LANES { + for tag in [FULL_TAG, PREFIX_TAG, STORED_TAG, 3, 255] { + for raw in [0u32, 1, 70_000] { + for frame_length in [0usize, 1, 7, 64] { + for declared in [frame_length as u32, frame_length as u32 + 1] { + for canonical in [ + 0, + WHOLE_FILE_CANONICAL_OVERHEAD, + WHOLE_FILE_CANONICAL_OVERHEAD + 1, + WHOLE_FILE_CANONICAL_OVERHEAD + 70_000, + ] { + let frame = vec![0xA5; frame_length]; + let whole = record(lane, tag, raw, declared, &frame); + // Every truncation too: a short record must be + // refused by both, never read past by either. + for cut in 0..=whole.len().min(HEADER_BYTES + 2) { + let record = if cut == whole.len().min(HEADER_BYTES + 2) { + &whole[..] + } else { + &whole[..cut] + }; + let complete = + stored_base(lane, record, canonical, ObjectRole::WholeFile); + let header = + header_base(lane, header_of(record), record.len(), canonical); + match (&complete, &header) { + (Ok(left), Ok(right)) => { + assert_eq!(left, right); + accepted += 1; + } + (Err(_), Err(_)) => refused += 1, + _ => panic!( + "{lane:?} tag {tag} raw {raw} frame {frame_length} \ + declared {declared} canonical {canonical} cut {cut}: \ + complete {complete:?}, header {header:?}" + ), + } + } + } + } + } + } + } + } + assert!(accepted > 100 && refused > 100, "{accepted} {refused}"); +} + +#[test] +fn the_header_reader_refuses_a_header_of_the_wrong_width_and_the_ordinary_lane() { + let whole = record(PackLane::Chunk, PREFIX_TAG, 9, 0, &[1; 200]); + assert_eq!( + header_base(PackLane::Chunk, header_of(&whole), whole.len(), 0).unwrap(), + Some(base()) + ); + // One byte short or long of the declared prefix is not the record's header. + assert!(header_base(PackLane::Chunk, &whole[..HEADER_BYTES - 1], whole.len(), 0).is_err()); + assert!(header_base(PackLane::Chunk, &whole[..HEADER_BYTES + 1], whole.len(), 0).is_err()); + assert!(header_base(PackLane::Ordinary, &[FULL_TAG, 1], 2, 0).is_err()); +} + +/// A Raw group body: the record count, each record's end offset, the records. +fn group(records: &[Vec]) -> Vec { + let mut out = (records.len() as u32).to_le_bytes().to_vec(); + let mut end = 0u32; + for record in records { + end += record.len() as u32; + out.extend_from_slice(&end.to_le_bytes()); + } + for record in records { + out.extend_from_slice(record); + } + out +} + +fn location(record_number: usize, canonical_length: usize) -> ObjectLocation { + ObjectLocation { + object_id: ObjectId::for_bytes(&record_number.to_le_bytes()), + role: ObjectRole::Chunk, + canonical_length, + pack_id: 7, + group_number: 0, + record_number, + } +} + +/// Runs the sparse readers the way a walk does: start with nothing held, +/// acquire each range they name, and count the acquisitions. +fn walk( + lane: PackLane, + view: GroupView, + pack: &[u8], + location: &ObjectLocation, +) -> (ChainEdge, usize, usize) { + let mut held: Vec<(usize, Vec)> = Vec::new(); + let mut acquired = 0; + for round in 0..8 { + let have = |offset: usize, length: usize| { + held.iter().find_map(|(at, bytes)| { + let from = offset.checked_sub(*at)?; + bytes.get(from..from + length) + }) + }; + let range = match sparse_directory(view, have).unwrap() { + Sparse::Need(range) => range, + Sparse::Ready(directory) => { + match sparse_edge(lane, view, &directory, location, have).unwrap() { + Sparse::Ready(edge) => return (edge, round, acquired), + Sparse::Need(range) => range, + } + } + }; + assert!(range.offset >= view.start && range.offset + range.length <= view.end); + acquired += range.length; + held.push(( + range.offset, + pack[range.offset..range.offset + range.length].to_vec(), + )); + } + panic!("the walk never settled"); +} + +#[test] +fn a_sparse_walk_reports_the_complete_groups_edge_without_its_payload() { + for lane in [PackLane::WholeFile, PackLane::Chunk] { + for count in [1usize, 2, 5, 130, 400] { + let canonical = WHOLE_FILE_CANONICAL_OVERHEAD + 9_000; + let records: Vec> = (0..count) + .map(|index| { + let tag = [FULL_TAG, PREFIX_TAG, STORED_TAG][index % 3]; + // Widths from one byte of frame to several kilobytes. + let frame = vec![index as u8; 1 + (index * 37) % 9_000]; + record(lane, tag, 9_000, 0, &frame) + }) + .collect(); + let body = group(&records); + let start = 40; + let mut pack = vec![0xEE; start]; + pack.extend_from_slice(&body); + pack.extend_from_slice(&[0xEE; 13]); + let view = GroupView { + start, + end: start + body.len(), + decoded_length: body.len(), + codec: GroupCodec::Raw, + }; + let directory = 4 + 4 * count; + for number in [0, count / 2, count - 1] { + let location = location(number, canonical); + let complete = framed_record(&body, number).unwrap(); + let expected = stored_base(lane, complete, canonical, location.role).unwrap(); + let (edge, rounds, acquired) = walk(lane, view, &pack, &location); + assert_eq!(edge.base, expected, "{lane:?} {count} {number}"); + assert_eq!(edge.width, complete.len() as u64); + assert!(rounds <= 3, "{rounds} rounds"); + // The head, at most a wider directory, at most one header: + // never the payload behind them. + assert!( + acquired <= HEAD_BYTES + directory + 2 * HEADER_BYTES, + "{lane:?} {count} {number}: {acquired} of {}", + body.len() + ); + } + // An ordinal the directory does not hold is refused, as it is by + // the complete reader. + let held = |offset: usize, length: usize| pack.get(offset..offset + length); + let Sparse::Ready(directory) = sparse_directory(view, held).unwrap() else { + panic!("the whole pack is held"); + }; + assert_eq!(directory, RecordDirectory::of(&body).unwrap()); + assert_eq!(directory.count(), count); + // Every record's range is the framing reader's. + for number in 0..count { + let (from, to) = directory.range(number).unwrap(); + assert_eq!(&body[from..to], framed_record(&body, number).unwrap()); + } + let past = location(count, canonical); + assert!(sparse_edge(lane, view, &directory, &past, held).is_err()); + assert!(directory.range(count).is_err()); + assert!(framed_record(&body, count).is_err()); + } + } +} + +#[test] +fn a_broken_directory_is_refused_and_an_encoded_group_is_left_to_the_decoder() { + let records = vec![ + record(PackLane::Chunk, FULL_TAG, 5, 0, &[1; 30]), + record(PackLane::Chunk, FULL_TAG, 5, 0, &[2; 30]), + ]; + let body = group(&records); + let view = |codec| GroupView { + start: 0, + end: body.len(), + decoded_length: body.len(), + codec, + }; + let location = location(1, 0); + let raw = view(GroupCodec::Raw); + let held = |offset: usize, length: usize| body.get(offset..offset + length); + let directory = RecordDirectory::of(&body).unwrap(); + assert!(matches!( + sparse_edge(PackLane::Chunk, raw, &directory, &location, held).unwrap(), + Sparse::Ready(_) + )); + // A compressed group and the Ordinary lane have no edge to read this way: + // the caller asks first, and the reader refuses rather than guess. + assert!(has_sparse_edges(PackLane::Chunk, raw)); + assert!(!has_sparse_edges( + PackLane::Chunk, + view(GroupCodec::Zstandard) + )); + assert!(!has_sparse_edges(PackLane::Ordinary, raw)); + let compressed = view(GroupCodec::Zstandard); + assert!(sparse_edge(PackLane::Chunk, compressed, &directory, &location, held).is_err()); + assert!(sparse_edge(PackLane::Ordinary, raw, &directory, &location, held).is_err()); + // A record count of zero, one past the body, and an end that runs backwards. + for (at, value) in [(0usize, 0u32), (0, 9_999), (4, 0), (8, 3)] { + let mut broken = body.clone(); + broken[at..at + 4].copy_from_slice(&value.to_le_bytes()); + let held = |offset: usize, length: usize| broken.get(offset..offset + length); + assert!( + sparse_directory(raw, held).is_err(), + "field at {at} = {value}" + ); + assert!( + RecordDirectory::of(&broken).is_err(), + "field at {at} = {value}" + ); + assert!( + framed_record(&broken, 1).is_err(), + "field at {at} = {value}" + ); + } + // A group shorter than its count field, and one truncated inside its + // directory, are refused by both readers. + for cut in [0, 3, 7, 11] { + assert!(RecordDirectory::of(&body[..cut]).is_err(), "cut {cut}"); + assert!(framed_record(&body[..cut], 0).is_err(), "cut {cut}"); + } + // A singleton pack holds one record: any other ordinal is refused. + assert!(sparse_edge(PackLane::Singleton, raw, &directory, &location, held).is_err()); +} + +#[test] +fn every_record_of_a_full_group_is_answered_in_one_pass_over_its_directory() { + // The widest group the format admits. One directory parse answers all of + // its records; asking the framing reader for each one scans the directory + // once per record, which is what this reader exists to avoid. + let count = layerfs_storage::policy::RECORD_COUNT_LIMIT; + let records: Vec> = (0..count) + .map(|index| { + let tag = [FULL_TAG, PREFIX_TAG][index % 2]; + record(PackLane::Chunk, tag, 7, 0, &[index as u8; 3]) + }) + .collect(); + let body = group(&records); + let view = GroupView { + start: 0, + end: body.len(), + decoded_length: body.len(), + codec: GroupCodec::Raw, + }; + let held = |offset: usize, length: usize| body.get(offset..offset + length); + let Sparse::Ready(directory) = sparse_directory(view, held).unwrap() else { + panic!("the whole group is held"); + }; + assert_eq!(directory.count(), count); + let mut bases = 0; + let mut width = 0; + for number in 0..count { + let location = location(number, 0); + let Sparse::Ready(edge) = + sparse_edge(PackLane::Chunk, view, &directory, &location, held).unwrap() + else { + panic!("record {number}"); + }; + bases += usize::from(edge.base == Some(base())); + width += edge.width as usize; + } + assert_eq!(bases, count / 2); + assert_eq!(width, body.len() - 4 - 4 * count); +} diff --git a/core/docs/issues/project-41/LayerFS-GC-spec.md b/core/docs/issues/project-41/LayerFS-GC-spec.md index 65abed9252..9f1f2f2397 100644 --- a/core/docs/issues/project-41/LayerFS-GC-spec.md +++ b/core/docs/issues/project-41/LayerFS-GC-spec.md @@ -602,9 +602,13 @@ finds (`engine/crates/layerfs-persistence/src/diagnostic/history.rs:155-198`). (#42 item 36). - Window by window, sort lookups by object ID before locating them. - Whole files and chunks have no logical references; only their delta base - is needed. Reading it still loads and authenticates the whole group - (`delta/read.rs:561-571`); on SQLite the payload part of the walk runs - inside the copy (§7 step 4), and the PostgreSQL mark reads it in Phase M. + is needed. In an uncompressed payload group the mark reads the group's + record directory and each record's first bytes (`delta/edge.rs`, + `history/objects/edges.rs`), never the payload, so those bytes are not + authenticated by the mark; a compressed group and the `Ordinary` lane still + load the whole group (`delta/read.rs`). On SQLite the payload part of the + walk runs inside the copy (§7 step 4), and the PostgreSQL mark reads it in + Phase M. ### 6.3 Visited set @@ -830,6 +834,17 @@ reported as `markBytes`; the Phase A table copy itself moves no pack bytes. **Phase B — swap (one short transaction).** +> **As built (2026-11-18, `0bb8989529`).** The row counts and sums of step 9 +> are taken at the start of this transaction, before step 5: the transaction +> first takes `ACCESS EXCLUSIVE` on the two scratch tables (a waiting lock, +> not `NOWAIT`; nothing else writes them) and counts and sums them in one +> statement, then takes the Store-wide lock of step 5. The scratch lock is +> held to COMMIT, so the rows counted are the rows moved, and the Store-wide +> lock is no longer held across those scans. A count or sum mismatch is +> therefore reported (`Integrity`) before the tuple re-check of step 6 can +> report `Conflict`. The fingerprint, trigger and `logical` checks and the +> single `live_heads_digest` read stay under the Store-wide lock. + 5. `LOCK TABLE` every base table of the Store schema (from `pg_class`, including `store_lane`, `store_totals`, `history_meta`, `content_signature`, the history tables and `gc_intent`/`gc_op`) in one diff --git a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md index 34f634fa6a..c7f24753d0 100644 --- a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md +++ b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md @@ -263,7 +263,7 @@ AC-51 does not depend on the projection alone. `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`. - Verified here: `sqlite_compact_matrix` (1 test); the persistence suites - `sqlite_compact` (34), `sqlite_compact_crash` (24), + `sqlite_compact` (36), `sqlite_compact_crash` (24), `sqlite_compact_fixtures` (1), `admission` (10), `path_removal` (4), `branch_removal` (4), `project_delete` (4), `epoch_fence` (7), `gc_backfill` (5), `kept_objects` (7); the Store-service path suites @@ -272,7 +272,7 @@ AC-51 does not depend on the projection alone. `sdk_path_removal` (5); and the `layerfs-control project_delete_worker` (8), the `layerfs-store project_delete` (18), `compact_report_status` (6) and the - PostgreSQL AC-44 suite `postgresql_compact` (30 tests; AC-44's nine are + PostgreSQL AC-44 suite `postgresql_compact` (31 tests; AC-44's nine are `a_fully_privileged_role_compacts`, `a_role_without_create_on_the_database_is_refused_as_unsupported`, `a_role_without_create_on_the_store_schema_is_refused_as_unsupported`, diff --git a/docs/architecture/engine/store/postgresql.md b/docs/architecture/engine/store/postgresql.md index c9bd97b48f..0a0566a49f 100644 --- a/docs/architecture/engine/store/postgresql.md +++ b/docs/architecture/engine/store/postgresql.md @@ -125,6 +125,7 @@ transaction-pooling proxy never carries one session's settings to another. | `Write` | `READ COMMITTED` + `synchronous_commit` + `lock_timeout` | `FOR UPDATE`, unless nested in the held lane | history writes, allocation, acquisition writes | | `Stage` | `READ COMMITTED` + `synchronous_commit` | no | schema create, merge records | | `Publish` | as `Write` | `FOR UPDATE` after the pack bodies, unless nested | one Save publication, up to 16 MiB − 1 | +| `Snapshot` | `REPEATABLE READ` + `synchronous_commit` + `jit` off | no | the garbage-collection mark: reads history, locators and packs from one snapshot and writes only `kept_node` | | `Immutable` | none; autocommit statements | no | locates, pack reads, inode values | | `Statement` | none; exactly one autocommit statement | no | `branch_snapshot`, a Commit or Layer by id, a LayerStack by name, `digest_roots` (up to `FILE_DIGEST_WINDOW` = 1,024 contents) | @@ -135,6 +136,21 @@ rows it resolves to coherently. Statements outside a transaction name the schema (`{s}` in [`queries/history`](../../../../engine/crates/layerfs-persistence/sql/postgresql/queries/history)). Writes are never `SERIALIZABLE` and never retried. +The mark runs with the server's JIT compiler off: the planner's estimate for +the history closure crosses the JIT threshold, and compiling it cost about +400 ms for a statement that runs in about 10 ms. Inside its snapshot the mark +also reads ahead, because each statement costs a round trip: + +| Read | One statement covers | Bound | Source | +| --- | --- | --- | --- | +| Pack heads and record ranges of payload packs | 256 packs; 1,024 ranges or 512 KiB | `HEAD_PACKS`, `BATCH_RANGES`, `BATCH_BYTES` | [`edges.rs`](../../../../engine/crates/layerfs-persistence/src/history/objects/edges.rs) | +| Metadata packs the walk decodes | the next 128 packs of the wave, each up to 32 KiB (a longer pack is read alone) | `AHEAD_PACKS`, `AHEAD_BYTES` | [`objects.rs`](../../../../engine/crates/layerfs-persistence/src/history/objects.rs), [`read.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/objects/read.rs) | +| Inode values of a leaf | the blocks holding up to 4,096 ordinals past the leaf's own; at most 65,536 values kept | `AHEAD_VALUES`, `HELD_VALUES` | [`objects.rs`](../../../../engine/crates/layerfs-persistence/src/history/objects.rs), [`reads.rs`](../../../../engine/crates/layerfs-persistence/src/backend/postgresql/metadata/reads.rs) | + +A SQLite Store ignores the pack hint and reads no values ahead. Rows read +ahead are the rows the single reads return, so the mark's result and its +`mark_bytes` do not change. + ### Generation fence The session keeps the `catalog_incarnation` its open read diff --git a/docs/architecture/engine/store/sqlite.md b/docs/architecture/engine/store/sqlite.md index b689a298a7..f4b8e0ff4f 100644 --- a/docs/architecture/engine/store/sqlite.md +++ b/docs/architecture/engine/store/sqlite.md @@ -154,12 +154,31 @@ quarantines the session and keeps the lock | Operation | Bound or rule | Source | | --- | --- | --- | -| `seal` | Consumes the sole session; never runs on a live shared Store (a daemon keeps its Store open across Saves and Commits) | [`seal.rs`](../../../../engine/crates/layerfs-persistence/src/store/seal.rs), [`lifecycle/seal.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs) | +| `seal` | Consumes the sole session; never runs on a live shared Store (a daemon keeps its Store open across Saves and Commits). It holds the writer lane from its checkpoint to its sidecar check; a refused seal closes its connection under that same hold and then frees the lane | [`seal.rs`](../../../../engine/crates/layerfs-persistence/src/store/seal.rs), [`lifecycle/seal.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal.rs) | | macOS extent release | Unused extents past logical EOF move to a same-volume scratch, which is removed; no logical byte is copied or truncated. The one module allowed `unsafe` | [`seal_allocation.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/seal_allocation.rs), [`file_control.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/file_control.rs) | | `snapshot` | One read transaction; page size, `auto_vacuum`, `user_version` and `application_id` carry over; destination must not exist; a busy source is `Busy`, never retried; a failed copy may leave a file for the caller to remove | [`snapshot.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/snapshot.rs) | | Free-space guard | A pack batch must leave 64 MiB after counting its bytes twice (WAL, then file); metadata-only batches are not guarded | [`headroom.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/headroom.rs) | | `reclaim_space(pages)` | `1..=RECLAMATION_PAGE_LIMIT` (512) pages by `incremental_vacuum`, stepped in one short write until the free list is empty or the page budget is spent (SQLite 3.40 removes one page per statement, not `N`); only Stores created with `auto_vacuum=2` | [`reclamation.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/lifecycle/reclamation.rs) | +## Compaction rebuild + +`SqliteHandles::compact` marks under the writer lane, seals the file, builds +`.compact` beside it from the retained rows, checks it through an +`immutable=1` reader and renames it into place. The build's cost is the copy +of the retained packs, so its steps are shaped around that copy. + +| Step | Rule | Source | +| --- | --- | --- | +| Op record | `.compact-op` is rewritten by rename at admission and at the build, verify, install and cleanup phases. The admission record stands through the mark: a later call judges a leftover by the run, generation and owner it names, and `store.status` reads its phase as `mark` | [`op.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/op.rs), [`run.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs) | +| Fresh file | The sealed Store's tables, then the rows, then its indexes and triggers; `journal_mode=OFF` and `synchronous=OFF` while loading, then WAL. The sealed file is attached `immutable=1` with a 256 MiB `mmap_size` | [`build.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/build.rs) | +| File size | macOS sizes the fresh file for the retained pack bytes before the load (one advisory `F_PREALLOCATE`, one `set_len`), because on APFS a page write that also moves the end of the file costs about four times one inside a sized file. The closed file is cut to the pages its header counts. Linux leaves the file to grow | [`space.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/space.rs) | +| Pack copy | The table that holds the pack payload (`pack`, or `pack_unit` in the group-row layouts) is copied one window per statement: `WINDOW_BYTES` (64 MiB) of retained payload, ended early by a dead run longer than `GAP_BYTES` (8 MiB). Every other table is one statement | [`window.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/window.rs), [`build.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/build.rs) | +| Read-ahead | Before each window the sealed file's matching range is asked for once, through a second read-only descriptor: `F_RDADVISE` on macOS, `POSIX_FADV_WILLNEED` in 128 KiB steps on Linux. Without it a source that is not in memory is read one pack per device round trip. The range assumes packs lie in the file in id order; a hint only, so another layout loses the benefit and nothing else | [`window.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/window.rs) | +| Visited scratch | The mark writes one `visited` row per reached object into `.compact-visited`; the retained packs (`live_pack`) are derived from those rows in one statement when the mark finishes | [`compact_visit.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact_visit.rs) | +| Verification | One `immutable=1` reader of the fresh file: the schema check, one statement that reads the row counts and recomputed sums with one pass per table, the live-heads digest and the tuple. The sealed file's tuple and digest are read back through one reader as well | [`verify.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/verify.rs) | +| Synchronization | The op record is synchronized at each write, the fresh file before the rename and the directory after it | [`op.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/op.rs), [`install.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/install.rs) | +| Settle | The run lets go of the writer lane after the rename, removes its artifacts, then takes the lane again for the one transaction that clears its admission intent and the `kept_node` rows in the installed Store. A writer has no busy handler, so a settle that wrote beside one would leave it `Busy`; the settle waits its turn instead. The op record is removed last | [`install.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/compact/install.rs) | + ## Heads | Function | Returns | Source | diff --git a/engine/crates/layerfs-persistence/examples/gc_profile.rs b/engine/crates/layerfs-persistence/examples/gc_profile.rs new file mode 100644 index 0000000000..fe95321e92 --- /dev/null +++ b/engine/crates/layerfs-persistence/examples/gc_profile.rs @@ -0,0 +1,125 @@ +//! Where one GC mark, or one whole compaction, spends its time on a kept +//! `gc_bench` Store: wall time, this thread's SQL work counters, and the +//! statements that cost the most. +//! +//! ```text +//! cargo run -p layerfs-persistence --release --example gc_bench -- --keep --tmpdir DIR +//! cargo run -p layerfs-persistence --release --example gc_profile -- DIR//store.sqlite +//! ``` +//! +//! `--marks N` runs N marks (default 2: the second shows a warm cache). +//! `--compact` then runs one compaction; it changes the Store. `--top N` +//! prints the N costliest statements (default 12). The rebuild's copy runs on +//! its own connection, so its statements are not in the profile; its time is +//! the compaction's wall time less the profiled statements. +#![allow(dead_code)] +#[path = "gc_bench/gc_fixture.rs"] +mod gc_fixture; + +use layerfs_persistence::{ + profile_statements, take_statement_profile, thread_work, CompactPath, CompactRequest, +}; +use std::time::Instant; + +fn counters(label: &str, wall_ms: f64, work: &layerfs_persistence::SqlWork) { + println!( + "{label}: wall_ms={wall_ms:.1} statements={} statement_ms={:.1} vm_steps={} \ + fullscan_steps={} sorts={} autoindex_rows={} returned_rows={} blob_opens={} \ + blob_reads={} blob_read_bytes={} blob_read_ms={:.1} transactions={} commit_ms={:.1}", + work.statements, + work.statement_ns as f64 / 1e6, + work.vm_steps, + work.fullscan_steps, + work.sorts, + work.autoindex_rows, + work.returned_rows, + work.blob_open_calls, + work.blob_read_calls, + work.blob_read_bytes, + work.blob_read_ns as f64 / 1e6, + work.transactions, + work.commit_ns as f64 / 1e6 + ); +} + +fn statements(top: usize) { + let profile = take_statement_profile(); + let total: u64 = profile.iter().map(|(_, cost)| cost.wall_ns).sum(); + println!( + " {} distinct statements, {:.1} ms; costliest first (ms, calls, rows, text):", + profile.len(), + total as f64 / 1e6 + ); + for (sql, cost) in profile.iter().take(top) { + println!( + " {:>9.2} {:>8} {:>9} {sql}", + cost.wall_ns as f64 / 1e6, + cost.calls, + cost.rows + ); + } +} + +fn main() { + let mut store = None; + let (mut marks, mut top, mut compact) = (2usize, 12usize, false); + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + let mut number = |name: &str| -> usize { + args.next() + .and_then(|value| value.parse().ok()) + .unwrap_or_else(|| panic!("{name} needs a number")) + }; + match arg.as_str() { + "--marks" => marks = number("--marks"), + "--top" => top = number("--top"), + "--compact" => compact = true, + _ => store = Some(std::path::PathBuf::from(arg)), + } + } + let store = store.expect("usage: gc_profile STORE [--marks N] [--top N] [--compact]"); + let handles = gc_fixture::reopen(&store); + for index in 1..=marks { + let before = thread_work(); + profile_statements(); + let started = Instant::now(); + let mark = handles.history.mark().expect("mark"); + let wall = started.elapsed().as_secs_f64() * 1e3; + counters( + &format!("mark {index}"), + wall, + &thread_work().since(&before), + ); + println!( + " objects={} packs={} history_nodes={} mark_bytes={}", + mark.objects().len(), + mark.objects().packs().len(), + mark.history().len(), + mark.mark_bytes() + ); + statements(top); + } + if compact { + let before = thread_work(); + profile_statements(); + let started = Instant::now(); + let receipt = handles + .compact(&CompactRequest { + op_id: [0xC7; 16], + ..CompactRequest::default() + }) + .expect("compact"); + let wall = started.elapsed().as_secs_f64() * 1e3; + counters("compact", wall, &thread_work().since(&before)); + println!( + " path={} physical_shrink={} peak_temp_bytes={}", + match receipt.path { + CompactPath::None => "none", + CompactPath::Rebuild => "rebuild", + }, + receipt.physical_shrink, + receipt.peak_temp_bytes + ); + statements(top); + } +} diff --git a/engine/crates/layerfs-persistence/examples/pg_gc_profile.rs b/engine/crates/layerfs-persistence/examples/pg_gc_profile.rs new file mode 100644 index 0000000000..394913576e --- /dev/null +++ b/engine/crates/layerfs-persistence/examples/pg_gc_profile.rs @@ -0,0 +1,663 @@ +//! A PostgreSQL garbage-collection fixture and one counted `store.compact` +//! or mark over it: the PostgreSQL counterpart of `gc_bench` and +//! `gc_profile`, which are SQLite only. A diagnostic, not a measurement +//! harness: it writes no receipt and claims nothing. +//! +//! ```text +//! LAYERFS_TEST_POSTGRES_URL="host=... dbname=postgres ..." \ +//! pg_gc_profile build # creates database with one Store +//! pg_gc_profile run