From 7ceff82a5f481e760a0fbc71a60b9e32db646d42 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 22:55:53 +0800 Subject: [PATCH 1/6] refactor(persistence): one helper for the SQLite additive migrations The eight open-time migrations in backend/sqlite/schema.rs each repeated the same skeleton: return when already applied, read the recorded schema, compare it with an earlier source and with the live definition, apply, record the new source and definition. `Session::migrate` now owns that skeleton; each migration states only its present check, the sources it accepts and records, and what it applies. No behaviour change: the same checks run in the same order inside the same single write transaction, and the sources are still computed only after the present check. `binding_hint_table()` joins the sibling text helpers. It panics if the compiled-in history schema lacks the table, where the old code returned Integrity; the text is a compile-time constant. Checks (macOS host, Rust 1.85.1): cargo check and fmt; persistence tests old_format_columns, old_format_open, gc_backfill, lineage_backfill, binding_hint_blocks, kept_node, diagnostic_schema, store_diagnostic and path_removal pass (46 tests). No test changed or deleted. Rebased onto 4c296e9297 (layerfs#734) on 2026-11-18. The test counts above are from before the rebase. This commit compiles with its tests on the new base; the suites were rerun at the branch tip, not at this commit. Production LOC: 188,972 -> 188,837 (delta -135) core 40,589 -> 40,589 (0) engine 90,631 -> 90,496 (-135) layerfs-persistence -135 control 51,230 -> 51,230 (0) api 6,522 -> 6,522 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py --json on parent 4c296e9297 and the committed tree (recomputed after the rebase onto 4c296e9297) Co-Authored-By: Claude Fable 5.2 --- .../src/backend/sqlite/schema.rs | 414 ++++++------------ 1 file changed, 138 insertions(+), 276 deletions(-) diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs index 3ee6235df..ca7081cda 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/schema.rs @@ -110,6 +110,24 @@ fn kept_node_table() -> &'static str { &GC[start..end] } +/// The `binding_hint_block` table as `history.sql` writes it. +fn binding_hint_table() -> &'static str { + let start = HISTORY + .find("CREATE TABLE binding_hint_block (") + .expect("the history schema holds binding_hint_block"); + let end = start + + HISTORY[start..] + .find(";\n") + .expect("binding_hint_block ends") + + 2; + &HISTORY[start..end] +} + +/// The query that returns a row when table `name` exists. +fn has_table(name: &str) -> String { + format!("SELECT 1 FROM sqlite_schema WHERE type='table' AND name='{name}'") +} + /// The `commit` and `layer` position column as `history.sql` writes it; it /// appears once in each of the two tables. fn history_serial_column() -> &'static str { @@ -356,19 +374,21 @@ impl super::session::connection::Session { })?; Ok(()) } - /// Adds the three position columns T1.2a introduced to a Store written - /// before them: `commit.history_serial`, `layer.history_serial` (default 0) - /// and `store_policy.trace_salt` (32 random bytes). It is an additive - /// migration (REBUILD.md kind 2) under the same recorded-schema gate as - /// [`Self::add_binding_hints`]; because it runs before the GC tables are - /// added, the recorded source it matches is the live one with `GC` stripped. - pub(crate) fn add_gc_columns(&self) -> Result<(), BackendError> { + /// One additive migration (REBUILD.md kind 2) in one write transaction. + /// A Store that already has it (`present` returns a row), has no + /// `history_meta` row, recorded a source other than the accepted ones, or + /// whose live schema no longer matches what it recorded is left for + /// `validate` to judge. Otherwise `apply` runs and the Store records the + /// new source with its new definition. `sources` maps the current source + /// to `(accepted earlier sources, source to record)`. + fn migrate( + &self, + present: &str, + sources: impl FnOnce(String) -> (Vec, String), + apply: impl FnOnce(&Transaction<'_>) -> Result<(), BackendError>, + ) -> Result<(), BackendError> { self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM pragma_table_info('commit') WHERE name='history_serial'", - vec![], - )?; - if !present.is_empty() { + if !tx.query(present, vec![])?.is_empty() { return Ok(()); } let rows = tx.query( @@ -378,301 +398,143 @@ impl super::session::connection::Session { let Some(row) = rows.first() else { return Ok(()); }; - let current = without_position_columns(source((self.layout, self.acquisition))) - .replacen(GC, "", 1); - if row.get::(0)? != current + let (accepted, record) = sources(source((self.layout, self.acquisition))); + if !accepted.contains(&row.get::(0)?) || row.get::(1)? != definition(tx.connection, tx.work)? { return Ok(()); } - add_position_columns(tx)?; + apply(tx)?; let definition = definition(tx.connection, tx.work)?; tx.query( "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![ - Param::Text(source((self.layout, self.acquisition)).replacen(GC, "", 1)), - Param::Text(definition), - ], + vec![Param::Text(record), Param::Text(definition)], )?; Ok(()) }) } - /// Adds the binding hint table (#352 `49499abac`, blocks since #91) to a Store written - /// before it existed, then records the current schema: an additive - /// migration (REBUILD.md kind 2). Only a Store whose recorded schema is - /// exactly the earlier one, and whose live schema still matches what it - /// recorded, is migrated; any other Store is left for `validate` to judge. - /// Saved trees keep no hints, so a difference over them takes the full - /// walk; hints are recorded for what is saved from then on. + /// Adds the three position columns T1.2a introduced to a Store written + /// before them: `commit.history_serial`, `layer.history_serial` (default 0) + /// and `store_policy.trace_salt` (32 random bytes). Because it runs before + /// the GC tables are added, the sources it matches and records are the + /// current one with `GC` stripped. + pub(crate) fn add_gc_columns(&self) -> Result<(), BackendError> { + self.migrate( + "SELECT 1 FROM pragma_table_info('commit') WHERE name='history_serial'", + |current| { + let record = current.replacen(GC, "", 1); + (vec![without_position_columns(record.clone())], record) + }, + add_position_columns, + ) + } + /// Adds the binding hint table (#352 `49499abac`, blocks since #91) to a + /// Store written before it existed. Saved trees keep no hints, so a + /// difference over them takes the full walk; hints are recorded for what + /// is saved from then on. The source it records is the one before the + /// lineage, merge, digest and GC tables; later migrations add those. pub(crate) fn add_binding_hints(&self) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='binding_hint_block'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - // The schema this migration records is the one before the - // lineage and merge tables; later migrations add those. - let current = without_position_columns(source((self.layout, self.acquisition))) - .replacen(DIGESTS, "", 1) - .replacen(SCRATCH, "", 1) - .replacen(LINEAGE, "", 1) - .replacen(GC, "", 1); - let table = HISTORY - .find("CREATE TABLE binding_hint_block (") - .and_then(|start| Some((start, start + HISTORY[start..].find(";\n")? + 2))) - .map(|(start, end)| &HISTORY[start..end]) - .ok_or(BackendError::Integrity)?; - let earlier = current.replacen(&format!("{table}\n"), "", 1); - if row.get::(0)? != earlier - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.bootstrap(table)?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + &has_table("binding_hint_block"), + |current| { + let record = without_position_columns(current) + .replacen(DIGESTS, "", 1) + .replacen(SCRATCH, "", 1) + .replacen(LINEAGE, "", 1) + .replacen(GC, "", 1); + let table = format!("{}\n", binding_hint_table()); + (vec![record.replacen(&table, "", 1)], record) + }, + |tx| tx.bootstrap(binding_hint_table()), + ) } - /// Adds the lineage tables (#353) to a Store written before them, fills - /// them with `backfill` in the same write transaction and records the - /// current schema: an additive migration (REBUILD.md kind 2) under the - /// same conditions as [`Self::add_binding_hints`]. + /// Adds the lineage tables (#353) to a Store written before them and + /// fills them with `backfill` in the same write transaction. pub(crate) fn add_lineage( &self, backfill: impl FnOnce(&Transaction<'_>) -> Result<(), BackendError>, ) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='commit_lineage'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - let current = without_position_columns(source((self.layout, self.acquisition))) - .replacen(DIGESTS, "", 1) - .replacen(SCRATCH, "", 1) - .replacen(GC, "", 1); - if row.get::(0)? != current.replacen(LINEAGE, "", 1) - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.bootstrap(LINEAGE)?; - backfill(tx)?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + &has_table("commit_lineage"), + |current| { + let record = without_position_columns(current) + .replacen(DIGESTS, "", 1) + .replacen(SCRATCH, "", 1) + .replacen(GC, "", 1); + (vec![record.replacen(LINEAGE, "", 1)], record) + }, + |tx| { + tx.bootstrap(LINEAGE)?; + backfill(tx) + }, + ) } - /// Adds the merge build tables (#355) to a Store written before them and - /// records the current schema: an additive migration (REBUILD.md kind 2) - /// under the same conditions as [`Self::add_binding_hints`]. + /// Adds the merge build tables (#355) to a Store written before them. pub(crate) fn add_merge_scratch(&self) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='merge_record'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - let current = without_position_columns(source((self.layout, self.acquisition))) - .replacen(DIGESTS, "", 1) - .replacen(GC, "", 1); - if row.get::(0)? != current.replacen(SCRATCH, "", 1) - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.bootstrap(SCRATCH)?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + &has_table("merge_record"), + |current| { + let record = without_position_columns(current) + .replacen(DIGESTS, "", 1) + .replacen(GC, "", 1); + (vec![record.replacen(SCRATCH, "", 1)], record) + }, + |tx| tx.bootstrap(SCRATCH), + ) } - /// Adds the file digest table (#364) to a Store written before it and - /// records the current schema: an additive migration (REBUILD.md kind 2) - /// under the same conditions as [`Self::add_binding_hints`]. Saved roots - /// keep no digest; a Commit records them for the roots it builds. + /// Adds the file digest table (#364) to a Store written before it. Saved + /// roots keep no digest; a Commit records them for the roots it builds. pub(crate) fn add_file_digests(&self) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='file_digest'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - let current = without_position_columns(source((self.layout, self.acquisition))) - .replacen(GC, "", 1); - if row.get::(0)? != current.replacen(DIGESTS, "", 1) - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.bootstrap(DIGESTS)?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + &has_table("file_digest"), + |current| { + let record = without_position_columns(current).replacen(GC, "", 1); + (vec![record.replacen(DIGESTS, "", 1)], record) + }, + |tx| tx.bootstrap(DIGESTS), + ) } /// Adds the GC marker, position and admission tables to a Store written - /// before them, then records the current schema: an additive migration - /// under the same conditions as [`Self::add_binding_hints`]. + /// before them. pub(crate) fn add_gc(&self) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='gc_intent'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - let current = source((self.layout, self.acquisition)); - if row.get::(0)? != current.replacen(GC, "", 1) - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.bootstrap(GC)?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + &has_table("gc_intent"), + |current| (vec![current.replacen(GC, "", 1)], current), + |tx| tx.bootstrap(GC), + ) } /// Adds the `kept_node` history-closure table (T7.1) to a Store written - /// before it, then records the current schema: an additive migration - /// (REBUILD.md kind 2) under the same conditions as - /// [`Self::add_binding_hints`]. The table is empty until a mark writes it. + /// before it, with or without the T1.4 next-serial column. The table is + /// empty until a mark writes it. pub(crate) fn add_kept_node(&self) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM sqlite_schema WHERE type='table' AND name='kept_node'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - let current = source((self.layout, self.acquisition)); - let recorded = row.get::(0)?; - let without = current.replacen(kept_node_table(), "", 1); - let earlier = without.replacen(next_history_serial_column(), "", 1); - if (recorded != earlier && recorded != without) - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.bootstrap(kept_node_table())?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + &has_table("kept_node"), + |current| { + let without = current.replacen(kept_node_table(), "", 1); + let earlier = without.replacen(next_history_serial_column(), "", 1); + (vec![earlier, without], current) + }, + |tx| tx.bootstrap(kept_node_table()), + ) } /// Adds the position state T1.4 introduced (`history_meta.next_history_serial`, /// appended NOT NULL DEFAULT 1) and backfills the `commit`/`layer` serials - /// and `project_epoch` rows in the same write transaction: an additive - /// migration (REBUILD.md kind 2) under the same recorded-schema gate as - /// [`Self::add_gc_columns`]. `recorded == current` also matches a Store an - /// earlier step of this same open just brought forward, because those steps - /// record the current source before this column is appended. + /// and `project_epoch` rows in the same write transaction. A recorded + /// source equal to the current one is also accepted: it is a Store an + /// earlier step of this same open just brought forward, because those + /// steps record the current source before this column is appended. pub(crate) fn add_position_state(&self) -> Result<(), BackendError> { - self.run::<_, BackendError>(true, |tx| { - let present = tx.query( - "SELECT 1 FROM pragma_table_info('history_meta') WHERE name='next_history_serial'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let Some(row) = rows.first() else { - return Ok(()); - }; - let current = source((self.layout, self.acquisition)); - let before = current.replacen(next_history_serial_column(), "", 1); - let recorded = row.get::(0)?; - if (recorded != before && recorded != current) - || row.get::(1)? != definition(tx.connection, tx.work)? - { - return Ok(()); - } - tx.query(POSITION_COLUMNS[3].2, vec![])?; - backfill_position_state(tx)?; - let definition = definition(tx.connection, tx.work)?; - tx.query( - "UPDATE history_meta SET schema_source=?1,schema_definition=?2 WHERE id=1", - vec![Param::Text(current), Param::Text(definition)], - )?; - Ok(()) - }) + self.migrate( + "SELECT 1 FROM pragma_table_info('history_meta') WHERE name='next_history_serial'", + |current| { + let before = current.replacen(next_history_serial_column(), "", 1); + (vec![before, current.clone()], current) + }, + |tx| { + tx.query(POSITION_COLUMNS[3].2, vec![])?; + backfill_position_state(tx) + }, + ) } pub(crate) fn validate( &self, From 4f136fdab8f35332b5e780433830035c14dac383 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 23:00:54 +0800 Subject: [PATCH 2/6] refactor(persistence): one helper for the PostgreSQL additive migrations The four open-time migrations in backend/postgresql/schema.rs (kept_node, the GC tables, next_history_serial, the position columns) repeated one skeleton: probe without a transaction, probe again under the lane, read the recorded schema, decide whether it is an earlier revision without or with the acquisition tables, compare the live definition, apply, record. `migrate` now owns it; each migration states its probe, its earlier source and what it applies. One ordering difference, with no visible effect: `add_gc` compared the table set before the live definition and now compares it after. Both are reads that refuse with the same Integrity error before anything is written. Checks (macOS host, Rust 1.85.1, PostgreSQL 17 in a private container with amcheck): Clippy -D warnings on layerfs-persistence, fmt; persistence tests postgresql_position_columns, postgresql_open_check, postgresql_statement_schema, postgresql_gc_mark, postgresql_diagnostic and postgresql_compact pass (52 tests, none NOT_RUN). No test changed or deleted. Rebased onto 4c296e9297 (layerfs#734) on 2026-11-18. The test counts above are from before the rebase. This commit compiles with its tests on the new base; the suites were rerun at the branch tip, not at this commit. Production LOC: 188,837 -> 188,739 (delta -98) core 40,589 -> 40,589 (0) engine 90,496 -> 90,398 (-98) layerfs-persistence -98 control 51,230 -> 51,230 (0) api 6,522 -> 6,522 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py --json on parent 7ceff82a5f and the committed tree (recomputed after the rebase onto 4c296e9297) Co-Authored-By: Claude Fable 5.2 --- .../src/backend/postgresql/schema.rs | 274 +++++------------- 1 file changed, 80 insertions(+), 194 deletions(-) diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs index 9cab2523d..1678341b9 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/schema.rs @@ -228,18 +228,25 @@ fn pre_kept_node_source(acquisition: bool) -> String { source(acquisition).replacen(kept_node_table(), "", 1) } -/// Adds the `kept_node` history-closure table (T7.1) to a Store written -/// before it, then records the current schema: an additive migration under -/// the same conditions as [`add_gc`]. A schema that already has the table is -/// left alone; a recorded source or definition that is not an earlier -/// revision of the current one is refused, never migrated. The table is empty -/// until a mark writes it. -pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { - // A Store that already has the table is left alone: one statement, no - // transaction and no lane wait. +/// One additive migration of an existing Store schema. `probe` is the +/// `information_schema` relation and condition that finds what the migration +/// adds; a schema that already has it is left alone, first in one statement +/// with no transaction and no lane wait, then again under the lane, because +/// another process may have migrated first. `earlier` gives the source a Store +/// recorded before the migration, without and with the acquisition tables. +/// A recorded source that is not a revision of one of those +/// ([`is_earlier_source`]), or a recorded definition that is not the live one, +/// is refused with `Integrity`, never migrated. Otherwise `apply` runs and the +/// Store records the current source and its new definition. +fn migrate( + session: &Session, + probe: &str, + earlier: fn(bool) -> String, + apply: impl FnOnce(&Transaction<'_>, bool) -> Result<(), PersistenceError>, +) -> Result<(), PersistenceError> { let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { let sql = format!( - "SELECT 1 FROM information_schema.tables WHERE table_schema='{}' AND table_name='kept_node' AND table_type='BASE TABLE'", + "SELECT 1 FROM information_schema.{probe} AND table_schema='{}'", tx.schema ); Ok(!tx.query(&sql, vec![])?.is_empty()) @@ -248,12 +255,9 @@ pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { return Ok(()); } session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { - // Re-check under the lane: another process may have migrated first. - let present = tx.query( - "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='kept_node' AND table_type='BASE TABLE'", - vec![], - )?; - if !present.is_empty() { + let sql = + format!("SELECT 1 FROM information_schema.{probe} AND table_schema=current_schema()"); + if !tx.query(&sql, vec![])?.is_empty() { return Ok(()); } let rows = tx.query( @@ -263,18 +267,17 @@ pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { let [row] = rows.as_slice() else { return Err(BackendError::Integrity.into()); }; - let recorded_source: String = row.get(0)?; - let acquisition = if is_earlier_source(&recorded_source, pre_kept_node_source(false)) { - false - } else if is_earlier_source(&recorded_source, pre_kept_node_source(true)) { - true - } else { + let recorded: String = row.get(0)?; + let Some(acquisition) = [false, true] + .into_iter() + .find(|&acquisition| is_earlier_source(&recorded, earlier(acquisition))) + else { return Err(BackendError::Integrity.into()); }; if row.get::(1)? != definition(tx)? { return Err(BackendError::Integrity.into()); } - tx.batch(kept_node_table())?; + apply(tx, acquisition)?; let definition = definition(tx)?; tx.query( "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", @@ -284,130 +287,56 @@ pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { }) } +/// Adds the `kept_node` history-closure table (T7.1) to a Store written +/// before it. The table is empty until a mark writes it. +pub(crate) fn add_kept_node(session: &Session) -> Result<(), PersistenceError> { + migrate( + session, + "tables WHERE table_name='kept_node' AND table_type='BASE TABLE'", + pre_kept_node_source, + |tx, _| Ok(tx.batch(kept_node_table())?), + ) +} + /// Adds the GC marker, position and admission tables to a Store written -/// before them, then records the current schema: an additive migration under -/// the same conditions as the SQLite Store's `add_gc`. A schema that already -/// has `gc_intent` is left alone; a recorded source, table set or definition -/// that is not the pre-GC schema is refused, never migrated. +/// before them. A table set that is not the pre-GC one is refused too. pub(crate) fn add_gc(session: &Session) -> Result<(), PersistenceError> { - // A Store that already has the GC tables is left alone: one statement, - // no transaction and no lane wait. - let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { - let sql = format!( - "SELECT 1 FROM information_schema.tables WHERE table_schema='{}' AND table_name='gc_intent' AND table_type='BASE TABLE'", - tx.schema - ); - Ok(!tx.query(&sql, vec![])?.is_empty()) - })?; - if present { - return Ok(()); - } - session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { - // Re-check under the lane: another process may have migrated first. - let present = tx.query( - "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='gc_intent' AND table_type='BASE TABLE'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let [row] = rows.as_slice() else { - return Err(BackendError::Integrity.into()); - }; - let recorded_source: String = row.get(0)?; - let acquisition = if is_earlier_source(&recorded_source, pre_gc_source(false)) { - false - } else if is_earlier_source(&recorded_source, pre_gc_source(true)) { - true - } else { - return Err(BackendError::Integrity.into()); - }; - let tables = tx.query(TABLE_NAMES, vec![])?; - let actual: Vec = tables.iter().map(|r| r.get(0)).collect::>()?; - let mut expected = PRE_GC_TABLES.to_vec(); - if acquisition { - expected.extend(ACQUISITION_TABLES); - } - expected.sort_unstable(); - if !actual.iter().map(String::as_str).eq(expected) { - return Err(BackendError::Integrity.into()); - } - if row.get::(1)? != definition(tx)? { - return Err(BackendError::Integrity.into()); - } - tx.batch(GC)?; - let definition = definition(tx)?; - tx.query( - "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", - vec![Param::Text(source(acquisition)), Param::Text(definition)], - )?; - Ok(()) - }) + migrate( + session, + "tables WHERE table_name='gc_intent' AND table_type='BASE TABLE'", + pre_gc_source, + |tx, acquisition| { + let tables = tx.query(TABLE_NAMES, vec![])?; + let actual: Vec = tables.iter().map(|r| r.get(0)).collect::>()?; + let mut expected = PRE_GC_TABLES.to_vec(); + if acquisition { + expected.extend(ACQUISITION_TABLES); + } + expected.sort_unstable(); + if !actual.iter().map(String::as_str).eq(expected) { + return Err(BackendError::Integrity.into()); + } + Ok(tx.batch(GC)?) + }, + ) } -/// Adds the position column `history_meta.next_history_serial` (T1.4) to a -/// Store written before it, then records the current schema: an additive -/// migration under the same conditions as [`add_gc`]. A schema that already -/// has the column is left alone; a recorded source or definition that is not -/// an earlier revision of the current one is refused, never migrated. The +/// Adds the position column `history_meta.next_history_serial` (T1.4) and one +/// `project_epoch` row per project to a Store written before them. The /// `commit`/`layer` serial columns and their backfill follow in -/// [`add_position_columns`], the next step of the same open; only the column -/// and one `project_epoch` row per project are recorded here. +/// [`add_position_columns`], the next step of the same open. pub(crate) fn add_position_state(session: &Session) -> Result<(), PersistenceError> { - // A Store that already has the column is left alone: one statement, no - // transaction and no lane wait. - let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { - let sql = format!( - "SELECT 1 FROM information_schema.columns WHERE table_schema='{}' AND table_name='history_meta' AND column_name='next_history_serial'", - tx.schema - ); - Ok(!tx.query(&sql, vec![])?.is_empty()) - })?; - if present { - return Ok(()); - } - session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { - // Re-check under the lane: another process may have migrated first. - let present = tx.query( - "SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='history_meta' AND column_name='next_history_serial'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let [row] = rows.as_slice() else { - return Err(BackendError::Integrity.into()); - }; - let recorded_source: String = row.get(0)?; - let acquisition = if is_earlier_source(&recorded_source, source(false)) { - false - } else if is_earlier_source(&recorded_source, source(true)) { - true - } else { - return Err(BackendError::Integrity.into()); - }; - if row.get::(1)? != definition(tx)? { - return Err(BackendError::Integrity.into()); - } - tx.batch("ALTER TABLE history_meta ADD COLUMN next_history_serial bigint NOT NULL DEFAULT 1;")?; - tx.batch( - "INSERT INTO project_epoch(layer_stack_id, epoch) SELECT s.layer_stack_id, 0 FROM layer_stack s WHERE NOT EXISTS (SELECT 1 FROM project_epoch e WHERE e.layer_stack_id = s.layer_stack_id);", - )?; - let definition = definition(tx)?; - tx.query( - "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", - vec![Param::Text(source(acquisition)), Param::Text(definition)], - )?; - Ok(()) - }) + migrate( + session, + "columns WHERE table_name='history_meta' AND column_name='next_history_serial'", + source, + |tx, _| { + tx.batch("ALTER TABLE history_meta ADD COLUMN next_history_serial bigint NOT NULL DEFAULT 1;")?; + Ok(tx.batch( + "INSERT INTO project_epoch(layer_stack_id, epoch) SELECT s.layer_stack_id, 0 FROM layer_stack s WHERE NOT EXISTS (SELECT 1 FROM project_epoch e WHERE e.layer_stack_id = s.layer_stack_id);", + )?) + }, + ) } /// The ALTERs an old Store needs for the position columns, the one random @@ -428,60 +357,17 @@ const POSITION_COLUMNS: &str = concat!( /// Adds the position columns T1.2c introduced to a Store written before them: /// `commit.history_serial` and `layer.history_serial` (default 0) and /// `store_policy.trace_salt` (32 random bytes), then backfills the serials -/// with T1.4's semantics in the same transaction. An additive migration under -/// the same conditions as [`add_gc`]: a schema that already has -/// `commit.history_serial` is left alone, and a recorded source or definition -/// that is not an earlier revision of the current one is refused, never -/// migrated. +/// with T1.4's semantics in the same transaction. pub(crate) fn add_position_columns(session: &Session) -> Result<(), PersistenceError> { - // A Store that already has the columns is left alone: one statement, no - // transaction and no lane wait. - let present = session.run_as::<_, BackendError>(Mode::Statement, |tx| { - let sql = format!( - "SELECT 1 FROM information_schema.columns WHERE table_schema='{}' AND table_name='commit' AND column_name='history_serial'", - tx.schema - ); - Ok(!tx.query(&sql, vec![])?.is_empty()) - })?; - if present { - return Ok(()); - } - session.run_as(Mode::Write, |tx| -> Result<(), PersistenceError> { - // Re-check under the lane: another process may have migrated first. - let present = tx.query( - "SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='commit' AND column_name='history_serial'", - vec![], - )?; - if !present.is_empty() { - return Ok(()); - } - let rows = tx.query( - "SELECT schema_source,schema_definition FROM history_meta WHERE id=1", - vec![], - )?; - let [row] = rows.as_slice() else { - return Err(BackendError::Integrity.into()); - }; - let recorded_source: String = row.get(0)?; - let acquisition = if is_earlier_source(&recorded_source, source(false)) { - false - } else if is_earlier_source(&recorded_source, source(true)) { - true - } else { - return Err(BackendError::Integrity.into()); - }; - if row.get::(1)? != definition(tx)? { - return Err(BackendError::Integrity.into()); - } - tx.batch(POSITION_COLUMNS)?; - backfill_position_serials(tx)?; - let definition = definition(tx)?; - tx.query( - "UPDATE history_meta SET schema_source=$1,schema_definition=$2 WHERE id=1", - vec![Param::Text(source(acquisition)), Param::Text(definition)], - )?; - Ok(()) - }) + migrate( + session, + "columns WHERE table_name='commit' AND column_name='history_serial'", + source, + |tx, _| { + tx.batch(POSITION_COLUMNS)?; + Ok(backfill_position_serials(tx)?) + }, + ) } /// The T1.4 serial backfill: Commits take the next Store serials in From 52a0697cdd454d9f6184ce9c215a19249ff03232 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 23:06:49 +0800 Subject: [PATCH 3/6] refactor(compact): one phase clock and one report type for both backends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `RunMetrics`, the phase clock and §11 values of one `store.compact` run, existed twice: backend/sqlite/compact/receipt.rs and backend/postgresql/compact/op.rs differed only in `mark_bytes`. One definition now lives in compact_metrics.rs beside the recorder it feeds; `mark_bytes` is `None` on SQLite and `Some` on PostgreSQL, as each reported before. - `CompactReport`, `CompactGeneration` and the one-implementor trait `CompactReportSource` restated `CompactReceipt` field for field. Both backends already return `CompactReceipt`, so the control serializer `layerfs_store::report::compact_report` now takes it directly. The wire shape is unchanged. Tests: sqlite_compact.rs `the_report_projects_every_section_11_field` is now `the_receipt_carries_every_section_11_field`; its comparisons of the removed projection against the receipt (ten field equalities and the trait/From equality) are gone with the type, and its assertions on the values stay. compact_report_status.rs passes the receipt to the serializer. No test file deleted. Checks (macOS host, Rust 1.85.1, PostgreSQL 17 in a private container): cargo check --workspace --all-targets; Clippy -D warnings on layerfs-persistence and layerfs-store; fmt; product boundary guard; persistence sqlite_compact (33), sqlite_compact_crash (15), compact_metrics (5), postgresql_compact (30), postgresql_status (8), postgresql_compact_units (8); layerfs-store whole package (80). sqlite_compact_capacity ran 0 tests on this host. Rebased onto 4c296e9297 (layerfs#734) on 2026-11-18. The test counts above are from before the rebase. This commit compiles with its tests on the new base; the suites were rerun at the branch tip, not at this commit. Production LOC: 188,739 -> 188,574 (delta -165) core 40,589 -> 40,589 (0) engine 90,398 -> 90,234 (-164) layerfs-persistence -164 control 51,230 -> 51,229 (-1) layerfs-store -1 api 6,522 -> 6,522 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py --json on parent 4f136fdab8 and the committed tree (recomputed after the rebase onto 4c296e9297) Co-Authored-By: Claude Fable 5.2 --- control/crates/layerfs-store/src/compact.rs | 9 +- control/crates/layerfs-store/src/report.rs | 12 +- .../tests/compact_report_status.rs | 11 +- .../src/backend/postgresql/compact/cleanup.rs | 2 +- .../src/backend/postgresql/compact/op.rs | 119 ------------------ .../src/backend/postgresql/compact/run.rs | 6 +- .../src/backend/sqlite/compact/install.rs | 2 +- .../src/backend/sqlite/compact/receipt.rs | 114 ----------------- .../src/backend/sqlite/compact/run.rs | 4 +- .../src/compact_metrics.rs | 114 ++++++++++++++++- engine/crates/layerfs-persistence/src/lib.rs | 5 +- .../crates/layerfs-persistence/src/report.rs | 103 +-------------- .../tests/sqlite_compact.rs | 28 ++--- 13 files changed, 147 insertions(+), 382 deletions(-) diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs index 7f2475170..d512e863e 100644 --- a/control/crates/layerfs-store/src/compact.rs +++ b/control/crates/layerfs-store/src/compact.rs @@ -28,8 +28,7 @@ use layerfs_home::outcome::{failed, refused, Outcome}; use layerfs_persistence::compact_metrics::{self, CompactRefusal}; use layerfs_persistence::PostgresHandles; use layerfs_persistence::{ - CompactFailpoint, CompactReport, CompactRequest, PersistenceConfig, SqliteHandles, - SqlitePersistenceProfile, + CompactFailpoint, CompactRequest, PersistenceConfig, SqliteHandles, SqlitePersistenceProfile, }; use layerfs_store_service::history::{admission, lookup}; use std::path::Path; @@ -148,9 +147,7 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { .map_err(|error| { lookup::history_in(error, GcScopeKind::Store, store, None, Some(&op_id)) })?; - // One serializer for the shared §11 report: the SQLite receipt projects - // into `CompactReport`, and T9's PostgreSQL receipt will do the same. - Ok(crate::report::compact_report(&CompactReport::from(receipt))) + Ok(crate::report::compact_report(&receipt)) } /// The §8 PostgreSQL path: dial the Store's recorded URL, open its schema as @@ -201,7 +198,7 @@ fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome .map_err(|error| { lookup::history_in(error, GcScopeKind::Store, store, None, Some(&op_id)) })?; - Ok(crate::report::compact_report(&CompactReport::from(receipt))) + Ok(crate::report::compact_report(&receipt)) } /// An open refusal before the run: `Busy` is definite, everything else a diff --git a/control/crates/layerfs-store/src/report.rs b/control/crates/layerfs-store/src/report.rs index 4983cb82a..3acedde6b 100644 --- a/control/crates/layerfs-store/src/report.rs +++ b/control/crates/layerfs-store/src/report.rs @@ -1,17 +1,17 @@ //! The wire rendering of the §11 compaction report and the `store.status` //! compaction projection (T10). //! -//! One serializer renders the shared [`CompactReport`], so the SQLite path and -//! T9's PostgreSQL path cannot disagree on a key: every key is snake_case (§10; +//! One serializer renders the [`CompactReceipt`] both backends return, so the +//! SQLite and PostgreSQL paths cannot disagree on a key: every key is snake_case (§10; //! §11 names are logical), a zero field is named by its wire key, and //! `residue.reason` appears only when something remains. use layerfs_bridge::control::SdkValue; -use layerfs_persistence::{CompactReport, CompactStatus}; +use layerfs_persistence::{CompactReceipt, CompactStatus}; /// The §11 report as `store.compact` returns it: /// `{kind: "Compacted", path, generation {before, after}, ...}`. #[must_use] -pub fn compact_report(report: &CompactReport) -> SdkValue { +pub fn compact_report(report: &CompactReceipt) -> SdkValue { let int = |value: u64| SdkValue::Int(i64::try_from(value).unwrap_or(i64::MAX)); let references: Vec = report .kept_waste @@ -66,8 +66,8 @@ pub fn compact_report(report: &CompactReport) -> SdkValue { ( "generation", SdkValue::map([ - ("before", int(report.generation.before)), - ("after", int(report.generation.after)), + ("before", int(report.generation_before)), + ("after", int(report.generation_after)), ]), ), ("dead_bytes_bound", int(report.dead_bytes_bound)), diff --git a/control/crates/layerfs-store/tests/compact_report_status.rs b/control/crates/layerfs-store/tests/compact_report_status.rs index 73d97b409..409d94cc4 100644 --- a/control/crates/layerfs-store/tests/compact_report_status.rs +++ b/control/crates/layerfs-store/tests/compact_report_status.rs @@ -1,5 +1,5 @@ //! T10: the §11 report wire shape and the `store.status` compaction op -//! projection. The report renders snake_case from the shared `CompactReport`; +//! projection. The report renders snake_case from the `CompactReceipt`; //! status reads only the `.compact-op` sidecar and never opens the //! Store database. use layerfs_bridge::control::SdkValue; @@ -7,8 +7,7 @@ use layerfs_content::ObjectId; use layerfs_home::home::{paths, private_dir, write_entity}; use layerfs_home::names::StoreName; use layerfs_persistence::{ - CompactKeptWaste, CompactPath, CompactReceipt, CompactReport, CompactResidue, - CompactZeroReason, KeptWaste, + CompactKeptWaste, CompactPath, CompactReceipt, CompactResidue, CompactZeroReason, KeptWaste, }; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -129,7 +128,7 @@ fn a_capped_reference_list_carries_its_cap() { capped.kept_waste.delta_base_references = vec![[9; 32]; layerfs_persistence::DELTA_BASE_REFERENCES_CAP]; capped.kept_waste.delta_base_references_capped = true; - let wire = layerfs_store::report::compact_report(&CompactReport::from(&capped)); + let wire = layerfs_store::report::compact_report(&capped); let bases = wire.get("kept_waste").unwrap().get("delta_bases").unwrap(); let references = bases.get("references").unwrap(); assert_eq!( @@ -161,7 +160,7 @@ fn a_mark_over_the_delta_base_cap_renders_value_and_cap() { let waste = KeptWaste::from_parts(0, 12, references, 0); let mut receipt = receipt(); receipt.kept_waste = CompactKeptWaste::from_waste(&waste); - let wire = layerfs_store::report::compact_report(&CompactReport::from(&receipt)); + let wire = layerfs_store::report::compact_report(&receipt); let bases = wire.get("kept_waste").unwrap().get("delta_bases").unwrap(); let references = bases.get("references").unwrap(); assert_eq!( @@ -180,7 +179,7 @@ fn a_mark_over_the_delta_base_cap_renders_value_and_cap() { #[test] fn the_report_wire_is_the_snake_case_contract_shape() { - let wire = layerfs_store::report::compact_report(&CompactReport::from(&receipt())); + let wire = layerfs_store::report::compact_report(&receipt()); assert_eq!(text(wire.get("kind").unwrap()), "Compacted"); assert_eq!(text(wire.get("path").unwrap()), "rebuild"); let generation = wire.get("generation").unwrap(); diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/cleanup.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/cleanup.rs index a7929e599..da1a9b3bb 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/cleanup.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/cleanup.rs @@ -9,7 +9,6 @@ use super::op::scalar; use super::op::scalar_session; use super::op::set_phase; use super::op::stage; -use super::op::RunMetrics; use super::report::report; use super::types::DeadTotals; use super::types::OpRow; @@ -19,6 +18,7 @@ use crate::backend::sqlite::compact::CompactPath; use crate::backend::sqlite::compact::CompactReceipt; use crate::backend::sqlite::compact::CompactResidue; use crate::compact_metrics::CompactPhase; +use crate::compact_metrics::RunMetrics; use crate::HistoryProvider; use crate::PostgresHandles; use layerfs_history::error::HistoryError; diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/op.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/op.rs index e12dc96ec..49ccb839a 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/op.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/op.rs @@ -11,134 +11,15 @@ use crate::backend::postgresql::session::transaction::Transaction; use crate::backend::records::Param; use crate::backend::sqlite::compact::CompactFailpoint; use crate::backend::sqlite::compact::CompactKeptWaste; -use crate::backend::sqlite::compact::CompactPath; -use crate::backend::sqlite::compact::CompactReceipt; use crate::backend::sqlite::compact::CompactRequest; -use crate::compact_metrics; -use crate::compact_metrics::CompactOutcome; -use crate::compact_metrics::CompactPhase; -use crate::compact_metrics::CompactRefusal; -use crate::compact_metrics::CompactRunObservation; use crate::history::GcTuple; use crate::MarkResult; use layerfs_history::error::HistoryError; use layerfs_history::error::HistoryResult; use std::sync::Arc; -use std::time::Instant; use std::time::SystemTime; use std::time::UNIX_EPOCH; -/// The phase clock and §11 values one measured PostgreSQL compaction observes. -/// -/// Phase M times the intent, the mark and the wholly-dead-pack decision, Phase -/// A the scratch copy, Phase B the swap, and Phase C the cleanup — also when a -/// later call resumes an installed `G + 1` leftover. The `pg_` phase values -/// keep these distinct from the SQLite §7 phases in the process totals. -pub(crate) struct RunMetrics { - pub(crate) started: Instant, - pub(crate) phase: Option<(CompactPhase, Instant)>, - pub(crate) phase_nanos: Vec<(CompactPhase, u64)>, - pub(crate) generation_before: u64, - pub(crate) generation_after: u64, - pub(crate) dead_bytes_bound: u64, - pub(crate) dead_bytes_refined: u64, - pub(crate) kept_waste: CompactKeptWaste, - pub(crate) residue_bytes: u64, - /// §11 `markBytes`: the Phase M mark's distinct live-pack body bytes. - pub(crate) mark_bytes: u64, - pub(crate) leftover_settled: bool, -} - -impl RunMetrics { - pub(crate) fn start() -> Self { - Self { - started: Instant::now(), - phase: None, - phase_nanos: Vec::new(), - generation_before: 0, - generation_after: 0, - dead_bytes_bound: 0, - dead_bytes_refined: 0, - kept_waste: CompactKeptWaste::default(), - residue_bytes: 0, - mark_bytes: 0, - leftover_settled: false, - } - } - - /// Ends the current phase and starts `phase`; a phase is inclusive wall - /// time, never an exclusive CPU decomposition. - pub(crate) fn begin_phase(&mut self, phase: CompactPhase) { - self.end_phase(); - self.phase = Some((phase, Instant::now())); - } - - pub(crate) fn end_phase(&mut self) { - if let Some((phase, started)) = self.phase.take() { - self.phase_nanos.push(( - phase, - u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), - )); - } - } - - /// Records this run once, whatever its result. The outcome and refusal - /// come from the typed error, so a `Busy`/`Unresolved` refusal is counted - /// without changing the error the caller receives. - pub(crate) fn finish(mut self, result: &HistoryResult) { - self.end_phase(); - let total_nanos = u64::try_from(self.started.elapsed().as_nanos()).unwrap_or(u64::MAX); - let (outcome, refusal) = match result { - Ok(receipt) => { - // A resumed `G + 1` round reports the original run's - // generation, so the receipt is the authority here. - self.generation_before = receipt.generation_before; - self.generation_after = receipt.generation_after; - self.dead_bytes_bound = receipt.dead_bytes_bound; - self.dead_bytes_refined = receipt.dead_bytes_refined; - self.kept_waste = receipt.kept_waste.clone(); - self.residue_bytes = receipt.residue.bytes; - let outcome = match receipt.path { - CompactPath::Rebuild => CompactOutcome::Rebuild, - CompactPath::None => CompactOutcome::None, - }; - (outcome, None) - } - Err(error) => { - if !error.unknown() { - self.generation_after = self.generation_before; - } - ( - if error.unknown() { - CompactOutcome::Uncertain - } else { - CompactOutcome::Failed - }, - match error { - HistoryError::Busy | HistoryError::BusyOp(_) => Some(CompactRefusal::Busy), - HistoryError::Unresolved(_) => Some(CompactRefusal::Unresolved), - _ => None, - }, - ) - } - }; - compact_metrics::record_run(&CompactRunObservation { - outcome, - generation_before: self.generation_before, - generation_after: self.generation_after, - dead_bytes_bound: self.dead_bytes_bound, - dead_bytes_refined: self.dead_bytes_refined, - kept_waste: self.kept_waste, - residue_bytes: self.residue_bytes, - mark_bytes: Some(self.mark_bytes), - phase_nanos: self.phase_nanos, - total_nanos, - leftover_settled: self.leftover_settled, - refusal, - }); - } -} - pub(crate) fn now_ms() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs index 999a880f3..44ca400df 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs @@ -12,7 +12,6 @@ use super::op::read_generation; use super::op::scratch_name; use super::op::stage; use super::op::stop; -use super::op::RunMetrics; use super::report::report; use super::swap::swap; use super::sync::effective_sync_rank; @@ -28,6 +27,7 @@ use crate::backend::sqlite::compact::CompactReceipt; use crate::backend::sqlite::compact::CompactRequest; use crate::backend::sqlite::compact::CompactResidue; use crate::compact_metrics::CompactPhase; +use crate::compact_metrics::RunMetrics; use crate::PostgresHandles; use layerfs_history::error::HistoryError; use layerfs_history::error::HistoryResult; @@ -48,7 +48,7 @@ pub(crate) fn round( metrics.begin_phase(CompactPhase::PostgresMark); let (mark, tuple) = handles.history.mark_gc()?; // §11 `markBytes`: the distinct live-pack body bytes this mark read. - metrics.mark_bytes = mark.mark_bytes(); + metrics.mark_bytes = Some(mark.mark_bytes()); load_visited(session, schema, &mark)?; // §3 decides over the mark's own snapshot, never a later Store total: a // publication after the mark raises `pack` bytes, and reading them here @@ -218,7 +218,7 @@ impl PostgresHandles { /// deletes only the persisted dead set. The caller keeps its session; the /// Store is usable at `G + 1` from the swap on. pub fn compact(&self, request: &CompactRequest) -> HistoryResult { - let mut metrics = RunMetrics::start(); + let mut metrics = RunMetrics::start(Some(0)); let result = self.compact_measured(request, &mut metrics); metrics.finish(&result); result diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/install.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/install.rs index 7722d718f..33973c5ac 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/install.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/install.rs @@ -10,7 +10,6 @@ use super::op::remove_op_record; use super::op::stop_at; use super::op::RunOp; use super::receipt::receipt; -use super::receipt::RunMetrics; use super::types::db_error; use super::types::io_error; use super::types::CompactFailpoint; @@ -24,6 +23,7 @@ use super::verify::file_tuple; use super::verify::open_immutable; use crate::backend::sqlite::lane::writer_admission::Writer; use crate::backend::sqlite::lifecycle::seal::SealedFacts; +use crate::compact_metrics::RunMetrics; use layerfs_history::error::HistoryError; use layerfs_history::error::HistoryResult; use layerfs_history::error::OpConflict; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/receipt.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/receipt.rs index fb50e9c43..35d6c058f 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/receipt.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/receipt.rs @@ -5,14 +5,6 @@ use super::types::CompactPath; use super::types::CompactReceipt; use super::types::CompactResidue; use super::types::CompactZeroReason; -use crate::compact_metrics; -use crate::compact_metrics::CompactOutcome; -use crate::compact_metrics::CompactPhase; -use crate::compact_metrics::CompactRefusal; -use crate::compact_metrics::CompactRunObservation; -use layerfs_history::error::HistoryError; -use layerfs_history::error::HistoryResult; -use std::time::Instant; /// §11: a zero field always carries one word of the closed vocabulary. #[allow(clippy::too_many_arguments)] @@ -111,109 +103,3 @@ pub fn receipt( zero_reasons, } } - -/// The phase clock and §11 values one measured `compact` run observes. -pub(crate) struct RunMetrics { - pub(crate) started: Instant, - pub(crate) phase: Option<(CompactPhase, Instant)>, - pub(crate) phase_nanos: Vec<(CompactPhase, u64)>, - pub(crate) generation_before: u64, - pub(crate) generation_after: u64, - pub(crate) dead_bytes_bound: u64, - pub(crate) dead_bytes_refined: u64, - pub(crate) kept_waste: CompactKeptWaste, - pub(crate) residue_bytes: u64, - pub(crate) leftover_settled: bool, -} - -impl RunMetrics { - pub(crate) fn start() -> Self { - Self { - started: Instant::now(), - phase: None, - phase_nanos: Vec::new(), - generation_before: 0, - generation_after: 0, - dead_bytes_bound: 0, - dead_bytes_refined: 0, - kept_waste: CompactKeptWaste::default(), - residue_bytes: 0, - leftover_settled: false, - } - } - - /// Ends the current phase and starts `phase`; a phase is inclusive wall - /// time, never an exclusive CPU decomposition. - pub(crate) fn begin_phase(&mut self, phase: CompactPhase) { - self.end_phase(); - self.phase = Some((phase, Instant::now())); - } - - pub(crate) fn end_phase(&mut self) { - if let Some((phase, started)) = self.phase.take() { - self.phase_nanos.push(( - phase, - u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), - )); - } - } - - /// Records this run once, whatever its result. The outcome and refusal - /// come from the typed error, so a `Busy`/`Unresolved` refusal is counted - /// without changing the error the caller receives. - pub(crate) fn finish(mut self, result: &HistoryResult) { - self.end_phase(); - let total_nanos = u64::try_from(self.started.elapsed().as_nanos()).unwrap_or(u64::MAX); - let (outcome, refusal) = match result { - Ok(receipt) => { - // A resumed `G + 1` round reports the original run's - // generation, so the receipt is the authority here. - self.generation_before = receipt.generation_before; - self.generation_after = receipt.generation_after; - self.dead_bytes_bound = receipt.dead_bytes_bound; - self.dead_bytes_refined = receipt.dead_bytes_refined; - self.kept_waste = receipt.kept_waste.clone(); - self.residue_bytes = receipt.residue.bytes; - let outcome = match receipt.path { - CompactPath::Rebuild => CompactOutcome::Rebuild, - CompactPath::None => CompactOutcome::None, - }; - (outcome, None) - } - Err(error) => { - if !error.unknown() { - self.generation_after = self.generation_before; - } - ( - if error.unknown() { - CompactOutcome::Uncertain - } else { - CompactOutcome::Failed - }, - match error { - HistoryError::Busy | HistoryError::BusyOp(_) => Some(CompactRefusal::Busy), - HistoryError::Unresolved(_) => Some(CompactRefusal::Unresolved), - _ => None, - }, - ) - } - }; - compact_metrics::record_run(&CompactRunObservation { - outcome, - generation_before: self.generation_before, - generation_after: self.generation_after, - dead_bytes_bound: self.dead_bytes_bound, - dead_bytes_refined: self.dead_bytes_refined, - kept_waste: self.kept_waste, - residue_bytes: self.residue_bytes, - // §11 `markBytes` is the PostgreSQL Phase M transfer metric; the - // SQLite mark reads only metadata bodies and sweeps the payload in - // the build copy, so SQLite reports no value. - mark_bytes: None, - phase_nanos: self.phase_nanos, - total_nanos, - leftover_settled: self.leftover_settled, - refusal, - }); - } -} diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs index b20882e33..7d8bc1d50 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs @@ -22,7 +22,6 @@ use super::op::visited_path; use super::op::InstalledReport; use super::op::RunOp; use super::receipt::receipt; -use super::receipt::RunMetrics; use super::types::seal_error; use super::types::CompactFailpoint; use super::types::CompactKeptWaste; @@ -35,6 +34,7 @@ use super::verify::sealed_state; use super::verify::verify; use super::verify::VerifyInput; use crate::compact_metrics::CompactPhase; +use crate::compact_metrics::RunMetrics; use crate::MarkResult; use crate::SqliteHandles; use layerfs_history::error::HistoryError; @@ -60,7 +60,7 @@ impl SqliteHandles { /// op record are kept for the next `store.compact` to judge (§9). pub fn compact(self, request: &CompactRequest) -> HistoryResult { let store = self.storage.session.path.clone(); - let mut metrics = RunMetrics::start(); + let mut metrics = RunMetrics::start(None); let result = self.compact_measured(request, &mut metrics); // §9: the run is replying, so its own record (when it leaves one) is // no longer in flight. Mark it before the reply so the next diff --git a/engine/crates/layerfs-persistence/src/compact_metrics.rs b/engine/crates/layerfs-persistence/src/compact_metrics.rs index 014fe6371..4d40f3769 100644 --- a/engine/crates/layerfs-persistence/src/compact_metrics.rs +++ b/engine/crates/layerfs-persistence/src/compact_metrics.rs @@ -13,10 +13,12 @@ //! [`CompactMetricsCapture`], so concurrent work on other threads does not //! pollute the observation. -use crate::backend::sqlite::compact::CompactKeptWaste; +use crate::backend::sqlite::compact::{CompactKeptWaste, CompactPath, CompactReceipt}; +use layerfs_history::error::{HistoryError, HistoryResult}; use std::cell::RefCell; use std::collections::BTreeMap; use std::sync::{Mutex, OnceLock}; +use std::time::Instant; /// The outcome of one Store compaction call. #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] @@ -590,3 +592,113 @@ impl Drop for CompactMetricsCapture { } } } + +/// The phase clock and §11 values one measured `store.compact` run observes, +/// on either backend. The SQLite §7 phases and the PostgreSQL `pg_` phases are +/// distinct [`CompactPhase`] values, so one clock serves both. +pub(crate) struct RunMetrics { + pub(crate) started: Instant, + pub(crate) phase: Option<(CompactPhase, Instant)>, + pub(crate) phase_nanos: Vec<(CompactPhase, u64)>, + pub(crate) generation_before: u64, + pub(crate) generation_after: u64, + pub(crate) dead_bytes_bound: u64, + pub(crate) dead_bytes_refined: u64, + pub(crate) kept_waste: CompactKeptWaste, + pub(crate) residue_bytes: u64, + pub(crate) leftover_settled: bool, + /// §11 `markBytes`, the PostgreSQL Phase M transfer metric. `None` on + /// SQLite, whose mark reads only metadata bodies and sweeps the payload + /// in the build copy. + pub(crate) mark_bytes: Option, +} + +impl RunMetrics { + pub(crate) fn start(mark_bytes: Option) -> Self { + Self { + started: Instant::now(), + phase: None, + phase_nanos: Vec::new(), + generation_before: 0, + generation_after: 0, + dead_bytes_bound: 0, + dead_bytes_refined: 0, + kept_waste: CompactKeptWaste::default(), + residue_bytes: 0, + leftover_settled: false, + mark_bytes, + } + } + + /// Ends the current phase and starts `phase`; a phase is inclusive wall + /// time, never an exclusive CPU decomposition. + pub(crate) fn begin_phase(&mut self, phase: CompactPhase) { + self.end_phase(); + self.phase = Some((phase, Instant::now())); + } + + pub(crate) fn end_phase(&mut self) { + if let Some((phase, started)) = self.phase.take() { + self.phase_nanos.push(( + phase, + u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX), + )); + } + } + + /// Records this run once, whatever its result. The outcome and refusal + /// come from the typed error, so a `Busy`/`Unresolved` refusal is counted + /// without changing the error the caller receives. + pub(crate) fn finish(mut self, result: &HistoryResult) { + self.end_phase(); + let total_nanos = u64::try_from(self.started.elapsed().as_nanos()).unwrap_or(u64::MAX); + let (outcome, refusal) = match result { + Ok(receipt) => { + // A resumed `G + 1` round reports the original run's + // generation, so the receipt is the authority here. + self.generation_before = receipt.generation_before; + self.generation_after = receipt.generation_after; + self.dead_bytes_bound = receipt.dead_bytes_bound; + self.dead_bytes_refined = receipt.dead_bytes_refined; + self.kept_waste = receipt.kept_waste.clone(); + self.residue_bytes = receipt.residue.bytes; + let outcome = match receipt.path { + CompactPath::Rebuild => CompactOutcome::Rebuild, + CompactPath::None => CompactOutcome::None, + }; + (outcome, None) + } + Err(error) => { + if !error.unknown() { + self.generation_after = self.generation_before; + } + ( + if error.unknown() { + CompactOutcome::Uncertain + } else { + CompactOutcome::Failed + }, + match error { + HistoryError::Busy | HistoryError::BusyOp(_) => Some(CompactRefusal::Busy), + HistoryError::Unresolved(_) => Some(CompactRefusal::Unresolved), + _ => None, + }, + ) + } + }; + record_run(&CompactRunObservation { + outcome, + generation_before: self.generation_before, + generation_after: self.generation_after, + dead_bytes_bound: self.dead_bytes_bound, + dead_bytes_refined: self.dead_bytes_refined, + kept_waste: self.kept_waste, + residue_bytes: self.residue_bytes, + mark_bytes: self.mark_bytes, + phase_nanos: self.phase_nanos, + total_nanos, + leftover_settled: self.leftover_settled, + refusal, + }); + } +} diff --git a/engine/crates/layerfs-persistence/src/lib.rs b/engine/crates/layerfs-persistence/src/lib.rs index 5cfa4930a..033453adb 100644 --- a/engine/crates/layerfs-persistence/src/lib.rs +++ b/engine/crates/layerfs-persistence/src/lib.rs @@ -43,10 +43,7 @@ pub use config::{ pub use history::{ HistoryProvider, KeptHistory, KeptNode, KeptNodeId, KeptObjects, KeptWaste, MarkResult, }; -pub use report::{ - CompactGeneration, CompactReport, CompactReportSource, CompactStatus, POSTGRES_COMPACT_PHASES, - SQLITE_COMPACT_PHASES, -}; +pub use report::{CompactStatus, POSTGRES_COMPACT_PHASES, SQLITE_COMPACT_PHASES}; pub(crate) use storage::provider as storage_provider; pub(crate) use storage::publication; pub use storage_provider::StorageProvider; diff --git a/engine/crates/layerfs-persistence/src/report.rs b/engine/crates/layerfs-persistence/src/report.rs index a4d8c9a03..73a206b1e 100644 --- a/engine/crates/layerfs-persistence/src/report.rs +++ b/engine/crates/layerfs-persistence/src/report.rs @@ -1,13 +1,8 @@ -//! The §11 compaction report and the `store.status` compaction projection. +//! The compaction phase lists and the `store.status` compaction projection. //! -//! One shape serves every backend. A SQLite [`CompactReceipt`] projects into -//! [`CompactReport`] through [`CompactReportSource`], and T9's PostgreSQL -//! rebuild implements that trait for its own receipt, so both paths share one -//! wire shape and one serializer. §11 names are logical; the rendered keys are -//! snake_case (§10). -use crate::backend::sqlite::compact::{ - CompactKeptWaste, CompactPath, CompactReceipt, CompactResidue, CompactZeroReason, -}; +//! The §11 report itself is [`crate::CompactReceipt`]: both backends return +//! that one type and the control layer renders it with one serializer. §11 +//! names are logical; the rendered keys are snake_case (§10). /// The phases a SQLite compaction passes, in order (§11): a run that crashes /// leaves its record at the phase it reached. @@ -18,96 +13,6 @@ pub const SQLITE_COMPACT_PHASES: [&str; 5] = ["mark", "build", "verify", "instal /// PostgreSQL `gc_op` row records, distinct from the SQLite §7 phases. pub const POSTGRES_COMPACT_PHASES: [&str; 4] = ["mark", "build", "swap", "cleanup"]; -/// The generation a compaction started at and, when it installed a rebuild, -/// the one it left (`none` keeps both equal). -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct CompactGeneration { - /// The generation the mark ran at. - pub before: u64, - /// The installed generation; equal to `before` on the `none` path. - pub after: u64, -} - -/// The §11 report `store.compact` returns, backend-neutral. -/// -/// SQLite fills it from [`CompactReceipt`]; T9's PostgreSQL rebuild fills the -/// same fields from its own receipt. A backend-only field is deliberately not -/// defined here: PostgreSQL reports its own zeros through the same keys. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CompactReport { - /// `none` or `rebuild`. - pub path: CompactPath, - /// The generation before and after. - pub generation: CompactGeneration, - /// Bytes in wholly dead packs after the metadata walk. - pub dead_bytes_bound: u64, - /// The same bytes after the payload sweep (§7 step 4). No second sweep - /// runs, so this equals `dead_bytes_bound` on every path (#42 A10). - pub dead_bytes_refined: u64, - /// The canonical bytes of the collected objects, deduplicated by object - /// id: not the history rows' `logical` sum (#42 A8). - pub logical_discarded: u64, - /// The source Store's pre-mark free pages × page size on a rebuild; the - /// live Store's count on `none` (#42 A8). - pub backend_reusable: u64, - /// Old file size minus the new one. - pub physical_shrink: u64, - /// Bytes the mark keeps that no retained root needs. - pub kept_waste: CompactKeptWaste, - /// Peak temporary bytes the run needed. - pub peak_temp_bytes: u64, - /// The artifact set left after settle; empty on success. - pub residue: CompactResidue, - /// §13: the report is an estimate, not a measurement. - pub estimate: bool, - /// One reason per zero report field, from the closed vocabulary. - pub zero_reasons: Vec, -} - -/// A backend receipt that projects to the one §11 [`CompactReport`]. -/// -/// SQLite's [`CompactReceipt`] implements it here. T9's PostgreSQL rebuild -/// implements it for the receipt of its Phase M/A/B/C run, which keeps the -/// PostgreSQL path on this shared shape without a PostgreSQL-only field. -pub trait CompactReportSource { - /// Projects this receipt as the §11 report. - fn compact_report(&self) -> CompactReport; -} - -impl CompactReportSource for CompactReceipt { - fn compact_report(&self) -> CompactReport { - CompactReport { - path: self.path, - generation: CompactGeneration { - before: self.generation_before, - after: self.generation_after, - }, - dead_bytes_bound: self.dead_bytes_bound, - dead_bytes_refined: self.dead_bytes_refined, - logical_discarded: self.logical_discarded, - backend_reusable: self.backend_reusable, - physical_shrink: self.physical_shrink, - kept_waste: self.kept_waste.clone(), - peak_temp_bytes: self.peak_temp_bytes, - residue: self.residue.clone(), - estimate: self.estimate, - zero_reasons: self.zero_reasons.clone(), - } - } -} - -impl From<&CompactReceipt> for CompactReport { - fn from(receipt: &CompactReceipt) -> Self { - receipt.compact_report() - } -} - -impl From for CompactReport { - fn from(receipt: CompactReceipt) -> Self { - receipt.compact_report() - } -} - /// The operation `store.status` shows while a compaction runs, or after a dead /// owner left one (§11): `mark`, `build`, `verify` or `install`. /// diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs index df77c7c87..4b051aab9 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -13,9 +13,8 @@ use layerfs_history::{ HistoryCatalogConfig, HistoryError, LayerStackId, LayerStackRecord, StackInitialization, }; use layerfs_persistence::{ - compact_op_status, CompactFailpoint, CompactPath, CompactReport, CompactReportSource, - CompactRequest, PersistenceConfig, SqliteAcquisitionSchema, SqliteHandles, SqlitePackLayout, - SqlitePersistenceProfile, + compact_op_status, CompactFailpoint, CompactPath, CompactRequest, PersistenceConfig, + SqliteAcquisitionSchema, SqliteHandles, SqlitePackLayout, SqlitePersistenceProfile, }; use layerfs_storage::{port::PackPersistence, Storage, StoragePolicy}; use std::cell::RefCell; @@ -910,7 +909,7 @@ fn an_unrecorded_non_removable_artifact_does_not_strand_the_store() { } #[test] -fn the_report_projects_every_section_11_field() { +fn the_receipt_carries_every_section_11_field() { let (_temp, handles, _path) = create_store(); let live = filesystem_root(&handles, 1); initialize(&handles, 1, live); @@ -918,20 +917,11 @@ fn the_report_projects_every_section_11_field() { let before = generation(&handles); let receipt = handles.compact(&request(30)).unwrap(); assert_eq!(receipt.path, CompactPath::Rebuild); - let report = CompactReport::from(&receipt); - assert_eq!(report.path, CompactPath::Rebuild); - assert_eq!(report.generation.before, before); - assert_eq!(report.generation.after, before + 1); - assert_eq!(report.dead_bytes_bound, receipt.dead_bytes_bound); + let report = &receipt; + assert_eq!(report.generation_before, before); + assert_eq!(report.generation_after, before + 1); assert_eq!(report.dead_bytes_refined, report.dead_bytes_bound); - assert_eq!(report.logical_discarded, receipt.logical_discarded); - assert_eq!(report.backend_reusable, receipt.backend_reusable); - assert_eq!(report.physical_shrink, receipt.physical_shrink); - assert_eq!(report.kept_waste, receipt.kept_waste); - assert_eq!(report.peak_temp_bytes, receipt.peak_temp_bytes); - assert_eq!(report.residue, receipt.residue); assert!(report.estimate); - assert_eq!(report.zero_reasons, receipt.zero_reasons); assert!( report .zero_reasons @@ -939,17 +929,15 @@ fn the_report_projects_every_section_11_field() { .all(|zero| zero.wire_field() != "dead_bytes_bound"), "a rebuild with dead bytes does not explain a zero" ); - // `From` and the shared source trait project the same report. - assert_eq!(report, receipt.compact_report()); } #[test] fn the_empty_store_report_explains_its_zeros() { let (_temp, handles, path) = create_store(); let receipt = handles.compact(&request(31)).unwrap(); - let report = CompactReport::from(&receipt); + let report = &receipt; assert_eq!(report.path, CompactPath::None); - assert_eq!(report.generation.before, report.generation.after); + assert_eq!(report.generation_before, report.generation_after); assert_eq!(report.dead_bytes_refined, report.dead_bytes_bound); assert!(report.estimate); assert!( From 64c76ccb34c04f0f84b5efbb156a6ca98ab205a1 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 23:11:00 +0800 Subject: [PATCH 4/6] refactor(store-service): use opt_text in refusal details; drop an uncalled bind - history/lookup.rs built eight optional text fields of the MountsActive, Unknown and Busy details with `map_or(SdkValue::Null, SdkValue::text)` chains. They now use the existing `SdkValue::opt_text`. Same values. - store/bind.rs: `bind_private_placed` had no caller in engine, control, api or any test; removed. Checks (macOS host, Rust 1.85.1): cargo check --workspace --all-targets; Clippy -D warnings on layerfs-store-service; fmt; store-service tests history_wire_codes, admission, branch_remove, project_delete, epoch_fence and method_refusal_order (23 tests); layerfs-store tests project_delete, project_delete_store and compact_in_process_call (23 tests). No test changed or deleted. Rebased onto 4c296e9297 (layerfs#734) on 2026-11-18. The test counts above are from before the rebase. This commit compiles with its tests on the new base; the suites were rerun at the branch tip, not at this commit. Production LOC: 188,574 -> 188,529 (delta -45) core 40,589 -> 40,589 (0) engine 90,234 -> 90,189 (-45) layerfs-store-service -45 control 51,229 -> 51,229 (0) api 6,522 -> 6,522 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py --json on parent 52a0697cdd and the committed tree (recomputed after the rebase onto 4c296e9297) Co-Authored-By: Claude Fable 5.2 --- .../src/history/lookup.rs | 52 +++---------------- .../layerfs-store-service/src/store/bind.rs | 11 ---- 2 files changed, 8 insertions(+), 55 deletions(-) diff --git a/engine/crates/layerfs-store-service/src/history/lookup.rs b/engine/crates/layerfs-store-service/src/history/lookup.rs index d04b81038..ebfb5e9c0 100644 --- a/engine/crates/layerfs-store-service/src/history/lookup.rs +++ b/engine/crates/layerfs-store-service/src/history/lookup.rs @@ -147,27 +147,9 @@ fn mounts_detail(state: &layerfs_history::MountHolders) -> SdkValue { SdkValue::map([ ("kind", SdkValue::text(holder.kind.as_str())), ("entity", SdkValue::text(&holder.entity)), - ( - "workspace", - holder - .workspace - .as_ref() - .map_or(SdkValue::Null, SdkValue::text), - ), - ( - "sandbox", - holder - .sandbox - .as_ref() - .map_or(SdkValue::Null, SdkValue::text), - ), - ( - "machine", - holder - .machine - .as_ref() - .map_or(SdkValue::Null, SdkValue::text), - ), + ("workspace", SdkValue::opt_text(holder.workspace.as_ref())), + ("sandbox", SdkValue::opt_text(holder.sandbox.as_ref())), + ("machine", SdkValue::opt_text(holder.machine.as_ref())), ("op", SdkValue::text(&holder.op)), ]) }) @@ -187,17 +169,8 @@ fn unknown_detail(state: &layerfs_history::UnknownOutcomeState) -> SdkValue { ("name", SdkValue::text(&state.entity)), ]), ), - ( - "phase", - state - .phase - .as_deref() - .map_or(SdkValue::Null, SdkValue::text), - ), - ( - "op", - state.op.as_deref().map_or(SdkValue::Null, SdkValue::text), - ), + ("phase", SdkValue::opt_text(state.phase.as_deref())), + ("op", SdkValue::opt_text(state.op.as_deref())), ]) } @@ -212,10 +185,7 @@ fn op_detail(state: &layerfs_history::OpConflict) -> SdkValue { ("name", SdkValue::text(&state.entity)), ]), ), - ( - "phase", - state.phase.as_ref().map_or(SdkValue::Null, SdkValue::text), - ), + ("phase", SdkValue::opt_text(state.phase.as_ref())), ("op", SdkValue::text(&state.op)), ("uncertain", SdkValue::Bool(state.uncertain)), // The PostgreSQL holder, when the refusal names one (spec 10); @@ -223,17 +193,11 @@ fn op_detail(state: &layerfs_history::OpConflict) -> SdkValue { ("pid", state.pid.map_or(SdkValue::Null, SdkValue::Int)), ( "client_addr", - state - .client_addr - .as_ref() - .map_or(SdkValue::Null, SdkValue::text), + SdkValue::opt_text(state.client_addr.as_ref()), ), ( "backend_start", - state - .backend_start - .as_ref() - .map_or(SdkValue::Null, SdkValue::text), + SdkValue::opt_text(state.backend_start.as_ref()), ), ]) } diff --git a/engine/crates/layerfs-store-service/src/store/bind.rs b/engine/crates/layerfs-store-service/src/store/bind.rs index 361c05443..aead23e52 100644 --- a/engine/crates/layerfs-store-service/src/store/bind.rs +++ b/engine/crates/layerfs-store-service/src/store/bind.rs @@ -87,17 +87,6 @@ impl Store { ) -> Result { self.bind_with(owner, BindIntent::Id(request), MountPlace::default(), true) } - /// [`Self::bind_private`] of a caller that knows the Sandbox and - /// machine the Workspace belongs to. - #[allow(clippy::result_large_err)] - pub fn bind_private_placed( - self: &Arc, - owner: OwnerClient, - request: BindRequest, - place: MountPlace, - ) -> Result { - self.bind_with(owner, BindIntent::Id(request), place, true) - } #[allow(clippy::result_large_err)] fn bind_with( self: &Arc, From b8ff1afdfe6ab2f969fbe752cbbaeefb3767faf6 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 23:14:12 +0800 Subject: [PATCH 5/6] refactor(persistence): one admission function for the three removals `remove_branch`, `delete_stack` and `remove_path` each wrote the same admission block before their marker: on PostgreSQL the Store key shared, a scan for a live conflicting intent and the scope's own key; on both backends the project epoch hold and the scan that settles dead owners' intents. `admit` now holds that block once. Each removal passes its scan statement and parameters, its exclusive key, and whether it also holds the project key shared (a Branch removal does). Lock order, statements and refusals are unchanged. Checks (macOS host, Rust 1.85.1, PostgreSQL 17 in a private container): Clippy -D warnings on layerfs-persistence (all targets), fmt; persistence tests admission, branch_removal, layerstack_removal, path_removal, project_delete, epoch_fence, visibility_guard, semantic_admission (32 tests), postgresql_gc_fence (19) and postgresql_compact (30). No test changed or deleted. Rebased onto 4c296e9297 (layerfs#734) on 2026-11-18. The test counts above are from before the rebase. This commit compiles with its tests on the new base; the suites were rerun at the branch tip, not at this commit. Production LOC: 188,529 -> 188,487 (delta -42) core 40,589 -> 40,589 (0) engine 90,189 -> 90,147 (-42) layerfs-persistence -42 control 51,229 -> 51,229 (0) api 6,522 -> 6,522 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py --json on parent 64c76ccb34 and the committed tree (recomputed after the rebase onto 4c296e9297) Co-Authored-By: Claude Fable 5.2 --- .../src/history/admission/removal.rs | 165 +++++++----------- 1 file changed, 66 insertions(+), 99 deletions(-) diff --git a/engine/crates/layerfs-persistence/src/history/admission/removal.rs b/engine/crates/layerfs-persistence/src/history/admission/removal.rs index 145e81591..57fcdbce7 100644 --- a/engine/crates/layerfs-persistence/src/history/admission/removal.rs +++ b/engine/crates/layerfs-persistence/src/history/admission/removal.rs @@ -5,6 +5,48 @@ use super::*; +/// The admission every removal runs before its marker, inside the marker's +/// own write transaction. On PostgreSQL it takes the Store key shared, the +/// project key shared too when `hold_project` (a Branch removal), scans once +/// for a live conflicting intent and then takes the scope's own key +/// `exclusive` (class, id). On both backends it then holds the project epoch +/// and scans again, settling the intents dead owners left. `scan` is the +/// statement and its parameters; a conflict is refused as [`classify`] names +/// it. +fn admit( + tx: &Transaction<'_>, + provider: &HistoryProvider, + owners: &OwnerView, + stack: LayerStackId, + hold_project: bool, + exclusive: (u8, &[u8]), + scan: (&str, Vec), +) -> HistoryResult<()> { + let postgres = postgres(provider); + let (statement, scope) = scan; + if postgres { + let catalog = provider.catalog_id; + lock_store_shared(tx, catalog)?; + if hold_project { + lock_scope(tx, catalog, 2, stack.as_slice(), true)?; + } + let live: Vec = query(tx, statement, scope.clone())? + .into_iter() + .filter(|intent| !owners.settleable(intent, postgres)) + .collect(); + if !live.is_empty() { + return Err(classify(&live)); + } + lock_scope(tx, catalog, exclusive.0, exclusive.1, false)?; + } + epoch_hold(tx, postgres, stack)?; + let found = settle_dead(tx, query(tx, statement, scope)?, postgres, owners)?; + if !found.is_empty() { + return Err(classify(&found)); + } + Ok(()) +} + /// `branch.remove`: scan the Branch scope and write the marker in one write /// transaction. pub(crate) fn remove_branch( @@ -13,51 +55,25 @@ pub(crate) fn remove_branch( removed_ms: i64, intent: &GcIntent, ) -> HistoryResult<()> { - let postgres = postgres(provider); - let catalog = provider.catalog_id; - let _ = catalog; provider.write(|tx| { let owners = OwnerView::now(); let record = branch::branch(tx, branch)?.ok_or(HistoryError::Missing(Missing::Branch))?; - if postgres { - lock_store_shared(tx, catalog)?; - lock_scope(tx, catalog, 2, record.stack.as_slice(), true)?; - let found = query( - tx, - ALL_BRANCH, - params![ - Some(record.stack.as_slice().to_vec()), - branch.as_slice().to_vec(), - intent.op_id.to_vec() - ], - )?; - let found: Vec = found - .into_iter() - .filter(|intent| !owners.settleable(intent, postgres)) - .collect(); - if !found.is_empty() { - return Err(classify(&found)); - } - lock_scope(tx, catalog, 3, branch.as_slice(), false)?; - } - epoch_hold(tx, postgres, record.stack)?; - let found = settle_dead( + admit( tx, - query( - tx, + provider, + &owners, + record.stack, + true, + (3, branch.as_slice()), + ( ALL_BRANCH, params![ Some(record.stack.as_slice().to_vec()), branch.as_slice().to_vec(), intent.op_id.to_vec() ], - )?, - postgres, - &owners, + ), )?; - if !found.is_empty() { - return Err(classify(&found)); - } super::super::removal::mark_branch_removed(tx, branch, removed_ms) }) } @@ -70,9 +86,6 @@ pub(crate) fn delete_stack( deleted_ms: i64, intent: &GcIntent, ) -> HistoryResult { - let postgres = postgres(provider); - let catalog = provider.catalog_id; - let _ = catalog; provider.write(|tx| { let owners = OwnerView::now(); if let Some(recorded) = layerstack::stack_deleted_ms(tx, stack)? { @@ -89,44 +102,22 @@ pub(crate) fn delete_stack( None => GcDeletion::Absent, }); }; - if postgres { - lock_store_shared(tx, catalog)?; - let found = query( - tx, - ALL_PROJECT, - params![ - stack.as_slice().to_vec(), - Some(record.name.as_str().to_owned()), - intent.op_id.to_vec() - ], - )?; - let found: Vec = found - .into_iter() - .filter(|intent| !owners.settleable(intent, postgres)) - .collect(); - if !found.is_empty() { - return Err(classify(&found)); - } - lock_scope(tx, catalog, 2, stack.as_slice(), false)?; - } - epoch_hold(tx, postgres, stack)?; - let found = settle_dead( + admit( tx, - query( - tx, + provider, + &owners, + stack, + false, + (2, stack.as_slice()), + ( ALL_PROJECT, params![ stack.as_slice().to_vec(), Some(record.name.as_str().to_owned()), intent.op_id.to_vec() ], - )?, - postgres, - &owners, + ), )?; - if !found.is_empty() { - return Err(classify(&found)); - } layerstack::mark_stack_deleted(tx, stack, deleted_ms)?; Ok(GcDeletion::Marked { deleted_ms }) }) @@ -140,51 +131,27 @@ pub(crate) fn remove_path( intent: &GcIntent, ) -> HistoryResult { let postgres = postgres(provider); - let catalog = provider.catalog_id; - let _ = catalog; provider.write(|tx| { let owners = OwnerView::now(); if layerstack::layer_stack(tx, request.stack)?.is_none() { return Err(HistoryError::Missing(Missing::LayerStack)); } - if postgres { - lock_store_shared(tx, catalog)?; - let found = query( - tx, - ALL_PROJECT, - params![ - request.stack.as_slice().to_vec(), - intent.project.clone(), - intent.op_id.to_vec() - ], - )?; - let found: Vec = found - .into_iter() - .filter(|intent| !owners.settleable(intent, postgres)) - .collect(); - if !found.is_empty() { - return Err(classify(&found)); - } - lock_scope(tx, catalog, 2, request.stack.as_slice(), false)?; - } - epoch_hold(tx, postgres, request.stack)?; - let found = settle_dead( + admit( tx, - query( - tx, + provider, + &owners, + request.stack, + false, + (2, request.stack.as_slice()), + ( ALL_PROJECT, params![ request.stack.as_slice().to_vec(), intent.project.clone(), intent.op_id.to_vec() ], - )?, - postgres, - &owners, + ), )?; - if !found.is_empty() { - return Err(classify(&found)); - } let key = trace_key(&provider.trace_salt, request.stack, &request.path); let stored = one( tx, From f2e27d7bdfbda30f7f4f852ae59e32bf38d0cfc6 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Wed, 18 Nov 2026 02:14:34 +0800 Subject: [PATCH 6/6] feat(compact): store.compact takes on_progress and sends one event per phase (#42 A13) Owner direction 2026-11-18: the caller of a compaction should see its stages as they happen, through the same on_progress events store.push and backup.create already send, so a client's job display needs no polling. What changes - API: store.compact declares the shared PROGRESS option, the twelfth call to do so. Node, Python and the CLI (--progress) get it from the table; index.d.ts, _client.pyi and aio.pyi are regenerated. - Control: compact() uses the event stream it already received and sends { phase, done, total } with null counts as each phase starts. - Engine: RunMetrics, the phase clock both backends share, holds an observer and tells it a phase once, when the phase changes. CompactPhase::step gives the name store.status uses. SqliteHandles and PostgresHandles gain compact_observed; compact() calls it with no observer, so every existing caller is unchanged. - Phases: mark, build, verify, install on SQLite; mark, build, swap, cleanup on PostgreSQL. A none run sends mark alone. The SQLite settle (cleanup in store.status, D4) is inside install and sends no event. What does not change - The op record and store.status: still the only record another process, or a call after a crash, reads. Events are in memory and write nothing. - No byte counts inside a phase, no estimate before a run, no cancel. Spec: the frozen GC spec says there is no PROGRESS option (#42 item 40). The spec file stays byte-identical; the change is recorded as amendment A13 in core/docs/issues/project-41/gc-42-decisions.md for sign-off. Tests - New: an_observed_run_is_told_each_phase_once_in_order (sqlite_compact, postgresql_compact) and store_compact_sends_each_phase_as_a_progress_event (gc_calls, through layerfs_api::call: none then rebuild). - Changed on purpose, because the contract changed: the two "four GC calls take ids only" tests (gc_calls, calls_table) asserted no options at all; they now assert that the three removals have none and that store.compact has exactly the on_progress function. The progress lists (progress_options, node progress.mjs, python test_progress, CLI progress) name twelve calls. No test deleted. - Ran, each under 120 s, macOS host, after the rebase onto 4c296e9297 (layerfs#734): all 89 layerfs-persistence test files (sqlite_compact 37); with PostgreSQL 17 in Docker on loopback, postgresql_compact (32), postgresql_gc_fence (19) and nine more PostgreSQL files; layerfs-api gc_calls (9), calls_table (22), progress_options (2), gc_trace (5), store_family (5); layerfs-cli gc (5), progress (5); layerfs-store compact_report_status (6), compact_in_process_call (2); nine layerfs-store-service removal and refusal files; Clippy -D warnings on the five changed crates. gen-node-types --check ran before the rebase only. - The T12.3a matrix report is regenerated: it records the test counts of sqlite_compact and postgresql_compact, which each gained one. - Not run: the Node and Python end-to-end progress tests and gen-coverage --check (they need the Python module and a Linux Store host; the coverage row is edited to what the generator writes), any Linux build, and the macOS worker route for these events. Docs: errors-and-events.md (call list, shape and phase tables), the API README count, sdk/store.md, coverage.md and the JS SDK skill reference. Production LOC: 188,487 -> 188,532 (delta +45) core 40,589 -> 40,589 (0) engine 90,147 -> 90,180 (+33) layerfs-persistence +33 control 51,229 -> 51,241 (+12) layerfs-store +12 api 6,522 -> 6,522 (0) reference absent (retired R9c; last counted 65,417 at tree 498dd1917812) Method: tools/production_loc.py --json on parent b8ff1afdfe and the committed tree (recomputed after the rebase onto 4c296e9297) Co-Authored-By: Claude Fable 5.2 --- .../references/methods/store.md | 4 +- api/crates/layerfs-api/src/store/compact.rs | 4 +- api/crates/layerfs-api/tests/calls_table.rs | 10 ++- api/crates/layerfs-api/tests/gc_calls.rs | 45 +++++++++++++- .../layerfs-api/tests/progress_options.rs | 5 +- api/crates/layerfs-cli/tests/progress.rs | 1 + api/node/src/index.d.ts | 7 ++- api/node/test/progress.mjs | 6 +- api/python/src/layerfs/_client.pyi | 8 ++- api/python/src/layerfs/aio.pyi | 8 ++- api/python/test/test_progress.py | 3 +- control/crates/layerfs-store/src/compact.rs | 62 ++++++++++++------- .../docs/issues/project-41/gc-42-decisions.md | 34 +++++++++- .../gc-t12.3a-sqlite-matrix-report.md | 4 +- docs/architecture/api/README.md | 2 +- docs/architecture/api/calls/coverage.md | 2 +- .../api/calls/errors-and-events.md | 7 ++- docs/architecture/api/sdk/store.md | 13 +++- .../src/backend/postgresql/compact/run.rs | 13 +++- .../src/backend/sqlite/compact/run.rs | 13 +++- .../src/compact_metrics.rs | 30 ++++++++- .../tests/postgresql_compact.rs | 26 ++++++++ .../tests/sqlite_compact.rs | 24 +++++++ 23 files changed, 273 insertions(+), 58 deletions(-) diff --git a/.agents/skills/layerfs-js-sdk/references/methods/store.md b/.agents/skills/layerfs-js-sdk/references/methods/store.md index 9cfeec57a..f9095c1af 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/store.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/store.md @@ -18,7 +18,7 @@ store.update(name, { machine, url, password, synchronous_commit, clear }) store.push(name, machine, { on_progress }) store.pull(name, { on_progress }) store.remove(name) -store.compact(name) +store.compact(name, { on_progress }) store.disconnect(name) ``` @@ -84,7 +84,7 @@ Store on another machine is read through its worker; a stopped worker is | `push(name, machine, …)` | Sends a sqlite Store's file from `native` to `machine` | `InvalidInput` unless the Store is on `native` (pull it first). Progress `{ phase, done, total }` on `on_progress` / `--progress`: `done` and `total` are bytes during `send`, null in the other phases (`check`, `snapshot`, `verify`, `install`, `record`, `retire`). `InUse` while the Store is held (Sandbox records, processes with it open; `detail.related` lists them as `{ kind, name }`); `Capacity` when the target has less free disk than the Store plus a headroom of 1 GiB (or a tenth of the Store), checked before anything changes; `Unavailable` naming `layerfs machine start ` for a stopped machine; while it runs, Store calls and operations on it are `Busy`, from the caller's process too. An uncertain push whose process is gone (it ended during install or record, or a step's answer was lost) leaves every later call `Unresolved`, naming `store status` and the push that clears it: a new `push` (or `pull`) first deletes the copy on the machine the record does not name, then runs. A push whose `retire` did not finish leaves the Store in use on `machine` and `last_transfer.retired: false`; the next `pull` deletes the old copy first | | `pull(name, …)` | Brings a sqlite Store's file from its machine to `native` | `InvalidInput` when it is already on `native`; otherwise as `push`. Between two machines: `pull`, then `push` | | `remove(name)` | Deletes a sqlite Store with every project in it | Returns `{ removed }`. `NotFound` for an unknown name (live); `Unavailable` for a Store whose machine is unknown; `Busy` while a live process pushes, pulls or removes it. It clears an `Unresolved` Store. Backups stay | -| `compact(name)` | Rebuilds the Store from its live history, reclaiming dead bytes | The only byte-reclaiming call: no confirm, force or dry run. Returns `{ kind: "Compacted", path, generation, … }` with `dead_bytes_bound`, `dead_bytes_refined`, `logical_discarded`, `backend_reusable`, `physical_shrink`, `kept_waste`, `peak_temp_bytes`, `residue`, `estimate` and `zero_reasons` (a list of `{field, reason}`). A Store with under 10% dead packs takes the `none` path and only marks. A sqlite Store on another machine is `Unsupported` (local-only route). `MountsActive`, `Busy`, `Unresolved`, `StaleStore`, `Conflict`, `Capacity`, `Unknown` | +| `compact(name, { on_progress })` | Rebuilds the Store from its live history, reclaiming dead bytes | The only byte-reclaiming call: no confirm, force or dry run. `on_progress` gets `{ phase, done, total }` as each phase starts (`mark`, `build`, `verify`, `install` on sqlite; `mark`, `build`, `swap`, `cleanup` on PostgreSQL), with null counts. Returns `{ kind: "Compacted", path, generation, … }` with `dead_bytes_bound`, `dead_bytes_refined`, `logical_discarded`, `backend_reusable`, `physical_shrink`, `kept_waste`, `peak_temp_bytes`, `residue`, `estimate` and `zero_reasons` (a list of `{field, reason}`). A Store with under 10% dead packs takes the `none` path and only marks. A sqlite Store on another machine is `Unsupported` (local-only route). `MountsActive`, `Busy`, `Unresolved`, `StaleStore`, `Conflict`, `Capacity`, `Unknown` | | `disconnect(name)` | Forgets a PostgreSQL Store; its data stays | `InUse` while another process holds the Store (its calls hold it while it stays open) or a Sandbox of it runs; `detail.related` lists them as `{ kind: "lease", name: }` and `{ kind: "sandbox", name }` | Pushing or pulling a Store moves where its Sandboxes can run: a Sandbox of a sqlite Store diff --git a/api/crates/layerfs-api/src/store/compact.rs b/api/crates/layerfs-api/src/store/compact.rs index dc1969db2..eb15ae9e0 100644 --- a/api/crates/layerfs-api/src/store/compact.rs +++ b/api/crates/layerfs-api/src/store/compact.rs @@ -4,7 +4,7 @@ //! local-only (spec S6 deviation D5, `core/docs/issues/project-41/gc-42-decisions.md`); //! this entry parses the name and hands the call over. use crate::calls::table::Kind::Name; -use crate::calls::table::{a, call, Call}; +use crate::calls::table::{a, call, Call, PROGRESS}; use crate::calls::Context; use layerfs_bridge::control::SdkValue; use layerfs_home::names::StoreName; @@ -15,7 +15,7 @@ pub(crate) const CALL: Call = call( "store", "compact", &[a("name", Name)], - &[], + &[PROGRESS], "reclaim dead bytes from one Store", run, ); diff --git a/api/crates/layerfs-api/tests/calls_table.rs b/api/crates/layerfs-api/tests/calls_table.rs index 2859fa9ab..0c1fc9b9b 100644 --- a/api/crates/layerfs-api/tests/calls_table.rs +++ b/api/crates/layerfs-api/tests/calls_table.rs @@ -567,7 +567,8 @@ fn sandbox_images_are_named_by_the_image_kind() { #[test] fn the_four_gc_calls_take_only_ids() { - // T11.1: the four GC calls, each one resource identifier and no option. + // T11.1: the four GC calls, each one resource identifier and no data + // option; `store.compact` alone declares `on_progress` (#42 item 45). use layerfs_api::Kind; let expected: &[(&str, &[(&str, Kind)])] = &[ ( @@ -586,6 +587,11 @@ fn the_four_gc_calls_take_only_ids() { let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); assert_eq!(given.as_slice(), args, "{name}"); assert!(spec.args.iter().all(|arg| !arg.optional), "{name}"); - assert!(spec.options.is_empty(), "{name} takes no option"); + let options: Vec<(&str, Kind)> = spec.options.iter().map(|o| (o.name, o.kind)).collect(); + if name == "store.compact" { + assert_eq!(options, [("on_progress", Kind::Function)], "{name}"); + } else { + assert!(options.is_empty(), "{name} takes no option"); + } } } diff --git a/api/crates/layerfs-api/tests/gc_calls.rs b/api/crates/layerfs-api/tests/gc_calls.rs index ad390c8ea..01f3d6370 100644 --- a/api/crates/layerfs-api/tests/gc_calls.rs +++ b/api/crates/layerfs-api/tests/gc_calls.rs @@ -92,7 +92,14 @@ fn the_four_gc_calls_take_ids_only() { let given: Vec<(&str, Kind)> = spec.args.iter().map(|arg| (arg.name, arg.kind)).collect(); assert_eq!(given.as_slice(), args, "{name} arguments"); assert!(spec.args.iter().all(|arg| !arg.optional), "{name} required"); - assert!(spec.options.is_empty(), "{name} has no options"); + // ids only: no call takes a data option. `store.compact` alone + // declares the shared `on_progress` function (#42 item 45). + let options: Vec<(&str, Kind)> = spec.options.iter().map(|o| (o.name, o.kind)).collect(); + if name == "store.compact" { + assert_eq!(options, [("on_progress", Kind::Function)], "{name}"); + } else { + assert!(options.is_empty(), "{name} has no options"); + } } } @@ -273,6 +280,42 @@ fn project_init_after_delete_is_nametaken_deleted() { let _ = std::fs::remove_dir_all(folder); } +/// `store.compact` sends each phase as a progress event before it answers: +/// `{ progress: { phase, done, total } }` with null counts. +#[test] +fn store_compact_sends_each_phase_as_a_progress_event() { + let (home, folder) = project(); + let compact = || { + let mut phases = Vec::new(); + let report = layerfs_api::call( + Some(&home), + "store.compact", + map(&[("name", t("shop"))]), + &mut |event| { + let progress = event.get("progress").expect("a progress event").clone(); + assert_eq!(progress.get("done"), Some(&SdkValue::Null), "{progress:?}"); + assert_eq!(progress.get("total"), Some(&SdkValue::Null), "{progress:?}"); + phases.push(progress.get_text("phase").expect("phase").to_owned()); + }, + ) + .unwrap_or_else(|refusal| panic!("store.compact: {refusal:?}")); + (report.get_text("path").expect("path").to_owned(), phases) + }; + // nothing is dead yet: the run stops after its mark + let (path, phases) = compact(); + assert_eq!( + (path.as_str(), phases.as_slice()), + ("none", ["mark".to_owned()].as_slice()) + ); + // a deleted project makes the next run a rebuild, through all four phases + ok(&home, "project.delete", &[p()]); + let (path, phases) = compact(); + assert_eq!(path, "rebuild"); + assert_eq!(phases, ["mark", "build", "verify", "install"]); + std::fs::remove_dir_all(home).unwrap(); + let _ = std::fs::remove_dir_all(folder); +} + #[test] fn store_compact_refuses_an_unknown_store_and_reports_a_run() { let home = tempdir("compact-none"); diff --git a/api/crates/layerfs-api/tests/progress_options.rs b/api/crates/layerfs-api/tests/progress_options.rs index 9c6c82f34..9c73c7dda 100644 --- a/api/crates/layerfs-api/tests/progress_options.rs +++ b/api/crates/layerfs-api/tests/progress_options.rs @@ -15,7 +15,7 @@ fn declaring(option: &str) -> Vec { } #[test] -fn eleven_calls_declare_on_progress_and_one_declares_the_exec_sinks() { +fn twelve_calls_declare_on_progress_and_one_declares_the_exec_sinks() { assert_eq!( declaring("on_progress"), [ @@ -28,6 +28,7 @@ fn eleven_calls_declare_on_progress_and_one_declares_the_exec_sinks() { "machine.create", "store.push", "store.pull", + "store.compact", "backup.create", "backup.restore" ] @@ -44,7 +45,7 @@ fn eleven_calls_declare_on_progress_and_one_declares_the_exec_sinks() { .count() }) .sum(); - assert_eq!(functions, 11 + 2); + assert_eq!(functions, 12 + 2); } #[test] diff --git a/api/crates/layerfs-cli/tests/progress.rs b/api/crates/layerfs-cli/tests/progress.rs index 3ab807dfe..10a97820d 100644 --- a/api/crates/layerfs-cli/tests/progress.rs +++ b/api/crates/layerfs-cli/tests/progress.rs @@ -177,6 +177,7 @@ fn only_calls_declaring_on_progress_accept_the_cli_switch() { ["project", "init", "--help"], ["store", "push", "--help"], ["store", "pull", "--help"], + ["store", "compact", "--help"], ["backup", "create", "--help"], ["backup", "restore", "--help"], ["workspace", "exec", "--help"], diff --git a/api/node/src/index.d.ts b/api/node/src/index.d.ts index b77b0b8ae..a2ab404e4 100644 --- a/api/node/src/index.d.ts +++ b/api/node/src/index.d.ts @@ -418,6 +418,11 @@ export interface StorePullOptions extends CallOptions { on_progress?: ProgressHandler; } +/** Options of `store.compact`; an unknown option is refused. */ +export interface StoreCompactOptions extends CallOptions { + on_progress?: ProgressHandler; +} + /** Options of `backup.connect`; an unknown option is refused. */ export interface BackupConnectOptions extends CallOptions { key?: string; @@ -647,7 +652,7 @@ export interface StoreFamily { /** delete a sqlite Store with every project in it */ remove(name: string, options?: CallOptions): Promise; /** reclaim dead bytes from one Store */ - compact(name: string, options?: CallOptions): Promise; + compact(name: string, options?: StoreCompactOptions): Promise; /** forget a PostgreSQL Store; its data stays */ disconnect(name: string, options?: CallOptions): Promise; } diff --git a/api/node/test/progress.mjs b/api/node/test/progress.mjs index 52107f35a..4ef77fd7b 100644 --- a/api/node/test/progress.mjs +++ b/api/node/test/progress.mjs @@ -39,11 +39,11 @@ function folderOf(files) { const calling = (name) => table.filter((c) => c.options.some((o) => o.name === name && o.kind === "function")).map((c) => `${c.family}.${c.call}`); -test("the table: eleven calls take on_progress and one takes the exec sinks", () => { - assert.deepEqual(calling("on_progress"), ["project.init", "project.export", "sandbox.create", "workspace.exec", "machine.load", "machine.start", "machine.create", "store.push", "store.pull", "backup.create", "backup.restore"]); +test("the table: twelve calls take on_progress and one takes the exec sinks", () => { + assert.deepEqual(calling("on_progress"), ["project.init", "project.export", "sandbox.create", "workspace.exec", "machine.load", "machine.start", "machine.create", "store.push", "store.pull", "store.compact", "backup.create", "backup.restore"]); assert.deepEqual(calling("stdout"), ["workspace.exec"]); assert.deepEqual(calling("stderr"), ["workspace.exec"]); - assert.equal(table.flatMap((c) => c.options).filter((o) => o.kind === "function").length, 11 + 2); + assert.equal(table.flatMap((c) => c.options).filter((o) => o.kind === "function").length, 12 + 2); }); test("project.init in this process: check, install, record with null counts, while it runs, before it settles", { timeout: 90_000 }, async () => { diff --git a/api/python/src/layerfs/_client.pyi b/api/python/src/layerfs/_client.pyi index 84e3ad46e..9e0318a80 100644 --- a/api/python/src/layerfs/_client.pyi +++ b/api/python/src/layerfs/_client.pyi @@ -574,7 +574,13 @@ class StoreFamily: """bring a sqlite Store from its machine to native""" def remove(self, name: str, *, log_tag: str | None = None) -> Result: """delete a sqlite Store with every project in it""" - def compact(self, name: str, *, log_tag: str | None = None) -> Result: + def compact( + self, + name: str, + *, + on_progress: Callable[[Progress], object] = ..., + log_tag: str | None = None, + ) -> Result: """reclaim dead bytes from one Store""" def disconnect(self, name: str, *, log_tag: str | None = None) -> Result: """forget a PostgreSQL Store; its data stays""" diff --git a/api/python/src/layerfs/aio.pyi b/api/python/src/layerfs/aio.pyi index f5b7060c0..84cc40c2a 100644 --- a/api/python/src/layerfs/aio.pyi +++ b/api/python/src/layerfs/aio.pyi @@ -585,7 +585,13 @@ class AsyncStoreFamily: """bring a sqlite Store from its machine to native""" async def remove(self, name: str, *, log_tag: str | None = None) -> Result: """delete a sqlite Store with every project in it""" - async def compact(self, name: str, *, log_tag: str | None = None) -> Result: + async def compact( + self, + name: str, + *, + on_progress: Callable[[Progress], object] = ..., + log_tag: str | None = None, + ) -> Result: """reclaim dead bytes from one Store""" async def disconnect(self, name: str, *, log_tag: str | None = None) -> Result: """forget a PostgreSQL Store; its data stays""" diff --git a/api/python/test/test_progress.py b/api/python/test/test_progress.py index c67ef3ee9..721857ce2 100644 --- a/api/python/test/test_progress.py +++ b/api/python/test/test_progress.py @@ -63,7 +63,7 @@ def setUp(self): self.addCleanup(shutil.rmtree, self.home, True) self.addCleanup(shutil.rmtree, self.folder, True) - def test_the_table_eleven_calls_take_on_progress(self): + def test_the_table_twelve_calls_take_on_progress(self): self.assertEqual( declaring("on_progress"), [ @@ -76,6 +76,7 @@ def test_the_table_eleven_calls_take_on_progress(self): "machine.create", "store.push", "store.pull", + "store.compact", "backup.create", "backup.restore", ], diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs index d512e863e..8328c1193 100644 --- a/control/crates/layerfs-store/src/compact.rs +++ b/control/crates/layerfs-store/src/compact.rs @@ -20,6 +20,7 @@ use crate::open_args::OpenArgs; use crate::place::{self, Executor}; use crate::postgresql; use crate::record::{StoreRecord, StoreRef}; +use crate::sqlite::family::progress; use layerfs_bridge::control::{SdkRefusal, SdkValue}; use layerfs_history::GcScopeKind; use layerfs_home::events::Events; @@ -71,8 +72,10 @@ fn crash_failpoint() -> Option { } /// `store.compact(name)`: reclaim one Store's dead bytes, answering -/// the §11 report. -pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { +/// the §11 report. Each phase is sent to `events` as it starts, as +/// `{ phase, done, total }` with null counts (the shape `store.push` sends); +/// the op record `store.status` reads is written by the run, not from here. +pub fn compact(home: &Path, events: &mut Events<'_>, store: &str) -> Outcome { let home = home.to_path_buf(); // A live operation is `Busy`, a dead owner's record `Unresolved`; the // refusal is a bounded metric label and never changes what is returned. @@ -85,7 +88,7 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { }; let store_ref = record.store_ref()?.clone(); if matches!(store_ref, StoreRef::Postgres { .. }) { - return compact_postgresql(&record, &home, store); + return compact_postgresql(&record, &home, events, store); } match place::plan(&home, &store_ref)? { Executor::Local => {} @@ -133,17 +136,20 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { layerfs_bridge::sdk_values::hex(&op_id) )); let receipt = handles - .compact(&CompactRequest { - op_id, - owner_pid: Some(std::process::id()), - owner_pid_start, - owner_pid_boot, - owner_runtime: Some(owner_runtime), - owner_boot: Some(owner_boot), - owner_live: Some(layerfs_home::process::recorded_owner_alive), - failpoint: crash_failpoint(), - ..CompactRequest::default() - }) + .compact_observed( + &CompactRequest { + op_id, + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + owner_runtime: Some(owner_runtime), + owner_boot: Some(owner_boot), + owner_live: Some(layerfs_home::process::recorded_owner_alive), + failpoint: crash_failpoint(), + ..CompactRequest::default() + }, + &mut |phase| progress(events, phase, None), + ) .map_err(|error| { lookup::history_in(error, GcScopeKind::Store, store, None, Some(&op_id)) })?; @@ -155,7 +161,12 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { /// G + 1 from the swap on; a lost reply is judged by the next call (§9). /// /// The PostgreSQL engine is always built beside SQLite (#726). -fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome { +fn compact_postgresql( + record: &StoreRecord, + home: &Path, + events: &mut Events<'_>, + store: &str, +) -> Outcome { let url = crate::record::url(record)?; let id = identity::identity(&OpenArgs { home: Some(home.to_path_buf()), @@ -186,15 +197,18 @@ fn compact_postgresql(record: &StoreRecord, home: &Path, store: &str) -> Outcome layerfs_bridge::sdk_values::hex(&op_id) )); let receipt = handles - .compact(&CompactRequest { - op_id, - owner_pid: Some(std::process::id()), - owner_pid_start, - owner_pid_boot, - owner_runtime: Some(owner_runtime), - owner_boot: Some(owner_boot), - ..CompactRequest::default() - }) + .compact_observed( + &CompactRequest { + op_id, + owner_pid: Some(std::process::id()), + owner_pid_start, + owner_pid_boot, + owner_runtime: Some(owner_runtime), + owner_boot: Some(owner_boot), + ..CompactRequest::default() + }, + &mut |phase| progress(events, phase, None), + ) .map_err(|error| { lookup::history_in(error, GcScopeKind::Store, store, None, Some(&op_id)) })?; diff --git a/core/docs/issues/project-41/gc-42-decisions.md b/core/docs/issues/project-41/gc-42-decisions.md index eb45bbac9..b01bda927 100644 --- a/core/docs/issues/project-41/gc-42-decisions.md +++ b/core/docs/issues/project-41/gc-42-decisions.md @@ -31,7 +31,7 @@ frozen whole-lane exclusion; its measurements live under --- -## A. Contract amendments (11) and one reading +## A. Contract amendments (12) and one reading ### A1. §4.1 `project.delete` sidecar gains `op_id` (additive) @@ -291,6 +291,34 @@ frozen whole-lane exclusion; its measurements live under - **No contract change:** neither frozen sentence is edited; the spec stays byte-identical until sign-off, like A8–A11. +### A13. §11: `store.compact` takes `on_progress` and sends one event per phase + +- **Frozen:** §11 says compaction writes its phase into the op, that + `store.status` shows it, and "there is no `PROGRESS` option (#42 item 40)". +- **Proposed (owner, 2026-11-18):** `store.compact` declares the shared + `on_progress` function, the twelfth call to do so. The run sends + `{ phase, done, total }` as each phase starts, once per phase and with null + counts: `mark`, `build`, `verify`, `install` on SQLite and `mark`, `build`, + `swap`, `cleanup` on PostgreSQL. A `none` run sends `mark` alone. The SQLite + settle (D4's `cleanup` in `store.status`) is inside `install` and sends no + event of its own. The op record and `store.status` are unchanged and stay + the only record another process, or a call after a crash, reads. +- **Reason:** the caller of a long rebuild saw nothing until the report; a + client that drives its progress display from `on_progress`, as it does for + `store.push` and `backup.create`, had to poll `store.status` for this one + call. The four GC calls still take ids only: `on_progress` is a function, + not a data option, and a function never crosses into the library. +- **Not in this entry:** byte counts inside a phase, an estimate before the + run, and cancellation. +- **Where:** `api/crates/layerfs-api/src/store/compact.rs` (`PROGRESS`), + `control/crates/layerfs-store/src/compact.rs` (the events), + `engine/crates/layerfs-persistence/src/compact_metrics.rs` + (`RunMetrics::begin_phase`, `CompactPhase::step`) and `compact_observed` on + both backends' handles; tests `an_observed_run_is_told_each_phase_once_in_order` + (`sqlite_compact.rs`, `postgresql_compact.rs`) and + `store_compact_sends_each_phase_as_a_progress_event` (`gc_calls.rs`). The + frozen spec file stays byte-identical until sign-off, like A8–A11. + --- ## D. SQLite implementation notes: four deviations and the D2 build decision @@ -573,9 +601,9 @@ rotational (HDD) backing — not NVMe** (88 vCPU across two Xeon E5-2699C v4, ## Request -Please record A1–A11 as #42 items (or amend the frozen text) and note the A12 +Please record A1–A11 and A13 as #42 items (or amend the frozen text) and note the A12 reading (it changes no contract text), and accept or reject D1–D5 individually when the owner signs off, and accept G7 as the record of a deferred design (it changes no contract text). Until then the implementation follows the frozen -text except where A8–A12 and D1–D5 are named, and the spec file in this branch +text except where A8–A13 and D1–D5 are named, and the spec file in this branch is byte-identical to the frozen version. diff --git a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md index c7f24753d..afc1ff935 100644 --- a/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md +++ b/core/docs/issues/project-41/gc-t12.3a-sqlite-matrix-report.md @@ -263,7 +263,7 @@ AC-51 does not depend on the projection alone. `/var/tmp`), rustc 1.85.1, `SqlitePersistenceProfile::Disposable` (WAL/OFF), `CARGO_TARGET_DIR=/tmp/target-t123c`. - Verified here: `sqlite_compact_matrix` (1 test); the persistence suites - `sqlite_compact` (36), `sqlite_compact_crash` (24), + `sqlite_compact` (37), `sqlite_compact_crash` (24), `sqlite_compact_fixtures` (1), `admission` (10), `path_removal` (4), `branch_removal` (4), `project_delete` (4), `epoch_fence` (7), `gc_backfill` (5), `kept_objects` (7); the Store-service path suites @@ -272,7 +272,7 @@ AC-51 does not depend on the projection alone. `sdk_path_removal` (5); and the `layerfs-control project_delete_worker` (8), the `layerfs-store project_delete` (18), `compact_report_status` (6) and the - PostgreSQL AC-44 suite `postgresql_compact` (31 tests; AC-44's nine are + PostgreSQL AC-44 suite `postgresql_compact` (32 tests; AC-44's nine are `a_fully_privileged_role_compacts`, `a_role_without_create_on_the_database_is_refused_as_unsupported`, `a_role_without_create_on_the_store_schema_is_refused_as_unsupported`, diff --git a/docs/architecture/api/README.md b/docs/architecture/api/README.md index 1429dc920..4dd684137 100644 --- a/docs/architecture/api/README.md +++ b/docs/architecture/api/README.md @@ -58,7 +58,7 @@ give each call's own arguments, answer and errors. | Uncertainty | `Unknown`, and `Failed` with `attempted: true`, mean the outcome is not known; `Unresolved` blocks later calls on that entity. Read state with `get`, `list` or `status` before acting again | [Uncertain outcomes](calls/errors-and-events.md#uncertain-outcomes) | | One attempt | a call runs once; nothing retries, refreshes or replays it | [`calls/dispatch.rs`](../../../api/crates/layerfs-api/src/calls/dispatch.rs) | | Paging | lists that page take `page_size` and `page_pos` and answer `{ items, has_more, next_page_pos }`; pass `next_page_pos` back as `page_pos`. Diffs add `paths_only` and a `cursor`; `logs.list` uses `cursor` and `limit` | [`calls/table.rs`](../../../api/crates/layerfs-api/src/calls/table.rs) `PAGE`, `DIFF` | -| Events | `on_progress` (eleven calls) and `workspace.exec`'s `stdout`/`stderr` functions receive events before the call answers; an event never means the call finished | [Errors and events](calls/errors-and-events.md) | +| Events | `on_progress` (twelve calls) and `workspace.exec`'s `stdout`/`stderr` functions receive events before the call answers; an event never means the call finished | [Errors and events](calls/errors-and-events.md) | | Home | explicit, else `LAYERFS_HOME`, else `$HOME/.layerfs`; absolute | [`calls/dispatch.rs`](../../../api/crates/layerfs-api/src/calls/dispatch.rs) `home` | | Scope | a client may be bound to one project, folder and Branch namespace: out-of-scope calls are `InvalidInput`, paths are relative to the folder and Branch names to the namespace | [`calls/scope.rs`](../../../api/crates/layerfs-api/src/calls/scope.rs), [folder scope](calls/README.md#folder-scope) | | Call log | every call except `logs.*` writes one line to `/trace/calls.jsonl` when it ends (not with `LAYERFS_LOGS=0`) | [the call log](calls/README.md#the-call-log) | diff --git a/docs/architecture/api/calls/coverage.md b/docs/architecture/api/calls/coverage.md index d7a178fa6..211949d7e 100644 --- a/docs/architecture/api/calls/coverage.md +++ b/docs/architecture/api/calls/coverage.md @@ -122,7 +122,7 @@ Front-end problems: none. | `store.push` | name, machine | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.pull` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.remove` | name | - | yes | yes | yes | yes | admin | -| `store.compact` | name | - | yes | yes | yes | yes | admin | +| `store.compact` | name | on_progress() | yes | yes | yes | yes (--progress) | admin | | `store.disconnect` | name | - | yes | yes | yes | yes | admin | | `backup.connect` | name, url | key, region, part_size | yes | yes | yes | yes | admin | | `backup.disconnect` | name | - | yes | yes | yes | yes | admin | diff --git a/docs/architecture/api/calls/errors-and-events.md b/docs/architecture/api/calls/errors-and-events.md index 20db4eb37..445fc740b 100644 --- a/docs/architecture/api/calls/errors-and-events.md +++ b/docs/architecture/api/calls/errors-and-events.md @@ -128,9 +128,9 @@ over [`Events`](../../../../control/crates/layerfs-home/src/events.rs): it came, `output(stream, bytes)` sends `{ stream, bytes }`. Events are sent on the call's own thread, before it returns. -Eleven calls declare the shared `PROGRESS` option (`on_progress`; CLI +Twelve calls declare the shared `PROGRESS` option (`on_progress`; CLI `--progress`) in [`calls/table.rs`](../../../../api/crates/layerfs-api/src/calls/table.rs): -`project.init`, `project.export`, `store.push`, `store.pull`, +`project.init`, `project.export`, `store.push`, `store.pull`, `store.compact`, `backup.create`, `backup.restore`, `machine.start`, `machine.create`, `machine.load`, `sandbox.create` and `workspace.exec`. `workspace.exec` alone also declares the `stdout` and `stderr` functions. A function never crosses into the library: @@ -139,7 +139,7 @@ refuses one as a value. | Shape | Calls | Source | | --- | --- | --- | -| `{ phase, done, total }` | `project.init`, `store.push`, `store.pull` | [`sqlite/family.rs`](../../../../control/crates/layerfs-store/src/sqlite/family.rs) `progress` | +| `{ phase, done, total }` | `project.init`, `store.push`, `store.pull`, `store.compact` | [`sqlite/family.rs`](../../../../control/crates/layerfs-store/src/sqlite/family.rs) `progress` | | `{ phase, files, directories, symlinks, bytes, done, total }` | `project.export` | [`project/export/writer.rs`](../../../../control/crates/layerfs-store/src/project/export/writer.rs) `phase_event` | | `{ phase, done, total, pieces }` | `backup.create`, `backup.restore` | [`backup/progress.rs`](../../../../control/crates/layerfs-store/src/backup/progress.rs) | | `{ exec: { id } }`, once, before the command starts | `workspace.exec` | [`workspace/exec.rs`](../../../../control/crates/layerfs-sandbox/src/workspace/exec.rs) | @@ -150,6 +150,7 @@ refuses one as a value. | --- | --- | | `project.init` | `check`, `send` (worker Stores only: first `done` 0, then one per 8 MiB of tree records, last `done == total`), `install`, `record` | | `store.push`, `store.pull` | `check`, `snapshot` (marker, then byte counts), `send` (8 MiB chunks, [`steps.rs`](../../../../control/crates/layerfs-store/src/sqlite/transfer/steps.rs) `CHUNK`), `verify` (marker, then byte counts), `install`, `record`, `retire` | +| `store.compact` | SQLite: `mark`, `build`, `verify`, `install`; PostgreSQL: `mark`, `build`, `swap`, `cleanup`. One event as each phase starts, with null counts; a run with nothing to rebuild sends `mark` alone. The SQLite settle (`cleanup` in `store.status`) is inside `install` | | `project.export` | `prepare`, `export` (first at zero, then at most one per 200 ms, last at the final counts), `finalize` (after the staging folder is renamed onto the destination) | | `backup.create` | `check`, `snapshot`, `send` (worker Stores only), `check`, `hash`, `upload` | | `backup.restore` | `check`, `download`, `verify`, `install` | diff --git a/docs/architecture/api/sdk/store.md b/docs/architecture/api/sdk/store.md index df826db40..fa27c06d4 100644 --- a/docs/architecture/api/sdk/store.md +++ b/docs/architecture/api/sdk/store.md @@ -426,11 +426,18 @@ call on that machine; the deviation and its owner-sign-off request are in D5, and the control route table names the same exception. ```text -JS await layerfs.store.compact(name) -Python layerfs.store.compact(name) -CLI layerfs store compact +JS await layerfs.store.compact(name, { on_progress }) +Python layerfs.store.compact(name, on_progress=None) +CLI layerfs store compact [--progress] ``` +`on_progress` receives `{ phase, done, total }` as each phase starts, with +null counts: `mark`, `build`, `verify`, `install` on SQLite and `mark`, +`build`, `swap`, `cleanup` on PostgreSQL; a run that takes the `none` path +sends `mark` alone. The events are for the caller only and are held in memory: +the run's op record is what [`store.status`](#storestatus) shows to another +process and what a later call judges after a crash (#42 A13). + Returns `{ kind: "Compacted", path, generation: {before, after}, ... }` with the report's snake_case fields: `dead_bytes_bound`, `dead_bytes_refined`, `logical_discarded`, `backend_reusable`, diff --git a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs index 44ca400df..8d80a115a 100644 --- a/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs +++ b/engine/crates/layerfs-persistence/src/backend/postgresql/compact/run.rs @@ -218,7 +218,18 @@ impl PostgresHandles { /// deletes only the persisted dead set. The caller keeps its session; the /// Store is usable at `G + 1` from the swap on. pub fn compact(&self, request: &CompactRequest) -> HistoryResult { - let mut metrics = RunMetrics::start(Some(0)); + self.compact_observed(request, &mut |_| {}) + } + + /// [`Self::compact`], telling `on_phase` each phase as it starts (`mark`, + /// `build`, `swap`, `cleanup`). The observer changes nothing about the + /// run; `store.status` stays the record another process reads. + pub fn compact_observed( + &self, + request: &CompactRequest, + on_phase: &mut dyn FnMut(&'static str), + ) -> HistoryResult { + let mut metrics = RunMetrics::start(Some(0), on_phase); let result = self.compact_measured(request, &mut metrics); metrics.finish(&result); result diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs index 7d8bc1d50..3b591c098 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/compact/run.rs @@ -59,8 +59,19 @@ impl SqliteHandles { /// directory-fsync failure is installed-but-uncertain: the new file and the /// op record are kept for the next `store.compact` to judge (§9). pub fn compact(self, request: &CompactRequest) -> HistoryResult { + self.compact_observed(request, &mut |_| {}) + } + + /// [`Self::compact`], telling `on_phase` each phase as it starts (`mark`, + /// `build`, `verify`, `install`). The observer changes nothing about the + /// run; `store.status` stays the record another process reads. + pub fn compact_observed( + self, + request: &CompactRequest, + on_phase: &mut dyn FnMut(&'static str), + ) -> HistoryResult { let store = self.storage.session.path.clone(); - let mut metrics = RunMetrics::start(None); + let mut metrics = RunMetrics::start(None, on_phase); let result = self.compact_measured(request, &mut metrics); // §9: the run is replying, so its own record (when it leaves one) is // no longer in flight. Mark it before the reply so the next diff --git a/engine/crates/layerfs-persistence/src/compact_metrics.rs b/engine/crates/layerfs-persistence/src/compact_metrics.rs index 4d40f3769..c70f4082e 100644 --- a/engine/crates/layerfs-persistence/src/compact_metrics.rs +++ b/engine/crates/layerfs-persistence/src/compact_metrics.rs @@ -91,6 +91,20 @@ impl CompactPhase { Self::PostgresCleanup => "pg_cleanup", } } + + /// The phase as `store.status` and a progress event name it (§11): the + /// two backends share `mark`, `build` and `cleanup`. + #[must_use] + pub const fn step(self) -> &'static str { + match self { + Self::Mark | Self::PostgresMark => "mark", + Self::Build | Self::PostgresBuild => "build", + Self::Verify => "verify", + Self::Install => "install", + Self::PostgresSwap => "swap", + Self::PostgresCleanup => "cleanup", + } + } } /// A refusal that stops a compaction before or during its admission. @@ -596,7 +610,10 @@ impl Drop for CompactMetricsCapture { /// The phase clock and §11 values one measured `store.compact` run observes, /// on either backend. The SQLite §7 phases and the PostgreSQL `pg_` phases are /// distinct [`CompactPhase`] values, so one clock serves both. -pub(crate) struct RunMetrics { +pub(crate) struct RunMetrics<'a> { + /// Told each phase as it starts, once per change of phase. It observes + /// only: it runs on the calling thread, in memory, and writes no record. + on_phase: &'a mut dyn FnMut(&'static str), pub(crate) started: Instant, pub(crate) phase: Option<(CompactPhase, Instant)>, pub(crate) phase_nanos: Vec<(CompactPhase, u64)>, @@ -613,9 +630,13 @@ pub(crate) struct RunMetrics { pub(crate) mark_bytes: Option, } -impl RunMetrics { - pub(crate) fn start(mark_bytes: Option) -> Self { +impl<'a> RunMetrics<'a> { + pub(crate) fn start( + mark_bytes: Option, + on_phase: &'a mut dyn FnMut(&'static str), + ) -> Self { Self { + on_phase, started: Instant::now(), phase: None, phase_nanos: Vec::new(), @@ -634,6 +655,9 @@ impl RunMetrics { /// time, never an exclusive CPU decomposition. pub(crate) fn begin_phase(&mut self, phase: CompactPhase) { self.end_phase(); + if self.phase_nanos.last().map(|(last, _)| *last) != Some(phase) { + (self.on_phase)(phase.step()); + } self.phase = Some((phase, Instant::now())); } diff --git a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs index 53ded67ff..e2db61642 100644 --- a/engine/crates/layerfs-persistence/tests/postgresql_compact.rs +++ b/engine/crates/layerfs-persistence/tests/postgresql_compact.rs @@ -826,6 +826,32 @@ fn a_wholly_dead_pack_is_rebuilt_away_and_totals_settle() { clean(&schema.open()); } +/// An observed run is told each phase once, in order, as it starts, by the +/// names `store.status` shows; a run with nothing to rebuild stops at `mark`. +#[test] +fn an_observed_run_is_told_each_phase_once_in_order() { + let Some(url) = url() else { return }; + let schema = Schema::new(&url, "observed"); + let handles = schema.create(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let mut phases = Vec::new(); + let receipt = handles + .compact_observed(&request(77), &mut |phase| phases.push(phase)) + .unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(phases, ["mark"]); + + let _ = whole_file(&handles, b"nobody reaches this object"); + let mut phases = Vec::new(); + let receipt = handles + .compact_observed(&request(78), &mut |phase| phases.push(phase)) + .unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(phases, ["mark", "build", "swap", "cleanup"]); + clean(&schema.open()); +} + #[test] fn retained_roots_survive_a_rebuild_that_collects_a_deleted_project() { let Some(url) = url() else { return }; diff --git a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs index 4b051aab9..d48b1d7ac 100644 --- a/engine/crates/layerfs-persistence/tests/sqlite_compact.rs +++ b/engine/crates/layerfs-persistence/tests/sqlite_compact.rs @@ -931,6 +931,30 @@ fn the_receipt_carries_every_section_11_field() { ); } +/// An observed run is told each phase once, in order, as it starts, by the +/// names `store.status` shows; a run with nothing to rebuild stops at `mark`. +#[test] +fn an_observed_run_is_told_each_phase_once_in_order() { + let (_temp, handles, _path) = create_store(); + let live = filesystem_root(&handles, 1); + initialize(&handles, 1, live); + let _ = whole_file(&handles, b"dead"); + let mut phases = Vec::new(); + let receipt = handles + .compact_observed(&request(34), &mut |phase| phases.push(phase)) + .unwrap(); + assert_eq!(receipt.path, CompactPath::Rebuild); + assert_eq!(phases, ["mark", "build", "verify", "install"]); + + let (_temp, handles, _path) = create_store(); + let mut phases = Vec::new(); + let receipt = handles + .compact_observed(&request(35), &mut |phase| phases.push(phase)) + .unwrap(); + assert_eq!(receipt.path, CompactPath::None); + assert_eq!(phases, ["mark"]); +} + #[test] fn the_empty_store_report_explains_its_zeros() { let (_temp, handles, path) = create_store();